From 8e79bac43d0ae23e80006c3b0d7c673a226cfee3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 19 Nov 2020 17:48:13 +0100 Subject: [PATCH] Add import/export feature for bookmarks (#14956) * Add ability to export bookmarks * Add support for importing bookmarks * Add bookmark import tests * Add bookmarks export test --- .../settings/exports/bookmarks_controller.rb | 19 ++++++++ app/models/export.rb | 12 +++++ app/models/import.rb | 2 +- app/services/import_service.rb | 45 +++++++++++++++++++ app/views/settings/exports/show.html.haml | 4 ++ config/locales/en.yml | 2 + config/routes.rb | 1 + .../exports/bookmarks_controller_specs.rb | 17 +++++++ spec/fixtures/files/bookmark-imports.txt | 4 ++ spec/services/import_service_spec.rb | 42 +++++++++++++++++ 10 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 app/controllers/settings/exports/bookmarks_controller.rb create mode 100644 spec/controllers/settings/exports/bookmarks_controller_specs.rb create mode 100644 spec/fixtures/files/bookmark-imports.txt diff --git a/app/controllers/settings/exports/bookmarks_controller.rb b/app/controllers/settings/exports/bookmarks_controller.rb new file mode 100644 index 0000000000..c12e2f147a --- /dev/null +++ b/app/controllers/settings/exports/bookmarks_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Settings + module Exports + class BookmarksController < BaseController + include ExportControllerConcern + + def index + send_export_file + end + + private + + def export_data + @export.to_bookmarks_csv + end + end + end +end diff --git a/app/models/export.rb b/app/models/export.rb index cab01f11ad..5216eed5ea 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -9,6 +9,14 @@ class Export @account = account end + def to_bookmarks_csv + CSV.generate do |csv| + account.bookmarks.includes(:status).reorder(id: :desc).each do |bookmark| + csv << [ActivityPub::TagManager.instance.uri_for(bookmark.status)] + end + end + end + def to_blocked_accounts_csv to_csv account.blocking.select(:username, :domain) end @@ -55,6 +63,10 @@ class Export account.statuses_count end + def total_bookmarks + account.bookmarks.count + end + def total_follows account.following_count end diff --git a/app/models/import.rb b/app/models/import.rb index c78a04d073..7024532895 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -24,7 +24,7 @@ class Import < ApplicationRecord belongs_to :account - enum type: [:following, :blocking, :muting, :domain_blocking] + enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks] validates :type, presence: true diff --git a/app/services/import_service.rb b/app/services/import_service.rb index 7e55452de7..288e47f1ea 100644 --- a/app/services/import_service.rb +++ b/app/services/import_service.rb @@ -18,6 +18,8 @@ class ImportService < BaseService import_mutes! when 'domain_blocking' import_domain_blocks! + when 'bookmarks' + import_bookmarks! end end @@ -88,6 +90,39 @@ class ImportService < BaseService end end + def import_bookmarks! + parse_import_data!(['#uri']) + items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip } + + if @import.overwrite? + presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } + + @account.bookmarks.find_each do |bookmark| + if presence_hash[bookmark.status.uri] + items.delete(bookmark.status.uri) + else + bookmark.destroy! + end + end + end + + statuses = items.map do |uri| + status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status) + next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri) + + status || ActivityPub::FetchRemoteStatusService.new.call(uri) + end.compact + + account_ids = statuses.map(&:account_id) + preloaded_relations = relations_map_for_account(@account, account_ids) + + statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? } + + statuses.each do |status| + @account.bookmarks.find_or_create_by!(account: @account, status: status) + end + end + def parse_import_data!(default_headers) data = CSV.parse(import_data, headers: true) data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ') @@ -101,4 +136,14 @@ class ImportService < BaseService def follow_limit FollowLimitValidator.limit_for_account(@account) end + + def relations_map_for_account(account, account_ids) + { + blocking: {}, + blocked_by: Account.blocked_by_map(account_ids, account.id), + muting: {}, + following: Account.following_map(account_ids, account.id), + domain_blocking_by_domain: {}, + } + end end diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index 0bb80e9372..18b52c0c2c 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -36,6 +36,10 @@ %th= t('exports.domain_blocks') %td= number_with_delimiter @export.total_domain_blocks %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv) + %tr + %th= t('exports.bookmarks') + %td= number_with_delimiter @export.total_bookmarks + %td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv) %hr.spacer/ diff --git a/config/locales/en.yml b/config/locales/en.yml index bec0990827..263ffcdc77 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -842,6 +842,7 @@ en: request: Request your archive size: Size blocks: You block + bookmarks: Bookmarks csv: CSV domain_blocks: Domain blocks lists: Lists @@ -918,6 +919,7 @@ en: success: Your data was successfully uploaded and will now be processed in due time types: blocking: Blocking list + bookmarks: Bookmarks domain_blocking: Domain blocking list following: Following list muting: Muting list diff --git a/config/routes.rb b/config/routes.rb index 54c76799ca..a534b433e0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -125,6 +125,7 @@ Rails.application.routes.draw do resources :mutes, only: :index, controller: :muted_accounts resources :lists, only: :index, controller: :lists resources :domain_blocks, only: :index, controller: :blocked_domains + resources :bookmarks, only: :index, controller: :bookmarks end resources :two_factor_authentication_methods, only: [:index] do diff --git a/spec/controllers/settings/exports/bookmarks_controller_specs.rb b/spec/controllers/settings/exports/bookmarks_controller_specs.rb new file mode 100644 index 0000000000..85761577bd --- /dev/null +++ b/spec/controllers/settings/exports/bookmarks_controller_specs.rb @@ -0,0 +1,17 @@ +require 'rails_helper' + +describe Settings::Exports::BookmarksController do + render_views + + describe 'GET #index' do + it 'returns a csv of the bookmarked toots' do + user = Fabricate(:user) + user.account.bookmarks.create!(status: Fabricate(:status, uri: 'https://foo.bar/statuses/1312')) + + sign_in user, scope: :user + get :index, format: :csv + + expect(response.body).to eq "https://foo.bar/statuses/1312\n" + end + end +end diff --git a/spec/fixtures/files/bookmark-imports.txt b/spec/fixtures/files/bookmark-imports.txt new file mode 100644 index 0000000000..7cc8901a0a --- /dev/null +++ b/spec/fixtures/files/bookmark-imports.txt @@ -0,0 +1,4 @@ +https://example.com/statuses/1312 +https://local.com/users/foo/statuses/42 +https://unknown-remote.com/users/bar/statuses/1 +https://example.com/statuses/direct diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb index b1909d4fd1..764225aa72 100644 --- a/spec/services/import_service_spec.rb +++ b/spec/services/import_service_spec.rb @@ -1,6 +1,8 @@ require 'rails_helper' RSpec.describe ImportService, type: :service do + include RoutingHelper + let!(:account) { Fabricate(:account, locked: false) } let!(:bob) { Fabricate(:account, username: 'bob', locked: false) } let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') } @@ -169,4 +171,44 @@ RSpec.describe ImportService, type: :service do end end end + + context 'import bookmarks' do + subject { ImportService.new } + + let(:csv) { attachment_fixture('bookmark-imports.txt') } + + around(:each) do |example| + local_before = Rails.configuration.x.local_domain + web_before = Rails.configuration.x.web_domain + Rails.configuration.x.local_domain = 'local.com' + Rails.configuration.x.web_domain = 'local.com' + example.run + Rails.configuration.x.web_domain = web_before + Rails.configuration.x.local_domain = local_before + end + + let(:local_account) { Fabricate(:account, username: 'foo', domain: '') } + let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') } + let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) } + + before do + service = double + allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service) + allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do + Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1') + end + end + + describe 'when no bookmarks are set' do + let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) } + it 'adds the toots the user has access to to bookmarks' do + local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true) + subject.call(import) + expect(account.bookmarks.map(&:status).map(&:id)).to include(local_status.id) + expect(account.bookmarks.map(&:status).map(&:id)).to include(remote_status.id) + expect(account.bookmarks.map(&:status).map(&:id)).not_to include(direct_status.id) + expect(account.bookmarks.count).to eq 3 + end + end + end end