From 5c9abdeff1d0cf3e14d84c5ae298e6a5beccaf18 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 27 Sep 2022 03:08:19 +0200 Subject: [PATCH] Add retention policy for cached content and media (#19232) --- app/lib/redis_configuration.rb | 6 +- app/lib/vacuum.rb | 3 + .../vacuum/access_tokens_vacuum.rb} | 17 ++++-- app/lib/vacuum/backups_vacuum.rb | 25 +++++++++ app/lib/vacuum/feeds_vacuum.rb | 34 +++++++++++ app/lib/vacuum/media_attachments_vacuum.rb | 40 +++++++++++++ app/lib/vacuum/preview_cards_vacuum.rb | 39 +++++++++++++ app/lib/vacuum/statuses_vacuum.rb | 54 ++++++++++++++++++ app/lib/vacuum/system_keys_vacuum.rb | 13 +++++ app/models/content_retention_policy.rb | 25 +++++++++ app/models/form/admin_settings.rb | 4 ++ app/views/admin/settings/edit.html.haml | 8 ++- .../scheduler/backup_cleanup_scheduler.rb | 17 ------ .../scheduler/feed_cleanup_scheduler.rb | 35 ------------ .../scheduler/media_cleanup_scheduler.rb | 17 ------ app/workers/scheduler/vacuum_scheduler.rb | 56 +++++++++++++++++++ config/locales/simple_form.en.yml | 8 +++ config/settings.yml | 1 + config/sidekiq.yml | 16 +----- spec/fabricators/access_grant_fabricator.rb | 6 ++ spec/fabricators/preview_card_fabricator.rb | 1 + spec/lib/vacuum/access_tokens_vacuum_spec.rb | 33 +++++++++++ spec/lib/vacuum/backups_vacuum_spec.rb | 24 ++++++++ spec/lib/vacuum/feeds_vacuum_spec.rb | 30 ++++++++++ .../vacuum/media_attachments_vacuum_spec.rb | 47 ++++++++++++++++ spec/lib/vacuum/preview_cards_vacuum_spec.rb | 36 ++++++++++++ spec/lib/vacuum/statuses_vacuum_spec.rb | 36 ++++++++++++ spec/lib/vacuum/system_keys_vacuum_spec.rb | 22 ++++++++ .../scheduler/feed_cleanup_scheduler_spec.rb | 26 --------- .../scheduler/media_cleanup_scheduler_spec.rb | 15 ----- 30 files changed, 559 insertions(+), 135 deletions(-) create mode 100644 app/lib/vacuum.rb rename app/{workers/scheduler/doorkeeper_cleanup_scheduler.rb => lib/vacuum/access_tokens_vacuum.rb} (56%) create mode 100644 app/lib/vacuum/backups_vacuum.rb create mode 100644 app/lib/vacuum/feeds_vacuum.rb create mode 100644 app/lib/vacuum/media_attachments_vacuum.rb create mode 100644 app/lib/vacuum/preview_cards_vacuum.rb create mode 100644 app/lib/vacuum/statuses_vacuum.rb create mode 100644 app/lib/vacuum/system_keys_vacuum.rb create mode 100644 app/models/content_retention_policy.rb delete mode 100644 app/workers/scheduler/backup_cleanup_scheduler.rb delete mode 100644 app/workers/scheduler/feed_cleanup_scheduler.rb delete mode 100644 app/workers/scheduler/media_cleanup_scheduler.rb create mode 100644 app/workers/scheduler/vacuum_scheduler.rb create mode 100644 spec/fabricators/access_grant_fabricator.rb create mode 100644 spec/lib/vacuum/access_tokens_vacuum_spec.rb create mode 100644 spec/lib/vacuum/backups_vacuum_spec.rb create mode 100644 spec/lib/vacuum/feeds_vacuum_spec.rb create mode 100644 spec/lib/vacuum/media_attachments_vacuum_spec.rb create mode 100644 spec/lib/vacuum/preview_cards_vacuum_spec.rb create mode 100644 spec/lib/vacuum/statuses_vacuum_spec.rb create mode 100644 spec/lib/vacuum/system_keys_vacuum_spec.rb delete mode 100644 spec/workers/scheduler/feed_cleanup_scheduler_spec.rb delete mode 100644 spec/workers/scheduler/media_cleanup_scheduler_spec.rb diff --git a/app/lib/redis_configuration.rb b/app/lib/redis_configuration.rb index e14d6c8b670..f0e86d985b8 100644 --- a/app/lib/redis_configuration.rb +++ b/app/lib/redis_configuration.rb @@ -7,9 +7,7 @@ class RedisConfiguration @pool = ConnectionPool.new(size: new_pool_size) { new.connection } end - def with - pool.with { |redis| yield redis } - end + delegate :with, to: :pool def pool @pool ||= establish_pool(pool_size) @@ -17,7 +15,7 @@ class RedisConfiguration def pool_size if Sidekiq.server? - Sidekiq.options[:concurrency] + Sidekiq[:concurrency] else ENV['MAX_THREADS'] || 5 end diff --git a/app/lib/vacuum.rb b/app/lib/vacuum.rb new file mode 100644 index 00000000000..9db1ec90bed --- /dev/null +++ b/app/lib/vacuum.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +module Vacuum; end diff --git a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb b/app/lib/vacuum/access_tokens_vacuum.rb similarity index 56% rename from app/workers/scheduler/doorkeeper_cleanup_scheduler.rb rename to app/lib/vacuum/access_tokens_vacuum.rb index 9303a352fb7..4f3878027ab 100644 --- a/app/workers/scheduler/doorkeeper_cleanup_scheduler.rb +++ b/app/lib/vacuum/access_tokens_vacuum.rb @@ -1,13 +1,18 @@ # frozen_string_literal: true -class Scheduler::DoorkeeperCleanupScheduler - include Sidekiq::Worker - - sidekiq_options retry: 0 - +class Vacuum::AccessTokensVacuum def perform + vacuum_revoked_access_tokens! + vacuum_revoked_access_grants! + end + + private + + def vacuum_revoked_access_tokens! Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all + end + + def vacuum_revoked_access_grants! Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all - SystemKey.expired.delete_all end end diff --git a/app/lib/vacuum/backups_vacuum.rb b/app/lib/vacuum/backups_vacuum.rb new file mode 100644 index 00000000000..3b83072f369 --- /dev/null +++ b/app/lib/vacuum/backups_vacuum.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class Vacuum::BackupsVacuum + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_expired_backups! if retention_period? + end + + private + + def vacuum_expired_backups! + backups_past_retention_period.in_batches.destroy_all + end + + def backups_past_retention_period + Backup.unscoped.where(Backup.arel_table[:created_at].lt(@retention_period.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb new file mode 100644 index 00000000000..f46bcf75f7f --- /dev/null +++ b/app/lib/vacuum/feeds_vacuum.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class Vacuum::FeedsVacuum + def perform + vacuum_inactive_home_feeds! + vacuum_inactive_list_feeds! + end + + private + + def vacuum_inactive_home_feeds! + inactive_users.select(:id, :account_id).find_in_batches do |users| + feed_manager.clean_feeds!(:home, users.map(&:account_id)) + end + end + + def vacuum_inactive_list_feeds! + inactive_users_lists.select(:id).find_in_batches do |lists| + feed_manager.clean_feeds!(:list, lists.map(&:id)) + end + end + + def inactive_users + User.confirmed.inactive + end + + def inactive_users_lists + List.where(account_id: inactive_users.select(:account_id)) + end + + def feed_manager + FeedManager.instance + end +end diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb new file mode 100644 index 00000000000..7fb347ce4d5 --- /dev/null +++ b/app/lib/vacuum/media_attachments_vacuum.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class Vacuum::MediaAttachmentsVacuum + TTL = 1.day.freeze + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_cached_files! if retention_period? + vacuum_orphaned_records! + end + + private + + def vacuum_cached_files! + media_attachments_past_retention_period.find_each do |media_attachment| + media_attachment.file.destroy + media_attachment.thumbnail.destroy + media_attachment.save + end + end + + def vacuum_orphaned_records! + orphaned_media_attachments.in_batches.destroy_all + end + + def media_attachments_past_retention_period + MediaAttachment.unscoped.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago)) + end + + def orphaned_media_attachments + MediaAttachment.unscoped.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb new file mode 100644 index 00000000000..84ef100ed94 --- /dev/null +++ b/app/lib/vacuum/preview_cards_vacuum.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class Vacuum::PreviewCardsVacuum + TTL = 1.day.freeze + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_cached_images! if retention_period? + vacuum_orphaned_records! + end + + private + + def vacuum_cached_images! + preview_cards_past_retention_period.find_each do |preview_card| + preview_card.image.destroy + preview_card.save + end + end + + def vacuum_orphaned_records! + orphaned_preview_cards.in_batches.destroy_all + end + + def preview_cards_past_retention_period + PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago)) + end + + def orphaned_preview_cards + PreviewCard.where('NOT EXISTS (SELECT 1 FROM preview_cards_statuses WHERE preview_cards_statuses.preview_card_id = preview_cards.id)').where(PreviewCard.arel_table[:created_at].lt(TTL.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb new file mode 100644 index 00000000000..41d6ba270cc --- /dev/null +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class Vacuum::StatusesVacuum + include Redisable + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_statuses! if retention_period? + end + + private + + def vacuum_statuses! + statuses_scope.find_in_batches do |statuses| + # Side-effects not covered by foreign keys, such + # as the search index, must be handled first. + + remove_from_account_conversations(statuses) + remove_from_search_index(statuses) + + # Foreign keys take care of most associated records + # for us. Media attachments will be orphaned. + + Status.where(id: statuses.map(&:id)).delete_all + end + end + + def statuses_scope + Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility) + end + + def retention_period_as_id + Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false) + end + + def analyze_statuses! + ActiveRecord::Base.connection.execute('ANALYZE statuses') + end + + def remove_from_account_conversations(statuses) + Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations) + end + + def remove_from_search_index(statuses) + with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled? + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/system_keys_vacuum.rb b/app/lib/vacuum/system_keys_vacuum.rb new file mode 100644 index 00000000000..ceee2fd164b --- /dev/null +++ b/app/lib/vacuum/system_keys_vacuum.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class Vacuum::SystemKeysVacuum + def perform + vacuum_expired_system_keys! + end + + private + + def vacuum_expired_system_keys! + SystemKey.expired.delete_all + end +end diff --git a/app/models/content_retention_policy.rb b/app/models/content_retention_policy.rb new file mode 100644 index 00000000000..b5e922c8c5f --- /dev/null +++ b/app/models/content_retention_policy.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class ContentRetentionPolicy + def self.current + new + end + + def media_cache_retention_period + retention_period Setting.media_cache_retention_period + end + + def content_cache_retention_period + retention_period Setting.content_cache_retention_period + end + + def backups_retention_period + retention_period Setting.backups_retention_period + end + + private + + def retention_period(value) + value.days if value.is_a?(Integer) && value.positive? + end +end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index 97fabc6ac97..3a71509162c 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -32,6 +32,9 @@ class Form::AdminSettings show_domain_blocks_rationale noindex require_invite_text + media_cache_retention_period + content_cache_retention_period + backups_retention_period ).freeze BOOLEAN_KEYS = %i( @@ -64,6 +67,7 @@ class Form::AdminSettings validates :bootstrap_timeline_accounts, existing_username: { multiple: true } validates :show_domain_blocks, inclusion: { in: %w(disabled users all) } validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) } + validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true } def initialize(_attributes = {}) super diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index 64687b7a60f..1dfd216439d 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -45,7 +45,6 @@ .fields-group = f.input :require_invite_text, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.require_invite_text.title'), hint: t('admin.settings.registrations.require_invite_text.desc_html'), disabled: !approved_registrations? - .fields-group %hr.spacer/ @@ -100,5 +99,12 @@ = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 } = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html') + %hr.spacer/ + + .fields-group + = f.input :media_cache_retention_period, wrapper: :with_block_label, input_html: { pattern: '[0-9]+' } + = f.input :content_cache_retention_period, wrapper: :with_block_label, input_html: { pattern: '[0-9]+' } + = f.input :backups_retention_period, wrapper: :with_block_label, input_html: { pattern: '[0-9]+' } + .actions = f.button :button, t('generic.save_changes'), type: :submit diff --git a/app/workers/scheduler/backup_cleanup_scheduler.rb b/app/workers/scheduler/backup_cleanup_scheduler.rb deleted file mode 100644 index 85d5312c0ad..00000000000 --- a/app/workers/scheduler/backup_cleanup_scheduler.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::BackupCleanupScheduler - include Sidekiq::Worker - - sidekiq_options retry: 0 - - def perform - old_backups.reorder(nil).find_each(&:destroy!) - end - - private - - def old_backups - Backup.where('created_at < ?', 7.days.ago) - end -end diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb deleted file mode 100644 index aa0cc8b8df3..00000000000 --- a/app/workers/scheduler/feed_cleanup_scheduler.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::FeedCleanupScheduler - include Sidekiq::Worker - include Redisable - - sidekiq_options retry: 0 - - def perform - clean_home_feeds! - clean_list_feeds! - end - - private - - def clean_home_feeds! - feed_manager.clean_feeds!(:home, inactive_account_ids) - end - - def clean_list_feeds! - feed_manager.clean_feeds!(:list, inactive_list_ids) - end - - def inactive_account_ids - @inactive_account_ids ||= User.confirmed.inactive.pluck(:account_id) - end - - def inactive_list_ids - List.where(account_id: inactive_account_ids).pluck(:id) - end - - def feed_manager - FeedManager.instance - end -end diff --git a/app/workers/scheduler/media_cleanup_scheduler.rb b/app/workers/scheduler/media_cleanup_scheduler.rb deleted file mode 100644 index 24d30a6be59..00000000000 --- a/app/workers/scheduler/media_cleanup_scheduler.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::MediaCleanupScheduler - include Sidekiq::Worker - - sidekiq_options retry: 0 - - def perform - unattached_media.find_each(&:destroy) - end - - private - - def unattached_media - MediaAttachment.reorder(nil).unattached.where('created_at < ?', 1.day.ago) - end -end diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb new file mode 100644 index 00000000000..ce88ff204d3 --- /dev/null +++ b/app/workers/scheduler/vacuum_scheduler.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class Scheduler::VacuumScheduler + include Sidekiq::Worker + + sidekiq_options retry: 0 + + def perform + vacuum_operations.each do |operation| + operation.perform + rescue => e + Rails.logger.error("Error while running #{operation.class.name}: #{e}") + end + end + + private + + def vacuum_operations + [ + statuses_vacuum, + media_attachments_vacuum, + preview_cards_vacuum, + backups_vacuum, + access_tokens_vacuum, + feeds_vacuum, + ] + end + + def statuses_vacuum + Vacuum::StatusesVacuum.new(content_retention_policy.content_cache_retention_period) + end + + def media_attachments_vacuum + Vacuum::MediaAttachmentsVacuum.new(content_retention_policy.media_cache_retention_period) + end + + def preview_cards_vacuum + Vacuum::PreviewCardsVacuum.new(content_retention_policy.media_cache_retention_period) + end + + def backups_vacuum + Vacuum::BackupsVacuum.new(content_retention_policy.backups_retention_period) + end + + def access_tokens_vacuum + Vacuum::AccessTokensVacuum.new + end + + def feeds_vacuum + Vacuum::FeedsVacuum.new + end + + def content_retention_policy + ContentRetentionPolicy.current + end +end diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index ddc83e8965c..db5b45e4103 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -73,6 +73,10 @@ en: actions: hide: Completely hide the filtered content, behaving as if it did not exist warn: Hide the filtered content behind a warning mentioning the filter's title + form_admin_settings: + backups_retention_period: Keep generated user archives for the specified number of days. + content_cache_retention_period: Posts from other servers will be deleted after the specified number of days when set to a positive value. This may be irreversible. + media_cache_retention_period: Downloaded media files will be deleted after the specified number of days when set to a positive value, and re-downloaded on demand. form_challenge: current_password: You are entering a secure area imports: @@ -207,6 +211,10 @@ en: actions: hide: Hide completely warn: Hide with a warning + form_admin_settings: + backups_retention_period: User archive retention period + content_cache_retention_period: Content cache retention period + media_cache_retention_period: Media cache retention period interactions: must_be_follower: Block notifications from non-followers must_be_following: Block notifications from people you don't follow diff --git a/config/settings.yml b/config/settings.yml index eaa05071e11..41742118bc8 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -70,6 +70,7 @@ defaults: &defaults show_domain_blocks: 'disabled' show_domain_blocks_rationale: 'disabled' require_invite_text: false + backups_retention_period: 7 development: <<: *defaults diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 9ec6eb5eca7..e3156aa346c 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -25,22 +25,14 @@ every: '5m' class: Scheduler::IndexingScheduler queue: scheduler - media_cleanup_scheduler: + vacuum_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' - class: Scheduler::MediaCleanupScheduler - queue: scheduler - feed_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * *' - class: Scheduler::FeedCleanupScheduler + class: Scheduler::VacuumScheduler queue: scheduler follow_recommendations_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(6..9) %> * * *' class: Scheduler::FollowRecommendationsScheduler queue: scheduler - doorkeeper_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * 0' - class: Scheduler::DoorkeeperCleanupScheduler - queue: scheduler user_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *' class: Scheduler::UserCleanupScheduler @@ -49,10 +41,6 @@ cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler queue: scheduler - backup_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' - class: Scheduler::BackupCleanupScheduler - queue: scheduler pghero_scheduler: cron: '0 0 * * *' class: Scheduler::PgheroScheduler diff --git a/spec/fabricators/access_grant_fabricator.rb b/spec/fabricators/access_grant_fabricator.rb new file mode 100644 index 00000000000..ae1945f2bb1 --- /dev/null +++ b/spec/fabricators/access_grant_fabricator.rb @@ -0,0 +1,6 @@ +Fabricator :access_grant, from: 'Doorkeeper::AccessGrant' do + application + resource_owner_id { Fabricate(:user).id } + expires_in 3_600 + redirect_uri { Doorkeeper.configuration.native_redirect_uri } +end diff --git a/spec/fabricators/preview_card_fabricator.rb b/spec/fabricators/preview_card_fabricator.rb index f119c117d98..99b5edc4357 100644 --- a/spec/fabricators/preview_card_fabricator.rb +++ b/spec/fabricators/preview_card_fabricator.rb @@ -3,4 +3,5 @@ Fabricator(:preview_card) do title { Faker::Lorem.sentence } description { Faker::Lorem.paragraph } type 'link' + image { attachment_fixture('attachment.jpg') } end diff --git a/spec/lib/vacuum/access_tokens_vacuum_spec.rb b/spec/lib/vacuum/access_tokens_vacuum_spec.rb new file mode 100644 index 00000000000..0244c344926 --- /dev/null +++ b/spec/lib/vacuum/access_tokens_vacuum_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe Vacuum::AccessTokensVacuum do + subject { described_class.new } + + describe '#perform' do + let!(:revoked_access_token) { Fabricate(:access_token, revoked_at: 1.minute.ago) } + let!(:active_access_token) { Fabricate(:access_token) } + + let!(:revoked_access_grant) { Fabricate(:access_grant, revoked_at: 1.minute.ago) } + let!(:active_access_grant) { Fabricate(:access_grant) } + + before do + subject.perform + end + + it 'deletes revoked access tokens' do + expect { revoked_access_token.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'deletes revoked access grants' do + expect { revoked_access_grant.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete active access tokens' do + expect { active_access_token.reload }.to_not raise_error + end + + it 'does not delete active access grants' do + expect { active_access_grant.reload }.to_not raise_error + end + end +end diff --git a/spec/lib/vacuum/backups_vacuum_spec.rb b/spec/lib/vacuum/backups_vacuum_spec.rb new file mode 100644 index 00000000000..4e2de083f8c --- /dev/null +++ b/spec/lib/vacuum/backups_vacuum_spec.rb @@ -0,0 +1,24 @@ +require 'rails_helper' + +RSpec.describe Vacuum::BackupsVacuum do + let(:retention_period) { 7.days } + + subject { described_class.new(retention_period) } + + describe '#perform' do + let!(:expired_backup) { Fabricate(:backup, created_at: (retention_period + 1.day).ago) } + let!(:current_backup) { Fabricate(:backup) } + + before do + subject.perform + end + + it 'deletes backups past the retention period' do + expect { expired_backup.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete backups within the retention period' do + expect { current_backup.reload }.to_not raise_error + end + end +end diff --git a/spec/lib/vacuum/feeds_vacuum_spec.rb b/spec/lib/vacuum/feeds_vacuum_spec.rb new file mode 100644 index 00000000000..0aec26740f3 --- /dev/null +++ b/spec/lib/vacuum/feeds_vacuum_spec.rb @@ -0,0 +1,30 @@ +require 'rails_helper' + +RSpec.describe Vacuum::FeedsVacuum do + subject { described_class.new } + + describe '#perform' do + let!(:active_user) { Fabricate(:user, current_sign_in_at: 2.days.ago) } + let!(:inactive_user) { Fabricate(:user, current_sign_in_at: 22.days.ago) } + + before do + redis.zadd(feed_key_for(inactive_user), 1, 1) + redis.zadd(feed_key_for(active_user), 1, 1) + redis.zadd(feed_key_for(inactive_user, 'reblogs'), 2, 2) + redis.sadd(feed_key_for(inactive_user, 'reblogs:2'), 3) + + subject.perform + end + + it 'clears feeds of inactive users and lists' do + expect(redis.zcard(feed_key_for(inactive_user))).to eq 0 + expect(redis.zcard(feed_key_for(active_user))).to eq 1 + expect(redis.exists?(feed_key_for(inactive_user, 'reblogs'))).to be false + expect(redis.exists?(feed_key_for(inactive_user, 'reblogs:2'))).to be false + end + end + + def feed_key_for(user, subtype = nil) + FeedManager.instance.key(:home, user.account_id, subtype) + end +end diff --git a/spec/lib/vacuum/media_attachments_vacuum_spec.rb b/spec/lib/vacuum/media_attachments_vacuum_spec.rb new file mode 100644 index 00000000000..be8458d9bf7 --- /dev/null +++ b/spec/lib/vacuum/media_attachments_vacuum_spec.rb @@ -0,0 +1,47 @@ +require 'rails_helper' + +RSpec.describe Vacuum::MediaAttachmentsVacuum do + let(:retention_period) { 7.days } + + subject { described_class.new(retention_period) } + + let(:remote_status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com')) } + let(:local_status) { Fabricate(:status) } + + describe '#perform' do + let!(:old_remote_media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', status: remote_status, created_at: (retention_period + 1.day).ago, updated_at: (retention_period + 1.day).ago) } + let!(:old_local_media) { Fabricate(:media_attachment, status: local_status, created_at: (retention_period + 1.day).ago, updated_at: (retention_period + 1.day).ago) } + let!(:new_remote_media) { Fabricate(:media_attachment, remote_url: 'https://example.com/foo.png', status: remote_status) } + let!(:new_local_media) { Fabricate(:media_attachment, status: local_status) } + let!(:old_unattached_media) { Fabricate(:media_attachment, account_id: nil, created_at: 10.days.ago) } + let!(:new_unattached_media) { Fabricate(:media_attachment, account_id: nil, created_at: 1.hour.ago) } + + before do + subject.perform + end + + it 'deletes cache of remote media attachments past the retention period' do + expect(old_remote_media.reload.file).to be_blank + end + + it 'does not touch local media attachments past the retention period' do + expect(old_local_media.reload.file).to_not be_blank + end + + it 'does not delete cache of remote media attachments within the retention period' do + expect(new_remote_media.reload.file).to_not be_blank + end + + it 'does not touch local media attachments within the retention period' do + expect(new_local_media.reload.file).to_not be_blank + end + + it 'deletes unattached media attachments past TTL' do + expect { old_unattached_media.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'does not delete unattached media attachments within TTL' do + expect(new_unattached_media.reload).to be_persisted + end + end +end diff --git a/spec/lib/vacuum/preview_cards_vacuum_spec.rb b/spec/lib/vacuum/preview_cards_vacuum_spec.rb new file mode 100644 index 00000000000..4a4a599fa47 --- /dev/null +++ b/spec/lib/vacuum/preview_cards_vacuum_spec.rb @@ -0,0 +1,36 @@ +require 'rails_helper' + +RSpec.describe Vacuum::PreviewCardsVacuum do + let(:retention_period) { 7.days } + + subject { described_class.new(retention_period) } + + describe '#perform' do + let!(:orphaned_preview_card) { Fabricate(:preview_card, created_at: 2.days.ago) } + let!(:old_preview_card) { Fabricate(:preview_card, updated_at: (retention_period + 1.day).ago) } + let!(:new_preview_card) { Fabricate(:preview_card) } + + before do + old_preview_card.statuses << Fabricate(:status) + new_preview_card.statuses << Fabricate(:status) + + subject.perform + end + + it 'deletes cache of preview cards last updated before the retention period' do + expect(old_preview_card.reload.image).to be_blank + end + + it 'does not delete cache of preview cards last updated within the retention period' do + expect(new_preview_card.reload.image).to_not be_blank + end + + it 'does not delete attached preview cards' do + expect(new_preview_card.reload).to be_persisted + end + + it 'deletes preview cards not attached to any status' do + expect { orphaned_preview_card.reload }.to raise_error ActiveRecord::RecordNotFound + end + end +end diff --git a/spec/lib/vacuum/statuses_vacuum_spec.rb b/spec/lib/vacuum/statuses_vacuum_spec.rb new file mode 100644 index 00000000000..83f3c5c9f14 --- /dev/null +++ b/spec/lib/vacuum/statuses_vacuum_spec.rb @@ -0,0 +1,36 @@ +require 'rails_helper' + +RSpec.describe Vacuum::StatusesVacuum do + let(:retention_period) { 7.days } + + let(:remote_account) { Fabricate(:account, domain: 'example.com') } + + subject { described_class.new(retention_period) } + + describe '#perform' do + let!(:remote_status_old) { Fabricate(:status, account: remote_account, created_at: (retention_period + 2.days).ago) } + let!(:remote_status_recent) { Fabricate(:status, account: remote_account, created_at: (retention_period - 2.days).ago) } + let!(:local_status_old) { Fabricate(:status, created_at: (retention_period + 2.days).ago) } + let!(:local_status_recent) { Fabricate(:status, created_at: (retention_period - 2.days).ago) } + + before do + subject.perform + end + + it 'deletes remote statuses past the retention period' do + expect { remote_status_old.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete local statuses past the retention period' do + expect { local_status_old.reload }.to_not raise_error + end + + it 'does not delete remote statuses within the retention period' do + expect { remote_status_recent.reload }.to_not raise_error + end + + it 'does not delete local statuses within the retention period' do + expect { local_status_recent.reload }.to_not raise_error + end + end +end diff --git a/spec/lib/vacuum/system_keys_vacuum_spec.rb b/spec/lib/vacuum/system_keys_vacuum_spec.rb new file mode 100644 index 00000000000..565892f0252 --- /dev/null +++ b/spec/lib/vacuum/system_keys_vacuum_spec.rb @@ -0,0 +1,22 @@ +require 'rails_helper' + +RSpec.describe Vacuum::SystemKeysVacuum do + subject { described_class.new } + + describe '#perform' do + let!(:expired_system_key) { Fabricate(:system_key, created_at: (SystemKey::ROTATION_PERIOD * 4).ago) } + let!(:current_system_key) { Fabricate(:system_key) } + + before do + subject.perform + end + + it 'deletes the expired key' do + expect { expired_system_key.reload }.to raise_error ActiveRecord::RecordNotFound + end + + it 'does not delete the current key' do + expect { current_system_key.reload }.to_not raise_error + end + end +end diff --git a/spec/workers/scheduler/feed_cleanup_scheduler_spec.rb b/spec/workers/scheduler/feed_cleanup_scheduler_spec.rb deleted file mode 100644 index 82d7945946c..00000000000 --- a/spec/workers/scheduler/feed_cleanup_scheduler_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -require 'rails_helper' - -describe Scheduler::FeedCleanupScheduler do - subject { described_class.new } - - let!(:active_user) { Fabricate(:user, current_sign_in_at: 2.days.ago) } - let!(:inactive_user) { Fabricate(:user, current_sign_in_at: 22.days.ago) } - - it 'clears feeds of inactives' do - redis.zadd(feed_key_for(inactive_user), 1, 1) - redis.zadd(feed_key_for(active_user), 1, 1) - redis.zadd(feed_key_for(inactive_user, 'reblogs'), 2, 2) - redis.sadd(feed_key_for(inactive_user, 'reblogs:2'), 3) - - subject.perform - - expect(redis.zcard(feed_key_for(inactive_user))).to eq 0 - expect(redis.zcard(feed_key_for(active_user))).to eq 1 - expect(redis.exists?(feed_key_for(inactive_user, 'reblogs'))).to be false - expect(redis.exists?(feed_key_for(inactive_user, 'reblogs:2'))).to be false - end - - def feed_key_for(user, subtype = nil) - FeedManager.instance.key(:home, user.account_id, subtype) - end -end diff --git a/spec/workers/scheduler/media_cleanup_scheduler_spec.rb b/spec/workers/scheduler/media_cleanup_scheduler_spec.rb deleted file mode 100644 index 8a0da67e1b6..00000000000 --- a/spec/workers/scheduler/media_cleanup_scheduler_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -require 'rails_helper' - -describe Scheduler::MediaCleanupScheduler do - subject { described_class.new } - - let!(:old_media) { Fabricate(:media_attachment, account_id: nil, created_at: 10.days.ago) } - let!(:new_media) { Fabricate(:media_attachment, account_id: nil, created_at: 1.hour.ago) } - - it 'removes old media records' do - subject.perform - - expect { old_media.reload }.to raise_error(ActiveRecord::RecordNotFound) - expect(new_media.reload).to be_persisted - end -end