From 793eea29823a44fd4950f87898ecf0ff3b49351d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 25 Aug 2018 13:25:39 +0200 Subject: [PATCH] Add improved CLI interface for removing remote media (#8411) ./bin/tootctl media remove --days 7 --background Make the old rake task point to it --- .../maintenance/destroy_media_worker.rb | 2 +- .../redownload_account_media_worker.rb | 2 +- .../maintenance/uncache_media_worker.rb | 2 +- bin/tootctl | 4 ++ lib/cli.rb | 11 +++++ lib/mastodon/media_cli.rb | 47 +++++++++++++++++++ lib/tasks/mastodon.rake | 11 ++--- 7 files changed, 68 insertions(+), 11 deletions(-) create mode 100755 bin/tootctl create mode 100644 lib/cli.rb create mode 100644 lib/mastodon/media_cli.rb diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb index 5f052983be6..cde33d6d79a 100644 --- a/app/workers/maintenance/destroy_media_worker.rb +++ b/app/workers/maintenance/destroy_media_worker.rb @@ -6,7 +6,7 @@ class Maintenance::DestroyMediaWorker sidekiq_options queue: 'pull' def perform(media_attachment_id) - media = MediaAttachment.find(media_attachment_id) + media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) media.destroy rescue ActiveRecord::RecordNotFound true diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb index fc26815f25e..6afbe6e19ce 100644 --- a/app/workers/maintenance/redownload_account_media_worker.rb +++ b/app/workers/maintenance/redownload_account_media_worker.rb @@ -6,7 +6,7 @@ class Maintenance::RedownloadAccountMediaWorker sidekiq_options queue: 'pull', retry: false def perform(account_id) - account = Account.find(account_id) + account = account_id.is_a?(Account) ? account_id : Account.find(account_id) account.reset_avatar! account.reset_header! account.save diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb index 2d1a670a76c..4bc62ef755c 100644 --- a/app/workers/maintenance/uncache_media_worker.rb +++ b/app/workers/maintenance/uncache_media_worker.rb @@ -6,7 +6,7 @@ class Maintenance::UncacheMediaWorker sidekiq_options queue: 'pull' def perform(media_attachment_id) - media = MediaAttachment.find(media_attachment_id) + media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) return if media.file.blank? diff --git a/bin/tootctl b/bin/tootctl new file mode 100755 index 00000000000..2fe02523a4a --- /dev/null +++ b/bin/tootctl @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +APP_PATH = File.expand_path('../config/application', __dir__) +require_relative '../lib/cli' +Mastodon::CLI.start(ARGV) diff --git a/lib/cli.rb b/lib/cli.rb new file mode 100644 index 00000000000..7e82806b661 --- /dev/null +++ b/lib/cli.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'thor' +require_relative 'mastodon/media_cli' + +module Mastodon + class CLI < Thor + desc 'media SUBCOMMAND ...ARGS', 'manage media files' + subcommand 'media', Mastodon::MediaCLI + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb new file mode 100644 index 00000000000..cc6ad07d965 --- /dev/null +++ b/lib/mastodon/media_cli.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative '../../config/boot' +require_relative '../../config/environment' + +# rubocop:disable Rails/Output + +module Mastodon + class MediaCLI < Thor + option :days, type: :numeric, default: 7 + option :background, type: :boolean, default: false + desc 'remove', 'remove remote media files' + long_desc <<-DESC + Removes locally cached copies of media attachments from other servers. + + The --days option specifies how old media attachments have to be before + they are removed. It defaults to 7 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + DESC + def remove + time_ago = options[:days].days.ago + queued = 0 + + MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| + if options[:background] + queued += media_attachments.size + Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) + else + media_attachments.each do |m| + Maintenance::UncacheMediaWorker.new.perform(m) + print '.' + end + end + end + + puts + puts "Scheduled the deletion of #{queued} media attachments" if options[:background] + end + end +end + +# rubocop:enable Rails/Output diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 191ce634c44..9ce39335deb 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -512,14 +512,9 @@ namespace :mastodon do desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)' task remove_remote: :environment do - time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago - nb_media_attachments = 0 - - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| - nb_media_attachments += media_attachments.length - Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) - end - puts "Scheduled the deletion of #{nb_media_attachments} media attachments" + require_relative '../mastodon/media_cli' + cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7) + cli.invoke(:remove) end desc 'Set unknown attachment type for remote-only attachments'