Add improved CLI interface for removing remote media (#8411)

./bin/tootctl media remove --days 7 --background

Make the old rake task point to it
pull/8429/head
Eugen Rochko 2018-08-25 13:25:39 +02:00 committed by GitHub
parent b4fc810bc3
commit 793eea2982
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 68 additions and 11 deletions

View File

@ -6,7 +6,7 @@ class Maintenance::DestroyMediaWorker
sidekiq_options queue: 'pull' sidekiq_options queue: 'pull'
def perform(media_attachment_id) def perform(media_attachment_id)
media = MediaAttachment.find(media_attachment_id) media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
media.destroy media.destroy
rescue ActiveRecord::RecordNotFound rescue ActiveRecord::RecordNotFound
true true

View File

@ -6,7 +6,7 @@ class Maintenance::RedownloadAccountMediaWorker
sidekiq_options queue: 'pull', retry: false sidekiq_options queue: 'pull', retry: false
def perform(account_id) def perform(account_id)
account = Account.find(account_id) account = account_id.is_a?(Account) ? account_id : Account.find(account_id)
account.reset_avatar! account.reset_avatar!
account.reset_header! account.reset_header!
account.save account.save

View File

@ -6,7 +6,7 @@ class Maintenance::UncacheMediaWorker
sidekiq_options queue: 'pull' sidekiq_options queue: 'pull'
def perform(media_attachment_id) def perform(media_attachment_id)
media = MediaAttachment.find(media_attachment_id) media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
return if media.file.blank? return if media.file.blank?

4
bin/tootctl Executable file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env ruby
APP_PATH = File.expand_path('../config/application', __dir__)
require_relative '../lib/cli'
Mastodon::CLI.start(ARGV)

11
lib/cli.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
require 'thor'
require_relative 'mastodon/media_cli'
module Mastodon
class CLI < Thor
desc 'media SUBCOMMAND ...ARGS', 'manage media files'
subcommand 'media', Mastodon::MediaCLI
end
end

47
lib/mastodon/media_cli.rb Normal file
View File

@ -0,0 +1,47 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
# rubocop:disable Rails/Output
module Mastodon
class MediaCLI < Thor
option :days, type: :numeric, default: 7
option :background, type: :boolean, default: false
desc 'remove', 'remove remote media files'
long_desc <<-DESC
Removes locally cached copies of media attachments from other servers.
The --days option specifies how old media attachments have to be before
they are removed. It defaults to 7 days.
With the --background option, instead of deleting the files sequentially,
they will be queued into Sidekiq and the command will exit as soon as
possible. In Sidekiq they will be processed with higher concurrency, but
it may impact other operations of the Mastodon server, and it may overload
the underlying file storage.
DESC
def remove
time_ago = options[:days].days.ago
queued = 0
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
if options[:background]
queued += media_attachments.size
Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
else
media_attachments.each do |m|
Maintenance::UncacheMediaWorker.new.perform(m)
print '.'
end
end
end
puts
puts "Scheduled the deletion of #{queued} media attachments" if options[:background]
end
end
end
# rubocop:enable Rails/Output

View File

@ -512,14 +512,9 @@ namespace :mastodon do
desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)' desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
task remove_remote: :environment do task remove_remote: :environment do
time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago require_relative '../mastodon/media_cli'
nb_media_attachments = 0 cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7)
cli.invoke(:remove)
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
nb_media_attachments += media_attachments.length
Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
end
puts "Scheduled the deletion of #{nb_media_attachments} media attachments"
end end
desc 'Set unknown attachment type for remote-only attachments' desc 'Set unknown attachment type for remote-only attachments'