Add separate cache directory for non-local uploads (#12821)

pull/1324/head
Eugen Rochko 2020-04-26 23:29:08 +02:00 committed by GitHub
parent 2744f61696
commit c3ca3801f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 319 additions and 105 deletions

View File

@ -3,50 +3,52 @@
# #
# Table name: accounts # Table name: accounts
# #
# id :bigint(8) not null, primary key # id :bigint(8) not null, primary key
# username :string default(""), not null # username :string default(""), not null
# domain :string # domain :string
# secret :string default(""), not null # secret :string default(""), not null
# private_key :text # private_key :text
# public_key :text default(""), not null # public_key :text default(""), not null
# remote_url :string default(""), not null # remote_url :string default(""), not null
# salmon_url :string default(""), not null # salmon_url :string default(""), not null
# hub_url :string default(""), not null # hub_url :string default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# note :text default(""), not null # note :text default(""), not null
# display_name :string default(""), not null # display_name :string default(""), not null
# uri :string default(""), not null # uri :string default(""), not null
# url :string # url :string
# avatar_file_name :string # avatar_file_name :string
# avatar_content_type :string # avatar_content_type :string
# avatar_file_size :integer # avatar_file_size :integer
# avatar_updated_at :datetime # avatar_updated_at :datetime
# header_file_name :string # header_file_name :string
# header_content_type :string # header_content_type :string
# header_file_size :integer # header_file_size :integer
# header_updated_at :datetime # header_updated_at :datetime
# avatar_remote_url :string # avatar_remote_url :string
# subscription_expires_at :datetime # subscription_expires_at :datetime
# locked :boolean default(FALSE), not null # locked :boolean default(FALSE), not null
# header_remote_url :string default(""), not null # header_remote_url :string default(""), not null
# last_webfingered_at :datetime # last_webfingered_at :datetime
# inbox_url :string default(""), not null # inbox_url :string default(""), not null
# outbox_url :string default(""), not null # outbox_url :string default(""), not null
# shared_inbox_url :string default(""), not null # shared_inbox_url :string default(""), not null
# followers_url :string default(""), not null # followers_url :string default(""), not null
# protocol :integer default("ostatus"), not null # protocol :integer default("ostatus"), not null
# memorial :boolean default(FALSE), not null # memorial :boolean default(FALSE), not null
# moved_to_account_id :bigint(8) # moved_to_account_id :bigint(8)
# featured_collection_url :string # featured_collection_url :string
# fields :jsonb # fields :jsonb
# actor_type :string # actor_type :string
# discoverable :boolean # discoverable :boolean
# also_known_as :string is an Array # also_known_as :string is an Array
# silenced_at :datetime # silenced_at :datetime
# suspended_at :datetime # suspended_at :datetime
# trust_level :integer # trust_level :integer
# hide_collections :boolean # hide_collections :boolean
# avatar_storage_schema_version :integer
# header_storage_schema_version :integer
# #
class Account < ApplicationRecord class Account < ApplicationRecord

View File

@ -3,20 +3,21 @@
# #
# Table name: custom_emojis # Table name: custom_emojis
# #
# id :bigint(8) not null, primary key # id :bigint(8) not null, primary key
# shortcode :string default(""), not null # shortcode :string default(""), not null
# domain :string # domain :string
# image_file_name :string # image_file_name :string
# image_content_type :string # image_content_type :string
# image_file_size :integer # image_file_size :integer
# image_updated_at :datetime # image_updated_at :datetime
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# disabled :boolean default(FALSE), not null # disabled :boolean default(FALSE), not null
# uri :string # uri :string
# image_remote_url :string # image_remote_url :string
# visible_in_picker :boolean default(TRUE), not null # visible_in_picker :boolean default(TRUE), not null
# category_id :bigint(8) # category_id :bigint(8)
# image_storage_schema_version :integer
# #
class CustomEmoji < ApplicationRecord class CustomEmoji < ApplicationRecord

View File

@ -3,23 +3,24 @@
# #
# Table name: media_attachments # Table name: media_attachments
# #
# id :bigint(8) not null, primary key # id :bigint(8) not null, primary key
# status_id :bigint(8) # status_id :bigint(8)
# file_file_name :string # file_file_name :string
# file_content_type :string # file_content_type :string
# file_file_size :integer # file_file_size :integer
# file_updated_at :datetime # file_updated_at :datetime
# remote_url :string default(""), not null # remote_url :string default(""), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# shortcode :string # shortcode :string
# type :integer default("image"), not null # type :integer default("image"), not null
# file_meta :json # file_meta :json
# account_id :bigint(8) # account_id :bigint(8)
# description :text # description :text
# scheduled_status_id :bigint(8) # scheduled_status_id :bigint(8)
# blurhash :string # blurhash :string
# processing :integer # processing :integer
# file_storage_schema_version :integer
# #
class MediaAttachment < ApplicationRecord class MediaAttachment < ApplicationRecord

View File

@ -3,25 +3,26 @@
# #
# Table name: preview_cards # Table name: preview_cards
# #
# id :bigint(8) not null, primary key # id :bigint(8) not null, primary key
# url :string default(""), not null # url :string default(""), not null
# title :string default(""), not null # title :string default(""), not null
# description :string default(""), not null # description :string default(""), not null
# image_file_name :string # image_file_name :string
# image_content_type :string # image_content_type :string
# image_file_size :integer # image_file_size :integer
# image_updated_at :datetime # image_updated_at :datetime
# type :integer default("link"), not null # type :integer default("link"), not null
# html :text default(""), not null # html :text default(""), not null
# author_name :string default(""), not null # author_name :string default(""), not null
# author_url :string default(""), not null # author_url :string default(""), not null
# provider_name :string default(""), not null # provider_name :string default(""), not null
# provider_url :string default(""), not null # provider_url :string default(""), not null
# width :integer default(0), not null # width :integer default(0), not null
# height :integer default(0), not null # height :integer default(0), not null
# created_at :datetime not null # created_at :datetime not null
# updated_at :datetime not null # updated_at :datetime not null
# embed_url :string default(""), not null # embed_url :string default(""), not null
# image_storage_schema_version :integer
# #
class PreviewCard < ApplicationRecord class PreviewCard < ApplicationRecord
@ -47,6 +48,10 @@ class PreviewCard < ApplicationRecord
before_save :extract_dimensions, if: :link? before_save :extract_dimensions, if: :link?
def local?
false
end
def missing_image? def missing_image?
width.present? && height.present? && image_file_name.blank? width.present? && height.present? && image_file_name.blank?
end end

View File

@ -10,9 +10,25 @@ Paperclip.interpolates :filename do |attachment, style|
end end
end end
Paperclip.interpolates :path_prefix do |attachment, style|
if attachment.storage_schema_version >= 1 && attachment.instance.respond_to?(:local?) && !attachment.instance.local?
'cache' + File::SEPARATOR
else
''
end
end
Paperclip.interpolates :url_prefix do |attachment, style|
if attachment.storage_schema_version >= 1 && attachment.instance.respond_to?(:local?) && !attachment.instance.local?
'cache/'
else
''
end
end
Paperclip::Attachment.default_options.merge!( Paperclip::Attachment.default_options.merge!(
use_timestamp: false, use_timestamp: false,
path: ':class/:attachment/:id_partition/:style/:filename', path: ':url_prefix:class/:attachment/:id_partition/:style/:filename',
storage: :fog storage: :fog
) )
@ -91,7 +107,7 @@ else
Paperclip::Attachment.default_options.merge!( Paperclip::Attachment.default_options.merge!(
storage: :filesystem, storage: :filesystem,
use_timestamp: true, use_timestamp: true,
path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':class', ':attachment', ':id_partition', ':style', ':filename'), path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':path_prefix:class', ':attachment', ':id_partition', ':style', ':filename'),
url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename', url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:url_prefix:class/:attachment/:id_partition/:style/:filename',
) )
end end

View File

@ -0,0 +1,9 @@
class AddStorageSchemaVersion < ActiveRecord::Migration[5.2]
def change
add_column :preview_cards, :image_storage_schema_version, :integer
add_column :accounts, :avatar_storage_schema_version, :integer
add_column :accounts, :header_storage_schema_version, :integer
add_column :media_attachments, :file_storage_schema_version, :integer
add_column :custom_emojis, :image_storage_schema_version, :integer
end
end

View File

@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_04_07_202420) do ActiveRecord::Schema.define(version: 2020_04_17_125749) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
@ -172,6 +172,8 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.datetime "suspended_at" t.datetime "suspended_at"
t.integer "trust_level" t.integer "trust_level"
t.boolean "hide_collections" t.boolean "hide_collections"
t.integer "avatar_storage_schema_version"
t.integer "header_storage_schema_version"
t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id" t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
@ -299,6 +301,7 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.string "image_remote_url" t.string "image_remote_url"
t.boolean "visible_in_picker", default: true, null: false t.boolean "visible_in_picker", default: true, null: false
t.bigint "category_id" t.bigint "category_id"
t.integer "image_storage_schema_version"
t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
end end
@ -464,6 +467,7 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.bigint "scheduled_status_id" t.bigint "scheduled_status_id"
t.string "blurhash" t.string "blurhash"
t.integer "processing" t.integer "processing"
t.integer "file_storage_schema_version"
t.index ["account_id"], name: "index_media_attachments_on_account_id" t.index ["account_id"], name: "index_media_attachments_on_account_id"
t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id" t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id"
t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true
@ -604,6 +608,7 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.datetime "created_at", null: false t.datetime "created_at", null: false
t.datetime "updated_at", null: false t.datetime "updated_at", null: false
t.string "embed_url", default: "", null: false t.string "embed_url", default: "", null: false
t.integer "image_storage_schema_version"
t.index ["url"], name: "index_preview_cards_on_url", unique: true t.index ["url"], name: "index_preview_cards_on_url", unique: true
end end

View File

@ -11,6 +11,7 @@ require_relative 'mastodon/statuses_cli'
require_relative 'mastodon/domains_cli' require_relative 'mastodon/domains_cli'
require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/preview_cards_cli'
require_relative 'mastodon/cache_cli' require_relative 'mastodon/cache_cli'
require_relative 'mastodon/upgrade_cli'
require_relative 'mastodon/version' require_relative 'mastodon/version'
module Mastodon module Mastodon
@ -49,6 +50,9 @@ module Mastodon
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache' desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
subcommand 'cache', Mastodon::CacheCLI subcommand 'cache', Mastodon::CacheCLI
desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
subcommand 'upgrade', Mastodon::UpgradeCLI
option :dry_run, type: :boolean option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation' desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC long_desc <<~LONG_DESC

View File

@ -10,6 +10,10 @@ Paperclip.options[:log] = false
module Mastodon module Mastodon
module CLIHelper module CLIHelper
def dry_run?
options[:dry_run]
end
def create_progress_bar(total = nil) def create_progress_bar(total = nil)
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e') ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end end

View File

@ -85,7 +85,9 @@ module Mastodon
record_map = preload_records_from_mixed_objects(objects) record_map = preload_records_from_mixed_objects(objects)
objects.each do |object| objects.each do |object|
path_segments = object.key.split('/') path_segments = object.key.split('/')
path_segments.delete('cache')
model_name = path_segments.first.classify model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize attachment_name = path_segments[1].singularize
record_id = path_segments[2..-2].join.to_i record_id = path_segments[2..-2].join.to_i
@ -120,8 +122,11 @@ module Mastodon
Find.find(File.join(*[root_path, prefix].compact)) do |path| Find.find(File.join(*[root_path, prefix].compact)) do |path|
next if File.directory?(path) next if File.directory?(path)
key = path.gsub("#{root_path}#{File::SEPARATOR}", '') key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache')
model_name = path_segments.first.classify model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i record_id = path_segments[2..-2].join.to_i
attachment_name = path_segments[1].singularize attachment_name = path_segments[1].singularize
@ -229,10 +234,13 @@ module Mastodon
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL' desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
def lookup(url) def lookup(url)
path = Addressable::URI.parse(url).path path = Addressable::URI.parse(url).path
path_segments = path.split('/')[2..-1] path_segments = path.split('/')[2..-1]
model_name = path_segments.first.classify path_segments.delete('cache')
record_id = path_segments[2..-2].join.to_i
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
unless PRELOAD_MODEL_WHITELIST.include?(model_name) unless PRELOAD_MODEL_WHITELIST.include?(model_name)
say("Cannot find corresponding model: #{model_name}", :red) say("Cannot find corresponding model: #{model_name}", :red)
@ -276,7 +284,9 @@ module Mastodon
preload_map = Hash.new { |hash, key| hash[key] = [] } preload_map = Hash.new { |hash, key| hash[key] = [] }
objects.map do |object| objects.map do |object|
segments = object.key.split('/') segments = object.key.split('/')
segments.delete('cache')
model_name = segments.first.classify model_name = segments.first.classify
record_id = segments[2..-2].join.to_i record_id = segments[2..-2].join.to_i

148
lib/mastodon/upgrade_cli.rb Normal file
View File

@ -0,0 +1,148 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module Mastodon
class UpgradeCLI < Thor
include CLIHelper
def self.exit_on_failure?
true
end
CURRENT_STORAGE_SCHEMA_VERSION = 1
option :dry_run, type: :boolean, default: false
option :verbose, type: :boolean, default: false, aliases: [:v]
desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
long_desc <<~LONG_DESC
Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.
Will most likely take a long time.
LONG_DESC
def storage_schema
progress = create_progress_bar(nil)
dry_run = dry_run? ? ' (DRY RUN)' : ''
records = 0
klasses = [
Account,
CustomEmoji,
MediaAttachment,
PreviewCard,
]
klasses.each do |klass|
attachment_names = klass.attachment_definitions.keys
klass.find_each do |record|
attachment_names.each do |attachment_name|
attachment = record.public_send(attachment_name)
next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION
attachment.styles.each_key do |style|
case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
progress.increment
end
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
end
if record.changed?
record.save unless dry_run?
records += 1
end
end
end
progress.total = progress.progress
progress.finish
say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
end
private
def upgrade_storage_s3(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
object = attachment.s3_object(style)
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
upgraded_path = attachment.path(style)
if upgraded_path != object.key && object.exists?
progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose]
begin
object.move_to(upgraded_path) unless dry_run?
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
end
end
# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
end
def upgrade_storage_fog(_progress, _attachment, _style)
say('The fog storage driver is not supported for this operation at this time', :red)
exit(1)
end
def upgrade_storage_filesystem(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
previous_path = attachment.path(style)
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
upgraded_path = attachment.path(style)
if upgraded_path != previous_path && File.exist?(previous_path)
progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]
begin
unless dry_run?
FileUtils.mkdir_p(File.dirname(upgraded_path))
FileUtils.mv(previous_path, upgraded_path)
begin
FileUtils.rmdir(previous_path, parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
rescue => e
progress.log(pastel.red("Error processing #{previous_path}: #{e}"))
unless dry_run?
begin
FileUtils.rmdir(upgraded_path, parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
end
end
# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
end
end
end

View File

@ -14,6 +14,15 @@ module Paperclip
end end
end end
def storage_schema_version
instance_read(:storage_schema_version) || 0
end
def assign_attributes
super
instance_write(:storage_schema_version, 1)
end
def variant?(other_filename) def variant?(other_filename)
return true if original_filename == other_filename return true if original_filename == other_filename
return false if original_filename.nil? return false if original_filename.nil?