From 71fce71c94b1e94ae3a7af17bfc141709b61c428 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 14 May 2020 23:28:06 +0200 Subject: [PATCH 1/9] Fix webfinger returning wrong status code on malformed or missing param (#13759) Fixes #13757 --- app/controllers/well_known/webfinger_controller.rb | 7 ++++++- app/lib/webfinger_resource.rb | 4 +++- .../well_known/webfinger_controller_spec.rb | 10 ++++++++++ spec/lib/webfinger_resource_spec.rb | 12 +++++++++++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb index 480e58f3f04..9de9db6ba8c 100644 --- a/app/controllers/well_known/webfinger_controller.rb +++ b/app/controllers/well_known/webfinger_controller.rb @@ -8,7 +8,8 @@ module WellKnown before_action :set_account before_action :check_account_suspension - rescue_from ActiveRecord::RecordNotFound, ActionController::ParameterMissing, with: :not_found + rescue_from ActiveRecord::RecordNotFound, with: :not_found + rescue_from ActionController::ParameterMissing, WebfingerResource::InvalidRequest, with: :bad_request def show expires_in 3.days, public: true @@ -37,6 +38,10 @@ module WellKnown expires_in(3.minutes, public: true) && gone if @account.suspended? end + def bad_request + head 400 + end + def not_found head 404 end diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb index 22d78874a4b..42094548595 100644 --- a/app/lib/webfinger_resource.rb +++ b/app/lib/webfinger_resource.rb @@ -3,6 +3,8 @@ class WebfingerResource attr_reader :resource + class InvalidRequest < StandardError; end + def initialize(resource) @resource = resource end @@ -14,7 +16,7 @@ class WebfingerResource when /\@/ username_from_acct else - raise(ActiveRecord::RecordNotFound) + raise InvalidRequest end end diff --git a/spec/controllers/well_known/webfinger_controller_spec.rb b/spec/controllers/well_known/webfinger_controller_spec.rb index 20275aa63bc..46f63185b9e 100644 --- a/spec/controllers/well_known/webfinger_controller_spec.rb +++ b/spec/controllers/well_known/webfinger_controller_spec.rb @@ -84,5 +84,15 @@ PEM expect(response).to have_http_status(:not_found) end + + it 'returns http bad request when not given a resource parameter' do + get :show, params: { }, format: :json + expect(response).to have_http_status(:bad_request) + end + + it 'returns http bad request when given a nonsense parameter' do + get :show, params: { resource: 'df/:dfkj' } + expect(response).to have_http_status(:bad_request) + end end end diff --git a/spec/lib/webfinger_resource_spec.rb b/spec/lib/webfinger_resource_spec.rb index 287537a2610..236e9f3e2d7 100644 --- a/spec/lib/webfinger_resource_spec.rb +++ b/spec/lib/webfinger_resource_spec.rb @@ -39,7 +39,7 @@ describe WebfingerResource do expect { WebfingerResource.new(resource).username - }.to raise_error(ActiveRecord::RecordNotFound) + }.to raise_error(WebfingerResource::InvalidRequest) end it 'finds the username in a valid https route' do @@ -123,5 +123,15 @@ describe WebfingerResource do expect(result).to eq 'alice' end end + + describe 'with a nonsense resource' do + it 'raises InvalidRequest' do + resource = 'df/:dfkj' + + expect { + WebfingerResource.new(resource).username + }.to raise_error(WebfingerResource::InvalidRequest) + end + end end end From 27ea7c13a554d41c4bd83a2712b711d2ef55629c Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 14 May 2020 23:37:37 +0200 Subject: [PATCH 2/9] Fix hashtag search performing account search as well (#13758) --- app/services/search_service.rb | 2 +- spec/services/search_service_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 830de4de326..19500a8d466 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -94,7 +94,7 @@ class SearchService < BaseService end def account_searchable? - account_search? && !(@query.include?('@') && @query.include?(' ')) + account_search? && !(@query.start_with?('#') || (@query.include?('@') && @query.include?(' '))) end def hashtag_searchable? diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb index 739bb9cf5d8..5b52662ba02 100644 --- a/spec/services/search_service_spec.rb +++ b/spec/services/search_service_spec.rb @@ -91,6 +91,14 @@ describe SearchService, type: :service do expect(Tag).not_to have_received(:search_for) expect(results).to eq empty_results end + it 'does not include account when starts with # character' do + query = '#tag' + allow(AccountSearchService).to receive(:new) + + results = subject.call(query, nil, 10) + expect(AccountSearchService).to_not have_received(:new) + expect(results).to eq empty_results + end end end end From 927f9ea4997676bcceea0d625703ac6ec66a68b2 Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 15 May 2020 11:38:12 +0200 Subject: [PATCH 3/9] Fix GifReader exceptions (#13760) --- lib/paperclip/gif_transcoder.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb index 64f12f9637e..9f3c8e8be33 100644 --- a/lib/paperclip/gif_transcoder.rb +++ b/lib/paperclip/gif_transcoder.rb @@ -6,7 +6,7 @@ class GifReader EXTENSION_LABELS = [0xf9, 0x01, 0xff].freeze GIF_HEADERS = %w(GIF87a GIF89a).freeze - class GifReaderException; end + class GifReaderException < StandardError; end class UnknownImageType < GifReaderException; end From 328c5a21d7c4a1641d25f1d01b043f39e9809926 Mon Sep 17 00:00:00 2001 From: Shlee Date: Fri, 15 May 2020 17:38:30 +0800 Subject: [PATCH 4/9] Update docker-compose.yml (#13756) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b415782741d..e190df6d611 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: redis: restart: always - image: redis:5.0-alpine + image: redis:6.0-alpine networks: - internal_network healthcheck: From a319c1e60f5ef125474122da6deb3b3251f7f0ef Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 15 May 2020 17:08:59 +0200 Subject: [PATCH 5/9] Add support for `summary` field for media description (#13763) --- app/lib/activitypub/activity/create.rb | 2 +- spec/lib/activitypub/activity/create_spec.rb | 25 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index c55cfe08eae..572b8087e6e 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -201,7 +201,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity begin href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) media_attachments << media_attachment next if unsupported_media_type?(attachment['mediaType']) || skip_download? diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index c4efb5cc932..5220deabb28 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -287,6 +287,31 @@ RSpec.describe ActivityPub::Activity::Create do end end + context 'with media attachments with long description as summary' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + url: 'http://example.com/attachment.png', + summary: '*' * 1500, + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.media_attachments.map(&:description)).to include('*' * 1500) + end + end + context 'with media attachments with focal points' do let(:object_json) do { From 2b91a3dac0a71108042481d27f549c2261e5de41 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 May 2020 17:15:24 +0200 Subject: [PATCH 6/9] Fix `tootctl upgrade storage-schema` misbehaving (#13761) - Fix not moving original files of custom emojis - Fix command failing to move any files with S3 storage - Fix command marking records as upgraded when move failed Fix #13594 --- lib/mastodon/upgrade_cli.rb | 41 +++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/lib/mastodon/upgrade_cli.rb b/lib/mastodon/upgrade_cli.rb index 779462a4f97..629834b19fc 100644 --- a/lib/mastodon/upgrade_cli.rb +++ b/lib/mastodon/upgrade_cli.rb @@ -41,23 +41,32 @@ module Mastodon klass.find_each do |record| attachment_names.each do |attachment_name| attachment = record.public_send(attachment_name) + upgraded = false next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION - attachment.styles.each_key do |style| - case Paperclip::Attachment.default_options[:storage] - when :s3 - upgrade_storage_s3(progress, attachment, style) - when :fog - upgrade_storage_fog(progress, attachment, style) - when :filesystem - upgrade_storage_filesystem(progress, attachment, style) + styles = attachment.styles.keys + + styles << :original unless styles.include?(:original) + + styles.each do |style| + success = begin + case Paperclip::Attachment.default_options[:storage] + when :s3 + upgrade_storage_s3(progress, attachment, style) + when :fog + upgrade_storage_fog(progress, attachment, style) + when :filesystem + upgrade_storage_filesystem(progress, attachment, style) + end end + upgraded = true if style == :original && success + progress.increment end - attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) + attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) if upgraded end if record.changed? @@ -78,18 +87,20 @@ module Mastodon def upgrade_storage_s3(progress, attachment, style) previous_storage_schema_version = attachment.storage_schema_version object = attachment.s3_object(style) + success = true attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) - upgraded_path = attachment.path(style) + new_object = attachment.s3_object(style) - if upgraded_path != object.key && object.exists? - progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose] + if new_object.key != object.key && object.exists? + progress.log("Moving #{object.key} to #{new_object.key}") if options[:verbose] begin - object.move_to(upgraded_path) unless dry_run? + object.move_to(new_object) unless dry_run? rescue => e progress.log(pastel.red("Error processing #{object.key}: #{e}")) + success = false end end @@ -97,6 +108,7 @@ module Mastodon # previous version at the end. The upgrade will be recorded after # all styles are updated attachment.instance_write(:storage_schema_version, previous_storage_schema_version) + success end def upgrade_storage_fog(_progress, _attachment, _style) @@ -107,6 +119,7 @@ module Mastodon def upgrade_storage_filesystem(progress, attachment, style) previous_storage_schema_version = attachment.storage_schema_version previous_path = attachment.path(style) + success = true attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) @@ -128,6 +141,7 @@ module Mastodon end rescue => e progress.log(pastel.red("Error processing #{previous_path}: #{e}")) + success = false unless dry_run? begin @@ -143,6 +157,7 @@ module Mastodon # previous version at the end. The upgrade will be recorded after # all styles are updated attachment.instance_write(:storage_schema_version, previous_storage_schema_version) + success end end end From 199bbbcb9fe0620000538ae0c7766ff4a8f4cf0c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 May 2020 18:41:27 +0200 Subject: [PATCH 7/9] Fix `tootctl media remove-orphans` choking on unknown files in storage (#13765) Fix #13762 Catch tootctl interrupt to prevent confusing stacktrace --- bin/tootctl | 8 +++++++- lib/mastodon/media_cli.rb | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/bin/tootctl b/bin/tootctl index f26e1c7edcc..a9ebb22c6dc 100755 --- a/bin/tootctl +++ b/bin/tootctl @@ -1,5 +1,11 @@ #!/usr/bin/env ruby APP_PATH = File.expand_path('../config/application', __dir__) + require_relative '../config/boot' require_relative '../lib/cli' -Mastodon::CLI.start(ARGV) + +begin + Mastodon::CLI.start(ARGV) +rescue Interrupt + exit(130) +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 2b165333598..c95f3410aea 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -88,6 +88,11 @@ module Mastodon path_segments = object.key.split('/') path_segments.delete('cache') + if path_segments.size != 7 + progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) + next + end + model_name = path_segments.first.classify attachment_name = path_segments[1].singularize record_id = path_segments[2..-2].join.to_i @@ -127,6 +132,11 @@ module Mastodon path_segments = key.split(File::SEPARATOR) path_segments.delete('cache') + if path_segments.size != 7 + progress.log(pastel.yellow("Unrecognized file found: #{key}")) + next + end + model_name = path_segments.first.classify record_id = path_segments[2..-2].join.to_i attachment_name = path_segments[1].singularize @@ -246,6 +256,11 @@ module Mastodon path_segments = path.split('/')[2..-1] path_segments.delete('cache') + if path_segments.size != 7 + say('Not a media URL', :red) + exit(1) + end + model_name = path_segments.first.classify record_id = path_segments[2..-2].join.to_i @@ -294,6 +309,8 @@ module Mastodon segments = object.key.split('/') segments.delete('cache') + next if segments.size != 7 + model_name = segments.first.classify record_id = segments[2..-2].join.to_i From ce87469d0a085b238646387fb01d78c206b83124 Mon Sep 17 00:00:00 2001 From: Stanislas Date: Sun, 17 May 2020 17:27:36 +0200 Subject: [PATCH 8/9] Fix `tootctl upgrade storage-schema` S3 ACL (#13768) --- lib/mastodon/upgrade_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/upgrade_cli.rb b/lib/mastodon/upgrade_cli.rb index 629834b19fc..570b7e6fa8a 100644 --- a/lib/mastodon/upgrade_cli.rb +++ b/lib/mastodon/upgrade_cli.rb @@ -97,7 +97,7 @@ module Mastodon progress.log("Moving #{object.key} to #{new_object.key}") if options[:verbose] begin - object.move_to(new_object) unless dry_run? + object.move_to(new_object, acl: attachment.s3_permissions(style)) unless dry_run? rescue => e progress.log(pastel.red("Error processing #{object.key}: #{e}")) success = false From 50524d13278806573a4a685e428a6c9f3d7a74e5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 20 May 2020 14:04:37 +0200 Subject: [PATCH 9/9] Bump elasticsearch from 7.6.0 to 7.7.0 (#13784) Bumps [elasticsearch](https://github.com/elastic/elasticsearch-ruby) from 7.6.0 to 7.7.0. - [Release notes](https://github.com/elastic/elasticsearch-ruby/releases) - [Changelog](https://github.com/elastic/elasticsearch-ruby/blob/master/CHANGELOG.md) - [Commits](https://github.com/elastic/elasticsearch-ruby/compare/v7.6.0...7.7.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index accac821b5b..ddedbd5f32e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -201,13 +201,13 @@ GEM dotenv (= 2.7.5) railties (>= 3.2, < 6.1) e2mmap (0.1.0) - elasticsearch (7.6.0) - elasticsearch-api (= 7.6.0) - elasticsearch-transport (= 7.6.0) - elasticsearch-api (7.6.0) + elasticsearch (7.7.0) + elasticsearch-api (= 7.7.0) + elasticsearch-transport (= 7.7.0) + elasticsearch-api (7.7.0) multi_json elasticsearch-dsl (0.1.9) - elasticsearch-transport (7.6.0) + elasticsearch-transport (7.7.0) faraday (~> 1) multi_json encryptor (3.0.0)