From d83faa1a8902c91a5dbd0bf3d9740e3e19c1d623 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 24 Aug 2022 19:00:37 +0200 Subject: [PATCH 01/22] Add ability to block sign-ups from IP (#19037) --- .../auth/registrations_controller.rb | 6 +- app/models/ip_block.rb | 1 + app/services/app_sign_up_service.rb | 66 +++++++++++++++---- config/locales/simple_form.en.yml | 2 + spec/services/app_sign_up_service_spec.rb | 2 +- 5 files changed, 64 insertions(+), 13 deletions(-) diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index 1c3adbd786b..7e86e01baa9 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -82,7 +82,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController end def check_enabled_registrations - redirect_to root_path if single_user_mode? || omniauth_only? || !allowed_registrations? + redirect_to root_path if single_user_mode? || omniauth_only? || !allowed_registrations? || ip_blocked? end def allowed_registrations? @@ -93,6 +93,10 @@ class Auth::RegistrationsController < Devise::RegistrationsController ENV['OMNIAUTH_ONLY'] == 'true' end + def ip_blocked? + IpBlock.where(severity: :sign_up_block).where('ip >>= ?', request.remote_ip.to_s).exists? + end + def invite_code if params[:user] params[:user][:invite_code] diff --git a/app/models/ip_block.rb b/app/models/ip_block.rb index aedd3ca0d4d..e1ab59806e8 100644 --- a/app/models/ip_block.rb +++ b/app/models/ip_block.rb @@ -19,6 +19,7 @@ class IpBlock < ApplicationRecord enum severity: { sign_up_requires_approval: 5000, + sign_up_block: 5500, no_access: 9999, } diff --git a/app/services/app_sign_up_service.rb b/app/services/app_sign_up_service.rb index e0069415772..3833327bbce 100644 --- a/app/services/app_sign_up_service.rb +++ b/app/services/app_sign_up_service.rb @@ -2,23 +2,67 @@ class AppSignUpService < BaseService def call(app, remote_ip, params) - return unless allowed_registrations? + @app = app + @remote_ip = remote_ip + @params = params - user_params = params.slice(:email, :password, :agreement, :locale) - account_params = params.slice(:username) - invite_request_params = { text: params[:reason] } - user = User.create!(user_params.merge(created_by_application: app, sign_up_ip: remote_ip, password_confirmation: user_params[:password], account_attributes: account_params, invite_request_attributes: invite_request_params)) + raise Mastodon::NotPermittedError unless allowed_registrations? - Doorkeeper::AccessToken.create!(application: app, - resource_owner_id: user.id, - scopes: app.scopes, - expires_in: Doorkeeper.configuration.access_token_expires_in, - use_refresh_token: Doorkeeper.configuration.refresh_token_enabled?) + ApplicationRecord.transaction do + create_user! + create_access_token! + end + + @access_token end private + def create_user! + @user = User.create!( + user_params.merge(created_by_application: @app, sign_up_ip: @remote_ip, password_confirmation: user_params[:password], account_attributes: account_params, invite_request_attributes: invite_request_params) + ) + end + + def create_access_token! + @access_token = Doorkeeper::AccessToken.create!( + application: @app, + resource_owner_id: @user.id, + scopes: @app.scopes, + expires_in: Doorkeeper.configuration.access_token_expires_in, + use_refresh_token: Doorkeeper.configuration.refresh_token_enabled? + ) + end + + def user_params + @params.slice(:email, :password, :agreement, :locale) + end + + def account_params + @params.slice(:username) + end + + def invite_request_params + { text: @params[:reason] } + end + def allowed_registrations? - Setting.registrations_mode != 'none' && !Rails.configuration.x.single_user_mode + registrations_open? && !single_user_mode? && !omniauth_only? && !ip_blocked? + end + + def registrations_open? + Setting.registrations_mode != 'none' + end + + def single_user_mode? + Rails.configuration.x.single_user_mode + end + + def omniauth_only? + ENV['OMNIAUTH_ONLY'] == 'true' + end + + def ip_blocked? + IpBlock.where(severity: :sign_up_block).where('ip >>= ?', @remote_ip.to_s).exists? end end diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index c17a62cbea3..28f78d50003 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -85,6 +85,7 @@ en: ip: Enter an IPv4 or IPv6 address. You can block entire ranges using the CIDR syntax. Be careful not to lock yourself out! severities: no_access: Block access to all resources + sign_up_block: New sign-ups will not be possible sign_up_requires_approval: New sign-ups will require your approval severity: Choose what will happen with requests from this IP rule: @@ -219,6 +220,7 @@ en: ip: IP severities: no_access: Block access + sign_up_block: Block sign-ups sign_up_requires_approval: Limit sign-ups severity: Rule notification_emails: diff --git a/spec/services/app_sign_up_service_spec.rb b/spec/services/app_sign_up_service_spec.rb index e0c83b7041d..8ec4d4a7a63 100644 --- a/spec/services/app_sign_up_service_spec.rb +++ b/spec/services/app_sign_up_service_spec.rb @@ -11,7 +11,7 @@ RSpec.describe AppSignUpService, type: :service do it 'returns nil when registrations are closed' do tmp = Setting.registrations_mode Setting.registrations_mode = 'none' - expect(subject.call(app, remote_ip, good_params)).to be_nil + expect { subject.call(app, remote_ip, good_params) }.to raise_error Mastodon::NotPermittedError Setting.registrations_mode = tmp end From 0412a4d03e3e075b8b4090774ebe5db4f95412de Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 24 Aug 2022 19:00:55 +0200 Subject: [PATCH 02/22] Change e-mail domain blocks to match subdomains of blocked domains (#18979) --- app/models/email_domain_block.rb | 66 ++++++++++++++++++-------- spec/models/email_domain_block_spec.rb | 27 ++++++++--- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb index 0e1e663c109..f9d74332b88 100644 --- a/app/models/email_domain_block.rb +++ b/app/models/email_domain_block.rb @@ -30,32 +30,56 @@ class EmailDomainBlock < ApplicationRecord @history ||= Trends::History.new('email_domain_blocks', id) end - def self.block?(domain_or_domains, attempt_ip: nil) - domains = Array(domain_or_domains).map do |str| - domain = begin - if str.include?('@') - str.split('@', 2).last - else - str - end + class Matcher + def initialize(domain_or_domains, attempt_ip: nil) + @uris = extract_uris(domain_or_domains) + @attempt_ip = attempt_ip + end + + def match? + blocking? || invalid_uri? + end + + private + + def invalid_uri? + @uris.any?(&:nil?) + end + + def blocking? + blocks = EmailDomainBlock.where(domain: domains_with_variants).order(Arel.sql('char_length(domain) desc')) + blocks.each { |block| block.history.add(@attempt_ip) } if @attempt_ip.present? + blocks.any? + end + + def domains_with_variants + @uris.flat_map do |uri| + next if uri.nil? + + segments = uri.normalized_host.split('.') + + segments.map.with_index { |_, i| segments[i..-1].join('.') } end - - TagManager.instance.normalize_domain(domain) if domain.present? - rescue Addressable::URI::InvalidURIError - nil end - # If some of the inputs passed in are invalid, we definitely want to - # block the attempt, but we also want to register hits against any - # other valid matches + def extract_uris(domain_or_domains) + Array(domain_or_domains).map do |str| + domain = begin + if str.include?('@') + str.split('@', 2).last + else + str + end + end - blocked = domains.any?(&:nil?) - - where(domain: domains).find_each do |block| - blocked = true - block.history.add(attempt_ip) if attempt_ip.present? + Addressable::URI.new.tap { |u| u.host = domain.strip } if domain.present? + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError + nil + end end + end - blocked + def self.block?(domain_or_domains, attempt_ip: nil) + Matcher.new(domain_or_domains, attempt_ip: attempt_ip).match? end end diff --git a/spec/models/email_domain_block_spec.rb b/spec/models/email_domain_block_spec.rb index 567a32c3280..e23116888c7 100644 --- a/spec/models/email_domain_block_spec.rb +++ b/spec/models/email_domain_block_spec.rb @@ -12,16 +12,29 @@ RSpec.describe EmailDomainBlock, type: :model do let(:input) { nil } context 'given an e-mail address' do - let(:input) { 'nyarn@example.com' } + let(:input) { "foo@#{domain}" } - it 'returns true if the domain is blocked' do - Fabricate(:email_domain_block, domain: 'example.com') - expect(EmailDomainBlock.block?(input)).to be true + context do + let(:domain) { 'example.com' } + + it 'returns true if the domain is blocked' do + Fabricate(:email_domain_block, domain: 'example.com') + expect(EmailDomainBlock.block?(input)).to be true + end + + it 'returns false if the domain is not blocked' do + Fabricate(:email_domain_block, domain: 'other-example.com') + expect(EmailDomainBlock.block?(input)).to be false + end end - it 'returns false if the domain is not blocked' do - Fabricate(:email_domain_block, domain: 'other-example.com') - expect(EmailDomainBlock.block?(input)).to be false + context do + let(:domain) { 'mail.example.com' } + + it 'returns true if it is a subdomain of a blocked domain' do + Fabricate(:email_domain_block, domain: 'example.com') + expect(described_class.block?(input)).to be true + end end end From b21aa828039610ef1750eff2291821b95da76896 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:54:36 +0900 Subject: [PATCH 03/22] Bump stackprof from 0.2.20 to 0.2.21 (#19027) Bumps [stackprof](https://github.com/tmm1/stackprof) from 0.2.20 to 0.2.21. - [Release notes](https://github.com/tmm1/stackprof/releases) - [Changelog](https://github.com/tmm1/stackprof/blob/master/CHANGELOG.md) - [Commits](https://github.com/tmm1/stackprof/compare/v0.2.20...v0.2.21) --- updated-dependencies: - dependency-name: stackprof dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 840fe0a78ce..266ddda1ad1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -648,7 +648,7 @@ GEM sshkit (1.21.2) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) - stackprof (0.2.20) + stackprof (0.2.21) statsd-ruby (1.5.0) stoplight (3.0.0) strong_migrations (0.7.9) From ca5536167105e2bb690233d53cdfb05f4b0c33db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:55:01 +0900 Subject: [PATCH 04/22] Bump @babel/core from 7.18.10 to 7.18.13 (#19032) Bumps [@babel/core](https://github.com/babel/babel/tree/HEAD/packages/babel-core) from 7.18.10 to 7.18.13. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.18.13/packages/babel-core) --- updated-dependencies: - dependency-name: "@babel/core" dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 56 ++++++++++++++++++++++++++-------------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 6c7f9f53afc..f116add2ebb 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ }, "private": true, "dependencies": { - "@babel/core": "^7.18.10", + "@babel/core": "^7.18.13", "@babel/plugin-proposal-decorators": "^7.18.10", "@babel/plugin-transform-react-inline-elements": "^7.18.6", "@babel/plugin-transform-runtime": "^7.18.10", diff --git a/yarn.lock b/yarn.lock index d98905b0748..a9b1cd757fe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -33,21 +33,21 @@ resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.18.8.tgz#2483f565faca607b8535590e84e7de323f27764d" integrity sha512-HSmX4WZPPK3FUxYp7g2T6EyO8j96HlZJlxmKPSh6KAcqwyDrfx7hKjXpAW/0FhFfTJsR0Yt4lAjLI2coMptIHQ== -"@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.18.10", "@babel/core@^7.7.2": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.18.10.tgz#39ad504991d77f1f3da91be0b8b949a5bc466fb8" - integrity sha512-JQM6k6ENcBFKVtWvLavlvi/mPcpYZ3+R+2EySDEMSMbp7Mn4FexlbbJVrx2R7Ijhr01T8gyqrOaABWIOgxeUyw== +"@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.18.13", "@babel/core@^7.7.2": + version "7.18.13" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.18.13.tgz#9be8c44512751b05094a4d3ab05fc53a47ce00ac" + integrity sha512-ZisbOvRRusFktksHSG6pjj1CSvkPkcZq/KHD45LAkVP/oiHJkNBZWfpvlLmX8OtHDG8IuzsFlVRWo08w7Qxn0A== dependencies: "@ampproject/remapping" "^2.1.0" "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.18.10" + "@babel/generator" "^7.18.13" "@babel/helper-compilation-targets" "^7.18.9" "@babel/helper-module-transforms" "^7.18.9" "@babel/helpers" "^7.18.9" - "@babel/parser" "^7.18.10" + "@babel/parser" "^7.18.13" "@babel/template" "^7.18.10" - "@babel/traverse" "^7.18.10" - "@babel/types" "^7.18.10" + "@babel/traverse" "^7.18.13" + "@babel/types" "^7.18.13" convert-source-map "^1.7.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -63,12 +63,12 @@ eslint-visitor-keys "^2.1.0" semver "^6.3.0" -"@babel/generator@^7.18.10", "@babel/generator@^7.7.2": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.18.10.tgz#794f328bfabdcbaf0ebf9bf91b5b57b61fa77a2a" - integrity sha512-0+sW7e3HjQbiHbj1NeU/vN8ornohYlacAfZIaXhdoGweQqgcNy69COVciYYqEXJ/v+9OBA7Frxm4CVAuNqKeNA== +"@babel/generator@^7.18.13", "@babel/generator@^7.7.2": + version "7.18.13" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.18.13.tgz#59550cbb9ae79b8def15587bdfbaa388c4abf212" + integrity sha512-CkPg8ySSPuHTYPJYo7IRALdqyjM9HCbt/3uOBEFbzyGVP6Mn8bwFPB0jX6982JVNBlYzM1nnPkfjuXSOPtQeEQ== dependencies: - "@babel/types" "^7.18.10" + "@babel/types" "^7.18.13" "@jridgewell/gen-mapping" "^0.3.2" jsesc "^2.5.1" @@ -337,10 +337,10 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.10.tgz#94b5f8522356e69e8277276adf67ed280c90ecc1" - integrity sha512-TYk3OA0HKL6qNryUayb5UUEhM/rkOQozIBEA5ITXh5DWrSp0TlUQXMyZmnWxG/DizSWBeeQ0Zbc5z8UGaaqoeg== +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.18.10", "@babel/parser@^7.18.13": + version "7.18.13" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.13.tgz#5b2dd21cae4a2c5145f1fbd8ca103f9313d3b7e4" + integrity sha512-dgXcIfMuQ0kgzLB2b9tRZs7TTFFaGM2AbtA4fJgUUYukzGH4jwsS7hzQHEGs67jdehpm22vkgKwvbU+aEflgwg== "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" @@ -1085,26 +1085,26 @@ "@babel/parser" "^7.18.10" "@babel/types" "^7.18.10" -"@babel/traverse@^7.18.10", "@babel/traverse@^7.18.6", "@babel/traverse@^7.18.9", "@babel/traverse@^7.7.2": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.18.10.tgz#37ad97d1cb00efa869b91dd5d1950f8a6cf0cb08" - integrity sha512-J7ycxg0/K9XCtLyHf0cz2DqDihonJeIo+z+HEdRe9YuT8TY4A66i+Ab2/xZCEW7Ro60bPCBBfqqboHSamoV3+g== +"@babel/traverse@^7.18.10", "@babel/traverse@^7.18.13", "@babel/traverse@^7.18.6", "@babel/traverse@^7.18.9", "@babel/traverse@^7.7.2": + version "7.18.13" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.18.13.tgz#5ab59ef51a997b3f10c4587d648b9696b6cb1a68" + integrity sha512-N6kt9X1jRMLPxxxPYWi7tgvJRH/rtoU+dbKAPDM44RFHiMH8igdsaSBgFeskhSl/kLWLDUvIh1RXCrTmg0/zvA== dependencies: "@babel/code-frame" "^7.18.6" - "@babel/generator" "^7.18.10" + "@babel/generator" "^7.18.13" "@babel/helper-environment-visitor" "^7.18.9" "@babel/helper-function-name" "^7.18.9" "@babel/helper-hoist-variables" "^7.18.6" "@babel/helper-split-export-declaration" "^7.18.6" - "@babel/parser" "^7.18.10" - "@babel/types" "^7.18.10" + "@babel/parser" "^7.18.13" + "@babel/types" "^7.18.13" debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.0.0-beta.49", "@babel/types@^7.18.10", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": - version "7.18.10" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.18.10.tgz#4908e81b6b339ca7c6b7a555a5fc29446f26dde6" - integrity sha512-MJvnbEiiNkpjo+LknnmRrqbY1GPUUggjv+wQVjetM/AONoupqRALB7I6jGqNUAZsKcRIEu2J6FRFvsczljjsaQ== +"@babel/types@^7.0.0", "@babel/types@^7.0.0-beta.49", "@babel/types@^7.18.10", "@babel/types@^7.18.13", "@babel/types@^7.18.6", "@babel/types@^7.18.9", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4": + version "7.18.13" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.18.13.tgz#30aeb9e514f4100f7c1cb6e5ba472b30e48f519a" + integrity sha512-ePqfTihzW0W6XAU+aMw2ykilisStJfDnsejDCXRchCcMJ4O0+8DhPXf2YUbZ6wjBlsEmZwLK/sPweWtu8hcJYQ== dependencies: "@babel/helper-string-parser" "^7.18.10" "@babel/helper-validator-identifier" "^7.18.6" From 37de05fdde75e32d6801e237545d4b67fd3d600e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:55:34 +0900 Subject: [PATCH 05/22] Bump sass from 1.54.4 to 1.54.5 (#19028) Bumps [sass](https://github.com/sass/dart-sass) from 1.54.4 to 1.54.5. - [Release notes](https://github.com/sass/dart-sass/releases) - [Changelog](https://github.com/sass/dart-sass/blob/main/CHANGELOG.md) - [Commits](https://github.com/sass/dart-sass/compare/1.54.4...1.54.5) --- updated-dependencies: - dependency-name: sass dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index f116add2ebb..6f6aa7b855e 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,7 @@ "requestidlecallback": "^0.3.0", "reselect": "^4.1.6", "rimraf": "^3.0.2", - "sass": "^1.54.4", + "sass": "^1.54.5", "sass-loader": "^10.2.0", "stacktrace-js": "^2.0.2", "stringz": "^2.1.0", diff --git a/yarn.lock b/yarn.lock index a9b1cd757fe..e78c4741bda 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9745,10 +9745,10 @@ sass-loader@^10.2.0: schema-utils "^3.0.0" semver "^7.3.2" -sass@^1.54.4: - version "1.54.4" - resolved "https://registry.yarnpkg.com/sass/-/sass-1.54.4.tgz#803ff2fef5525f1dd01670c3915b4b68b6cba72d" - integrity sha512-3tmF16yvnBwtlPrNBHw/H907j8MlOX8aTBnlNX1yrKx24RKcJGPyLhFUwkoKBKesR3unP93/2z14Ll8NicwQUA== +sass@^1.54.5: + version "1.54.5" + resolved "https://registry.yarnpkg.com/sass/-/sass-1.54.5.tgz#93708f5560784f6ff2eab8542ade021a4a947b3a" + integrity sha512-p7DTOzxkUPa/63FU0R3KApkRHwcVZYC0PLnLm5iyZACyp15qSi32x7zVUhRdABAATmkALqgGrjCJAcWvobmhHw== dependencies: chokidar ">=3.0.0 <4.0.0" immutable "^4.0.0" From 26a1d1ac5f4e582296f813530be68c07cc85033a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:56:16 +0900 Subject: [PATCH 06/22] Bump webpack-bundle-analyzer from 4.5.0 to 4.6.1 (#19029) Bumps [webpack-bundle-analyzer](https://github.com/webpack-contrib/webpack-bundle-analyzer) from 4.5.0 to 4.6.1. - [Release notes](https://github.com/webpack-contrib/webpack-bundle-analyzer/releases) - [Changelog](https://github.com/webpack-contrib/webpack-bundle-analyzer/blob/master/CHANGELOG.md) - [Commits](https://github.com/webpack-contrib/webpack-bundle-analyzer/compare/v4.5.0...v4.6.1) --- updated-dependencies: - dependency-name: webpack-bundle-analyzer dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 6f6aa7b855e..9ae7573d9c5 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,7 @@ "uuid": "^8.3.1", "webpack": "^4.46.0", "webpack-assets-manifest": "^4.0.6", - "webpack-bundle-analyzer": "^4.5.0", + "webpack-bundle-analyzer": "^4.6.1", "webpack-cli": "^3.3.12", "webpack-merge": "^5.8.0", "wicg-inert": "^3.1.2", diff --git a/yarn.lock b/yarn.lock index e78c4741bda..0703b0a7ef9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2139,12 +2139,7 @@ acorn@^7.1.1, acorn@^7.4.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.0.4: - version "8.3.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.3.0.tgz#1193f9b96c4e8232f00b11a9edff81b2c8b98b88" - integrity sha512-tqPKHZ5CaBJw0Xmy0ZZvLs1qTV+BNFSyvn77ASXkpBNfIRk8ev26fKrD9iLGwGA9zedPao52GSHzq8lyZG0NUw== - -acorn@^8.5.0, acorn@^8.7.1: +acorn@^8.0.4, acorn@^8.5.0, acorn@^8.7.1: version "8.7.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== @@ -11343,10 +11338,10 @@ webpack-assets-manifest@^4.0.6: tapable "^1.0" webpack-sources "^1.0" -webpack-bundle-analyzer@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.5.0.tgz#1b0eea2947e73528754a6f9af3e91b2b6e0f79d5" - integrity sha512-GUMZlM3SKwS8Z+CKeIFx7CVoHn3dXFcUAjT/dcZQQmfSZGvitPfMob2ipjai7ovFFqPvTqkEZ/leL4O0YOdAYQ== +webpack-bundle-analyzer@^4.6.1: + version "4.6.1" + resolved "https://registry.yarnpkg.com/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.6.1.tgz#bee2ee05f4ba4ed430e4831a319126bb4ed9f5a6" + integrity sha512-oKz9Oz9j3rUciLNfpGFjOb49/jEpXNmWdVH8Ls//zNcnLlQdTGXQQMsBbb/gR7Zl8WNLxVCq+0Hqbx3zv6twBw== dependencies: acorn "^8.0.4" acorn-walk "^8.0.0" From 115b8311c1f3e38fba5eb6bdbd272e8a925aa06c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:57:16 +0900 Subject: [PATCH 07/22] Bump sidekiq from 6.5.4 to 6.5.5 (#19030) Bumps [sidekiq](https://github.com/mperham/sidekiq) from 6.5.4 to 6.5.5. - [Release notes](https://github.com/mperham/sidekiq/releases) - [Changelog](https://github.com/mperham/sidekiq/blob/main/Changes.md) - [Commits](https://github.com/mperham/sidekiq/compare/v6.5.4...v6.5.5) --- updated-dependencies: - dependency-name: sidekiq dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 266ddda1ad1..8306efec2cd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -610,7 +610,7 @@ GEM activerecord (>= 4.0.0) railties (>= 4.0.0) semantic_range (3.0.0) - sidekiq (6.5.4) + sidekiq (6.5.5) connection_pool (>= 2.2.2) rack (~> 2.0) redis (>= 4.5.0) From d156e9b823e5be9e33de4a5d667b2cd32a94cbe4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:58:04 +0900 Subject: [PATCH 08/22] Bump stylelint from 14.10.0 to 14.11.0 (#19034) Bumps [stylelint](https://github.com/stylelint/stylelint) from 14.10.0 to 14.11.0. - [Release notes](https://github.com/stylelint/stylelint/releases) - [Changelog](https://github.com/stylelint/stylelint/blob/main/CHANGELOG.md) - [Commits](https://github.com/stylelint/stylelint/compare/14.10.0...14.11.0) --- updated-dependencies: - dependency-name: stylelint dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 9ae7573d9c5..b605fe114f0 100644 --- a/package.json +++ b/package.json @@ -155,7 +155,7 @@ "raf": "^3.4.1", "react-intl-translations-manager": "^5.0.3", "react-test-renderer": "^16.14.0", - "stylelint": "^14.10.0", + "stylelint": "^14.11.0", "stylelint-config-standard-scss": "^4.0.0", "webpack-dev-server": "^3.11.3", "yargs": "^17.5.1" diff --git a/yarn.lock b/yarn.lock index 0703b0a7ef9..ef957156df6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3361,10 +3361,10 @@ color@^3.0.0: color-convert "^1.9.1" color-string "^1.5.2" -colord@^2.9.2: - version "2.9.2" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.2.tgz#25e2bacbbaa65991422c07ea209e2089428effb1" - integrity sha512-Uqbg+J445nc1TKn4FoDPS6ZZqAvEDnwrH42yo8B40JSOgSLxMZ/gt3h4nmCtPLQeXhjJJkqBx7SCY35WnIixaQ== +colord@^2.9.3: + version "2.9.3" + resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.3.tgz#4f8ce919de456f1d5c1c368c307fe20f3e59fb43" + integrity sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw== colorette@^1.2.2: version "1.2.2" @@ -10509,14 +10509,14 @@ stylelint-scss@^4.0.0: postcss-selector-parser "^6.0.6" postcss-value-parser "^4.1.0" -stylelint@^14.10.0: - version "14.10.0" - resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-14.10.0.tgz#c588f5cd47cd214cf1acee5bc165961b6a3ad836" - integrity sha512-VAmyKrEK+wNFh9R8mNqoxEFzaa4gsHGhcT4xgkQDuOA5cjF6CaNS8loYV7gpi4tIZBPUyXesotPXzJAMN8VLOQ== +stylelint@^14.11.0: + version "14.11.0" + resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-14.11.0.tgz#e2ecb28bbacab05e1fbeb84cbba23883b27499cc" + integrity sha512-OTLjLPxpvGtojEfpESWM8Ir64Z01E89xsisaBMUP/ngOx1+4VG2DPRcUyCCiin9Rd3kPXPsh/uwHd9eqnvhsYA== dependencies: "@csstools/selector-specificity" "^2.0.2" balanced-match "^2.0.0" - colord "^2.9.2" + colord "^2.9.3" cosmiconfig "^7.0.1" css-functions-list "^3.1.0" debug "^4.3.4" @@ -10551,7 +10551,7 @@ stylelint@^14.10.0: svg-tags "^1.0.0" table "^6.8.0" v8-compile-cache "^2.3.0" - write-file-atomic "^4.0.1" + write-file-atomic "^4.0.2" stylis@4.0.13: version "4.0.13" @@ -11614,10 +11614,10 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= -write-file-atomic@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-4.0.1.tgz#9faa33a964c1c85ff6f849b80b42a88c2c537c8f" - integrity sha512-nSKUxgAbyioruk6hU87QzVbY279oYT6uiwgDoujth2ju4mJ+TZau7SQBhtbTmUyuNYTuXnSyRn66FV0+eCgcrQ== +write-file-atomic@^4.0.1, write-file-atomic@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-4.0.2.tgz#a9df01ae5b77858a027fd2e80768ee433555fcfd" + integrity sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg== dependencies: imurmurhash "^0.1.4" signal-exit "^3.0.7" From 50487db1224851a49ee523bbc013d5f8686a7a55 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Aug 2022 04:27:47 +0200 Subject: [PATCH 09/22] Add ability to filter individual posts (#18945) * Add database table for status-specific filters * Add REST endpoints, entities and attributes * Show status filters in /filters interface * Perform server-side filtering for individual posts filters * Fix filtering on context mismatch * Refactor `toServerSideType` by moving it to its own module * Move loupe and delete icons to their own module * Add ability to filter individual posts from WebUI * Replace keyword list by warnings (expired, context mismatch) * Refactor server-side filtering code * Add tests --- .../api/v1/filters/statuses_controller.rb | 44 ++++ .../filters/statuses_controller.rb | 49 +++++ app/controllers/filters_controller.rb | 2 +- app/javascript/mastodon/actions/filters.js | 93 +++++++++ app/javascript/mastodon/actions/statuses.js | 4 +- app/javascript/mastodon/components/status.js | 1 + .../mastodon/components/status_action_bar.js | 16 +- .../mastodon/containers/status_container.js | 9 +- .../compose/components/language_dropdown.js | 19 +- .../features/filters/added_to_filter.js | 102 ++++++++++ .../features/filters/select_filter.js | 192 ++++++++++++++++++ .../features/ui/components/filter_modal.js | 134 ++++++++++++ .../features/ui/components/modal_root.js | 2 + .../features/ui/util/async-components.js | 4 + app/javascript/mastodon/reducers/filters.js | 11 +- app/javascript/mastodon/selectors/index.js | 19 +- app/javascript/mastodon/utils/filters.js | 16 ++ app/javascript/mastodon/utils/icons.js | 13 ++ .../styles/mastodon/components.scss | 18 ++ app/models/concerns/account_interactions.rb | 10 +- app/models/custom_filter.rb | 28 ++- app/models/custom_filter_status.rb | 37 ++++ app/models/form/status_filter_batch_action.rb | 34 ++++ app/presenters/filter_result_presenter.rb | 2 +- .../status_relationships_presenter.rb | 7 +- .../rest/filter_result_serializer.rb | 5 + app/serializers/rest/filter_serializer.rb | 1 + .../rest/filter_status_serializer.rb | 13 ++ app/views/filters/_filter.html.haml | 9 + app/views/filters/_filter_fields.html.haml | 7 + .../filters/statuses/_status_filter.html.haml | 37 ++++ app/views/filters/statuses/index.html.haml | 38 ++++ config/locales/en.yml | 15 ++ config/routes.rb | 11 +- ...808101323_create_custom_filter_statuses.rb | 12 ++ db/schema.rb | 13 +- .../v1/filters/statuses_controller_spec.rb | 116 +++++++++++ .../api/v1/statuses_controller_spec.rb | 27 +++ .../custom_filter_status_fabricator.rb | 4 + .../status_relationships_presenter_spec.rb | 27 +++ 40 files changed, 1138 insertions(+), 63 deletions(-) create mode 100644 app/controllers/api/v1/filters/statuses_controller.rb create mode 100644 app/controllers/filters/statuses_controller.rb create mode 100644 app/javascript/mastodon/actions/filters.js create mode 100644 app/javascript/mastodon/features/filters/added_to_filter.js create mode 100644 app/javascript/mastodon/features/filters/select_filter.js create mode 100644 app/javascript/mastodon/features/ui/components/filter_modal.js create mode 100644 app/javascript/mastodon/utils/filters.js create mode 100644 app/javascript/mastodon/utils/icons.js create mode 100644 app/models/custom_filter_status.rb create mode 100644 app/models/form/status_filter_batch_action.rb create mode 100644 app/serializers/rest/filter_status_serializer.rb create mode 100644 app/views/filters/statuses/_status_filter.html.haml create mode 100644 app/views/filters/statuses/index.html.haml create mode 100644 db/migrate/20220808101323_create_custom_filter_statuses.rb create mode 100644 spec/controllers/api/v1/filters/statuses_controller_spec.rb create mode 100644 spec/fabricators/custom_filter_status_fabricator.rb diff --git a/app/controllers/api/v1/filters/statuses_controller.rb b/app/controllers/api/v1/filters/statuses_controller.rb new file mode 100644 index 00000000000..b6bed306fca --- /dev/null +++ b/app/controllers/api/v1/filters/statuses_controller.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +class Api::V1::Filters::StatusesController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:filters' }, only: [:index, :show] + before_action -> { doorkeeper_authorize! :write, :'write:filters' }, except: [:index, :show] + before_action :require_user! + + before_action :set_status_filters, only: :index + before_action :set_status_filter, only: [:show, :destroy] + + def index + render json: @status_filters, each_serializer: REST::FilterStatusSerializer + end + + def create + @status_filter = current_account.custom_filters.find(params[:filter_id]).statuses.create!(resource_params) + + render json: @status_filter, serializer: REST::FilterStatusSerializer + end + + def show + render json: @status_filter, serializer: REST::FilterStatusSerializer + end + + def destroy + @status_filter.destroy! + render_empty + end + + private + + def set_status_filters + filter = current_account.custom_filters.includes(:statuses).find(params[:filter_id]) + @status_filters = filter.statuses + end + + def set_status_filter + @status_filter = CustomFilterStatus.includes(:custom_filter).where(custom_filter: { account: current_account }).find(params[:id]) + end + + def resource_params + params.permit(:status_id) + end +end diff --git a/app/controllers/filters/statuses_controller.rb b/app/controllers/filters/statuses_controller.rb new file mode 100644 index 00000000000..cc493c22c64 --- /dev/null +++ b/app/controllers/filters/statuses_controller.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class Filters::StatusesController < ApplicationController + layout 'admin' + + before_action :authenticate_user! + before_action :set_filter + before_action :set_status_filters + before_action :set_body_classes + + PER_PAGE = 20 + + def index + @status_filter_batch_action = Form::StatusFilterBatchAction.new + end + + def batch + @status_filter_batch_action = Form::StatusFilterBatchAction.new(status_filter_batch_action_params.merge(current_account: current_account, filter_id: params[:filter_id], type: action_from_button)) + @status_filter_batch_action.save! + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.statuses.no_status_selected') + ensure + redirect_to edit_filter_path(@filter) + end + + private + + def set_filter + @filter = current_account.custom_filters.find(params[:filter_id]) + end + + def set_status_filters + @status_filters = @filter.statuses.preload(:status).page(params[:page]).per(PER_PAGE) + end + + def status_filter_batch_action_params + params.require(:form_status_filter_batch_action).permit(status_filter_ids: []) + end + + def action_from_button + if params[:remove] + 'remove' + end + end + + def set_body_classes + @body_classes = 'admin' + end +end diff --git a/app/controllers/filters_controller.rb b/app/controllers/filters_controller.rb index 5ed53bce1d2..cc5cb5d9f28 100644 --- a/app/controllers/filters_controller.rb +++ b/app/controllers/filters_controller.rb @@ -8,7 +8,7 @@ class FiltersController < ApplicationController before_action :set_body_classes def index - @filters = current_account.custom_filters.includes(:keywords).order(:phrase) + @filters = current_account.custom_filters.includes(:keywords, :statuses).order(:phrase) end def new diff --git a/app/javascript/mastodon/actions/filters.js b/app/javascript/mastodon/actions/filters.js new file mode 100644 index 00000000000..76326802eae --- /dev/null +++ b/app/javascript/mastodon/actions/filters.js @@ -0,0 +1,93 @@ +import api from '../api'; +import { openModal } from './modal'; + +export const FILTERS_FETCH_REQUEST = 'FILTERS_FETCH_REQUEST'; +export const FILTERS_FETCH_SUCCESS = 'FILTERS_FETCH_SUCCESS'; +export const FILTERS_FETCH_FAIL = 'FILTERS_FETCH_FAIL'; + +export const FILTERS_STATUS_CREATE_REQUEST = 'FILTERS_STATUS_CREATE_REQUEST'; +export const FILTERS_STATUS_CREATE_SUCCESS = 'FILTERS_STATUS_CREATE_SUCCESS'; +export const FILTERS_STATUS_CREATE_FAIL = 'FILTERS_STATUS_CREATE_FAIL'; + +export const FILTERS_CREATE_REQUEST = 'FILTERS_CREATE_REQUEST'; +export const FILTERS_CREATE_SUCCESS = 'FILTERS_CREATE_SUCCESS'; +export const FILTERS_CREATE_FAIL = 'FILTERS_CREATE_FAIL'; + +export const initAddFilter = (status, { contextType }) => dispatch => + dispatch(openModal('FILTER', { + statusId: status?.get('id'), + contextType: contextType, + })); + +export const fetchFilters = () => (dispatch, getState) => { + dispatch({ + type: FILTERS_FETCH_REQUEST, + skipLoading: true, + }); + + api(getState) + .get('/api/v2/filters') + .then(({ data }) => dispatch({ + type: FILTERS_FETCH_SUCCESS, + filters: data, + skipLoading: true, + })) + .catch(err => dispatch({ + type: FILTERS_FETCH_FAIL, + err, + skipLoading: true, + skipAlert: true, + })); +}; + +export const createFilterStatus = (params, onSuccess, onFail) => (dispatch, getState) => { + dispatch(createFilterStatusRequest()); + + api(getState).post(`/api/v1/filters/${params.filter_id}/statuses`, params).then(response => { + dispatch(createFilterStatusSuccess(response.data)); + if (onSuccess) onSuccess(); + }).catch(error => { + dispatch(createFilterStatusFail(error)); + if (onFail) onFail(); + }); +}; + +export const createFilterStatusRequest = () => ({ + type: FILTERS_STATUS_CREATE_REQUEST, +}); + +export const createFilterStatusSuccess = filter_status => ({ + type: FILTERS_STATUS_CREATE_SUCCESS, + filter_status, +}); + +export const createFilterStatusFail = error => ({ + type: FILTERS_STATUS_CREATE_FAIL, + error, +}); + +export const createFilter = (params, onSuccess, onFail) => (dispatch, getState) => { + dispatch(createFilterRequest()); + + api(getState).post('/api/v2/filters', params).then(response => { + dispatch(createFilterSuccess(response.data)); + if (onSuccess) onSuccess(response.data); + }).catch(error => { + dispatch(createFilterFail(error)); + if (onFail) onFail(); + }); +}; + +export const createFilterRequest = () => ({ + type: FILTERS_CREATE_REQUEST, +}); + +export const createFilterSuccess = filter => ({ + type: FILTERS_CREATE_SUCCESS, + filter, +}); + +export const createFilterFail = error => ({ + type: FILTERS_CREATE_FAIL, + error, +}); diff --git a/app/javascript/mastodon/actions/statuses.js b/app/javascript/mastodon/actions/statuses.js index adc24eabfbf..32a4f1f8575 100644 --- a/app/javascript/mastodon/actions/statuses.js +++ b/app/javascript/mastodon/actions/statuses.js @@ -42,9 +42,9 @@ export function fetchStatusRequest(id, skipLoading) { }; }; -export function fetchStatus(id) { +export function fetchStatus(id, forceFetch = false) { return (dispatch, getState) => { - const skipLoading = getState().getIn(['statuses', id], null) !== null; + const skipLoading = !forceFetch && getState().getIn(['statuses', id], null) !== null; dispatch(fetchContext(id)); diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index dee935a6cda..d36311c6772 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -80,6 +80,7 @@ class Status extends ImmutablePureComponent { onOpenMedia: PropTypes.func, onOpenVideo: PropTypes.func, onBlock: PropTypes.func, + onAddFilter: PropTypes.func, onEmbed: PropTypes.func, onHeightChange: PropTypes.func, onToggleHidden: PropTypes.func, diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js index d44da482dfd..4b384e6e542 100644 --- a/app/javascript/mastodon/components/status_action_bar.js +++ b/app/javascript/mastodon/components/status_action_bar.js @@ -44,6 +44,7 @@ const messages = defineMessages({ unblockDomain: { id: 'account.unblock_domain', defaultMessage: 'Unblock domain {domain}' }, unmute: { id: 'account.unmute', defaultMessage: 'Unmute @{name}' }, unblock: { id: 'account.unblock', defaultMessage: 'Unblock @{name}' }, + filter: { id: 'status.filter', defaultMessage: 'Filter this post' }, }); const mapStateToProps = (state, { status }) => ({ @@ -80,6 +81,7 @@ class StatusActionBar extends ImmutablePureComponent { onPin: PropTypes.func, onBookmark: PropTypes.func, onFilter: PropTypes.func, + onAddFilter: PropTypes.func, withDismiss: PropTypes.bool, withCounters: PropTypes.bool, scrollKey: PropTypes.string, @@ -211,8 +213,8 @@ class StatusActionBar extends ImmutablePureComponent { this.props.onMuteConversation(this.props.status); } - handleFilter = () => { - this.props.onFilter(); + handleFilterClick = () => { + this.props.onAddFilter(this.props.status); } handleCopy = () => { @@ -235,7 +237,7 @@ class StatusActionBar extends ImmutablePureComponent { } - handleFilterClick = () => { + handleHideClick = () => { this.props.onFilter(); } @@ -294,6 +296,12 @@ class StatusActionBar extends ImmutablePureComponent { menu.push({ text: intl.formatMessage(messages.block, { name: account.get('username') }), action: this.handleBlockClick }); } + if (!this.props.onFilter) { + menu.push(null); + menu.push({ text: intl.formatMessage(messages.filter), action: this.handleFilterClick }); + menu.push(null); + } + menu.push({ text: intl.formatMessage(messages.report, { name: account.get('username') }), action: this.handleReport }); if (account.get('acct') !== account.get('username')) { @@ -343,7 +351,7 @@ class StatusActionBar extends ImmutablePureComponent { ); const filterButton = this.props.onFilter && ( - + ); return ( diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js index ef0aca13a6d..28698b082d6 100644 --- a/app/javascript/mastodon/containers/status_container.js +++ b/app/javascript/mastodon/containers/status_container.js @@ -34,6 +34,9 @@ import { blockDomain, unblockDomain, } from '../actions/domain_blocks'; +import { + initAddFilter, +} from '../actions/filters'; import { initMuteModal } from '../actions/mutes'; import { initBlockModal } from '../actions/blocks'; import { initBoostModal } from '../actions/boosts'; @@ -66,7 +69,7 @@ const makeMapStateToProps = () => { return mapStateToProps; }; -const mapDispatchToProps = (dispatch, { intl }) => ({ +const mapDispatchToProps = (dispatch, { intl, contextType }) => ({ onReply (status, router) { dispatch((_, getState) => { @@ -176,6 +179,10 @@ const mapDispatchToProps = (dispatch, { intl }) => ({ dispatch(initReport(status.get('account'), status)); }, + onAddFilter (status) { + dispatch(initAddFilter(status, { contextType })); + }, + onMute (account) { dispatch(initMuteModal(account)); }, diff --git a/app/javascript/mastodon/features/compose/components/language_dropdown.js b/app/javascript/mastodon/features/compose/components/language_dropdown.js index d76490c775f..0af3db7a461 100644 --- a/app/javascript/mastodon/features/compose/components/language_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/language_dropdown.js @@ -8,6 +8,7 @@ import spring from 'react-motion/lib/spring'; import { supportsPassiveEvents } from 'detect-passive-events'; import classNames from 'classnames'; import { languages as preloadedLanguages } from 'mastodon/initial_state'; +import { loupeIcon, deleteIcon } from 'mastodon/utils/icons'; import fuzzysort from 'fuzzysort'; const messages = defineMessages({ @@ -16,22 +17,6 @@ const messages = defineMessages({ clear: { id: 'emoji_button.clear', defaultMessage: 'Clear' }, }); -// Copied from emoji-mart for consistency with emoji picker and since -// they don't export the icons in the package -const icons = { - loupe: ( - - - - ), - - delete: ( - - - - ), -}; - const listenerOptions = supportsPassiveEvents ? { passive: true } : false; class LanguageDropdownMenu extends React.PureComponent { @@ -242,7 +227,7 @@ class LanguageDropdownMenu extends React.PureComponent {
- +
diff --git a/app/javascript/mastodon/features/filters/added_to_filter.js b/app/javascript/mastodon/features/filters/added_to_filter.js new file mode 100644 index 00000000000..3785eb3c5aa --- /dev/null +++ b/app/javascript/mastodon/features/filters/added_to_filter.js @@ -0,0 +1,102 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import { FormattedMessage } from 'react-intl'; +import { toServerSideType } from 'mastodon/utils/filters'; +import Button from 'mastodon/components/button'; +import { connect } from 'react-redux'; + +const mapStateToProps = (state, { filterId }) => ({ + filter: state.getIn(['filters', filterId]), +}); + +export default @connect(mapStateToProps) +class AddedToFilter extends React.PureComponent { + + static propTypes = { + onClose: PropTypes.func.isRequired, + contextType: PropTypes.string, + filter: ImmutablePropTypes.map.isRequired, + dispatch: PropTypes.func.isRequired, + }; + + handleCloseClick = () => { + const { onClose } = this.props; + onClose(); + }; + + render () { + const { filter, contextType } = this.props; + + let expiredMessage = null; + if (filter.get('expires_at') && filter.get('expires_at') < new Date()) { + expiredMessage = ( + +

+

+ +

+ + ); + } + + let contextMismatchMessage = null; + if (contextType && !filter.get('context').includes(toServerSideType(contextType))) { + contextMismatchMessage = ( + +

+

+ +

+ + ); + } + + const settings_link = ( + + + + ); + + return ( + +

+

+ +

+ + {expiredMessage} + {contextMismatchMessage} + +

+

+ +

+ +
+ +
+ +
+ + ); + } + +} diff --git a/app/javascript/mastodon/features/filters/select_filter.js b/app/javascript/mastodon/features/filters/select_filter.js new file mode 100644 index 00000000000..b5b3545296a --- /dev/null +++ b/app/javascript/mastodon/features/filters/select_filter.js @@ -0,0 +1,192 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { connect } from 'react-redux'; +import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; +import { toServerSideType } from 'mastodon/utils/filters'; +import { loupeIcon, deleteIcon } from 'mastodon/utils/icons'; +import Icon from 'mastodon/components/icon'; +import fuzzysort from 'fuzzysort'; + +const messages = defineMessages({ + search: { id: 'filter_modal.select_filter.search', defaultMessage: 'Search or create' }, + clear: { id: 'emoji_button.clear', defaultMessage: 'Clear' }, +}); + +const mapStateToProps = (state, { contextType }) => ({ + filters: Array.from(state.get('filters').values()).map((filter) => [ + filter.get('id'), + filter.get('title'), + filter.get('keywords')?.map((keyword) => keyword.get('keyword')).join('\n'), + filter.get('expires_at') && filter.get('expires_at') < new Date(), + contextType && !filter.get('context').includes(toServerSideType(contextType)), + ]), +}); + +export default @connect(mapStateToProps) +@injectIntl +class SelectFilter extends React.PureComponent { + + static propTypes = { + onSelectFilter: PropTypes.func.isRequired, + onNewFilter: PropTypes.func.isRequired, + filters: PropTypes.arrayOf(PropTypes.arrayOf(PropTypes.object)), + intl: PropTypes.object.isRequired, + }; + + state = { + searchValue: '', + }; + + search () { + const { filters } = this.props; + const { searchValue } = this.state; + + if (searchValue === '') { + return filters; + } + + return fuzzysort.go(searchValue, filters, { + keys: ['1', '2'], + limit: 5, + threshold: -10000, + }).map(result => result.obj); + } + + renderItem = filter => { + let warning = null; + if (filter[3] || filter[4]) { + warning = ( + + ( + {filter[3] && } + {filter[3] && filter[4] && ', '} + {filter[4] && } + ) + + ); + } + + return ( +
+ {filter[1]} {warning} +
+ ); + } + + renderCreateNew (name) { + return ( +
+ +
+ ); + } + + handleSearchChange = ({ target }) => { + this.setState({ searchValue: target.value }); + } + + setListRef = c => { + this.listNode = c; + } + + handleKeyDown = e => { + const index = Array.from(this.listNode.childNodes).findIndex(node => node === e.currentTarget); + + let element = null; + + switch(e.key) { + case ' ': + case 'Enter': + e.currentTarget.click(); + break; + case 'ArrowDown': + element = this.listNode.childNodes[index + 1] || this.listNode.firstChild; + break; + case 'ArrowUp': + element = this.listNode.childNodes[index - 1] || this.listNode.lastChild; + break; + case 'Tab': + if (e.shiftKey) { + element = this.listNode.childNodes[index - 1] || this.listNode.lastChild; + } else { + element = this.listNode.childNodes[index + 1] || this.listNode.firstChild; + } + break; + case 'Home': + element = this.listNode.firstChild; + break; + case 'End': + element = this.listNode.lastChild; + break; + } + + if (element) { + element.focus(); + e.preventDefault(); + e.stopPropagation(); + } + } + + handleSearchKeyDown = e => { + let element = null; + + switch(e.key) { + case 'Tab': + case 'ArrowDown': + element = this.listNode.firstChild; + + if (element) { + element.focus(); + e.preventDefault(); + e.stopPropagation(); + } + + break; + } + } + + handleClear = () => { + this.setState({ searchValue: '' }); + } + + handleItemClick = e => { + const value = e.currentTarget.getAttribute('data-index'); + + e.preventDefault(); + + this.props.onSelectFilter(value); + } + + handleNewFilterClick = e => { + e.preventDefault(); + + this.props.onNewFilter(this.state.searchValue); + }; + + render () { + const { intl } = this.props; + + const { searchValue } = this.state; + const isSearching = searchValue !== ''; + const results = this.search(); + + return ( + +

+

+ +
+ + +
+ +
+ {results.map(this.renderItem)} + {isSearching && this.renderCreateNew(searchValue) } +
+ + + ); + } + +} diff --git a/app/javascript/mastodon/features/ui/components/filter_modal.js b/app/javascript/mastodon/features/ui/components/filter_modal.js new file mode 100644 index 00000000000..376db961d1e --- /dev/null +++ b/app/javascript/mastodon/features/ui/components/filter_modal.js @@ -0,0 +1,134 @@ +import React from 'react'; +import { connect } from 'react-redux'; +import { fetchStatus } from 'mastodon/actions/statuses'; +import { fetchFilters, createFilter, createFilterStatus } from 'mastodon/actions/filters'; +import PropTypes from 'prop-types'; +import { defineMessages, FormattedMessage, injectIntl } from 'react-intl'; +import ImmutablePureComponent from 'react-immutable-pure-component'; +import IconButton from 'mastodon/components/icon_button'; +import SelectFilter from 'mastodon/features/filters/select_filter'; +import AddedToFilter from 'mastodon/features/filters/added_to_filter'; + +const messages = defineMessages({ + close: { id: 'lightbox.close', defaultMessage: 'Close' }, +}); + +export default @connect(undefined) +@injectIntl +class FilterModal extends ImmutablePureComponent { + + static propTypes = { + statusId: PropTypes.string.isRequired, + contextType: PropTypes.string, + dispatch: PropTypes.func.isRequired, + intl: PropTypes.object.isRequired, + }; + + state = { + step: 'select', + filterId: null, + isSubmitting: false, + isSubmitted: false, + }; + + handleNewFilterSuccess = (result) => { + this.handleSelectFilter(result.id); + }; + + handleSuccess = () => { + const { dispatch, statusId } = this.props; + dispatch(fetchStatus(statusId, true)); + this.setState({ isSubmitting: false, isSubmitted: true, step: 'submitted' }); + }; + + handleFail = () => { + this.setState({ isSubmitting: false }); + }; + + handleNextStep = step => { + this.setState({ step }); + }; + + handleSelectFilter = (filterId) => { + const { dispatch, statusId } = this.props; + + this.setState({ isSubmitting: true, filterId }); + + dispatch(createFilterStatus({ + filter_id: filterId, + status_id: statusId, + }, this.handleSuccess, this.handleFail)); + }; + + handleNewFilter = (title) => { + const { dispatch } = this.props; + + this.setState({ isSubmitting: true }); + + dispatch(createFilter({ + title, + context: ['home', 'notifications', 'public', 'thread', 'account'], + action: 'warn', + }, this.handleNewFilterSuccess, this.handleFail)); + }; + + componentDidMount () { + const { dispatch } = this.props; + + dispatch(fetchFilters()); + } + + render () { + const { + intl, + statusId, + contextType, + onClose, + } = this.props; + + const { + step, + filterId, + } = this.state; + + let stepComponent; + + switch(step) { + case 'select': + stepComponent = ( + + ); + break; + case 'create': + stepComponent = null; + break; + case 'submitted': + stepComponent = ( + + ); + } + + return ( +
+
+ + +
+ +
+ {stepComponent} +
+
+ ); + } + +} diff --git a/app/javascript/mastodon/features/ui/components/modal_root.js b/app/javascript/mastodon/features/ui/components/modal_root.js index 3fc235849df..b2c30e07913 100644 --- a/app/javascript/mastodon/features/ui/components/modal_root.js +++ b/app/javascript/mastodon/features/ui/components/modal_root.js @@ -20,6 +20,7 @@ import { ListEditor, ListAdder, CompareHistoryModal, + FilterModal, } from 'mastodon/features/ui/util/async-components'; const MODAL_COMPONENTS = { @@ -37,6 +38,7 @@ const MODAL_COMPONENTS = { 'FOCAL_POINT': () => Promise.resolve({ default: FocalPointModal }), 'LIST_ADDER': ListAdder, 'COMPARE_HISTORY': CompareHistoryModal, + 'FILTER': FilterModal, }; export default class ModalRoot extends React.PureComponent { diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js index 92c683e2f89..29b06206a15 100644 --- a/app/javascript/mastodon/features/ui/util/async-components.js +++ b/app/javascript/mastodon/features/ui/util/async-components.js @@ -161,3 +161,7 @@ export function CompareHistoryModal () { export function Explore () { return import(/* webpackChunkName: "features/explore" */'../../explore'); } + +export function FilterModal () { + return import(/*webpackChunkName: "modals/filter_modal" */'../components/filter_modal'); +} diff --git a/app/javascript/mastodon/reducers/filters.js b/app/javascript/mastodon/reducers/filters.js index 14b7040273e..cc1d3349c59 100644 --- a/app/javascript/mastodon/reducers/filters.js +++ b/app/javascript/mastodon/reducers/filters.js @@ -1,4 +1,5 @@ import { FILTERS_IMPORT } from '../actions/importer'; +import { FILTERS_FETCH_SUCCESS, FILTERS_CREATE_SUCCESS } from '../actions/filters'; import { Map as ImmutableMap, is, fromJS } from 'immutable'; const normalizeFilter = (state, filter) => { @@ -7,13 +8,17 @@ const normalizeFilter = (state, filter) => { title: filter.title, context: filter.context, filter_action: filter.filter_action, + keywords: filter.keywords, expires_at: filter.expires_at ? Date.parse(filter.expires_at) : null, }); if (is(state.get(filter.id), normalizedFilter)) { return state; } else { - return state.set(filter.id, normalizedFilter); + // Do not overwrite keywords when receiving a partial filter + return state.update(filter.id, ImmutableMap(), (old) => ( + old.mergeWith(((old_value, new_value) => (new_value === undefined ? old_value : new_value)), normalizedFilter) + )); } }; @@ -27,6 +32,10 @@ const normalizeFilters = (state, filters) => { export default function filters(state = ImmutableMap(), action) { switch(action.type) { + case FILTERS_CREATE_SUCCESS: + return normalizeFilter(state, action.filter); + case FILTERS_FETCH_SUCCESS: + //TODO: handle deleting obsolete filters case FILTERS_IMPORT: return normalizeFilters(state, action.filters); default: diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index 187e3306dd9..3dd7f489722 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -1,5 +1,6 @@ import { createSelector } from 'reselect'; import { List as ImmutableList, Map as ImmutableMap } from 'immutable'; +import { toServerSideType } from 'mastodon/utils/filters'; import { me } from '../initial_state'; const getAccountBase = (state, id) => state.getIn(['accounts', id], null); @@ -20,23 +21,6 @@ export const makeGetAccount = () => { }); }; -const toServerSideType = columnType => { - switch (columnType) { - case 'home': - case 'notifications': - case 'public': - case 'thread': - case 'account': - return columnType; - default: - if (columnType.indexOf('list:') > -1) { - return 'home'; - } else { - return 'public'; // community, account, hashtag - } - } -}; - const getFilters = (state, { contextType }) => { if (!contextType) return null; @@ -73,6 +57,7 @@ export const makeGetStatus = () => { if (filterResults.some((result) => filters.getIn([result.get('filter'), 'filter_action']) === 'hide')) { return null; } + filterResults = filterResults.filter(result => filters.has(result.get('filter'))); if (!filterResults.isEmpty()) { filtered = filterResults.map(result => filters.getIn([result.get('filter'), 'title'])); } diff --git a/app/javascript/mastodon/utils/filters.js b/app/javascript/mastodon/utils/filters.js new file mode 100644 index 00000000000..97b433a991f --- /dev/null +++ b/app/javascript/mastodon/utils/filters.js @@ -0,0 +1,16 @@ +export const toServerSideType = columnType => { + switch (columnType) { + case 'home': + case 'notifications': + case 'public': + case 'thread': + case 'account': + return columnType; + default: + if (columnType.indexOf('list:') > -1) { + return 'home'; + } else { + return 'public'; // community, account, hashtag + } + } +}; diff --git a/app/javascript/mastodon/utils/icons.js b/app/javascript/mastodon/utils/icons.js new file mode 100644 index 00000000000..be566032e06 --- /dev/null +++ b/app/javascript/mastodon/utils/icons.js @@ -0,0 +1,13 @@ +// Copied from emoji-mart for consistency with emoji picker and since +// they don't export the icons in the package +export const loupeIcon = ( + + + +); + +export const deleteIcon = ( + + + +); diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index a0a39812b0f..f5377a85896 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -5233,6 +5233,16 @@ a.status-card.compact:hover { line-height: 22px; color: lighten($inverted-text-color, 16%); margin-bottom: 30px; + + a { + text-decoration: none; + color: $inverted-text-color; + font-weight: 500; + + &:hover { + text-decoration: underline; + } + } } &__actions { @@ -5379,6 +5389,14 @@ a.status-card.compact:hover { background: transparent; margin: 15px 0; } + + .emoji-mart-search { + padding-right: 10px; + } + + .emoji-mart-search-icon { + right: 10px + 5px; + } } .report-modal__container { diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index a7401362f40..9b358d338e3 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -249,15 +249,7 @@ module AccountInteractions def status_matches_filters(status) active_filters = CustomFilter.cached_filters_for(id) - - filter_matches = active_filters.filter_map do |filter, rules| - next if rules[:keywords].blank? - - match = rules[:keywords].match(status.proper.searchable_text) - FilterResultPresenter.new(filter: filter, keyword_matches: [match.to_s]) unless match.nil? - end - - filter_matches + CustomFilter.apply_cached_filters(active_filters, status) end def followers_for_local_distribution diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 985eab1254c..da2a914934c 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -34,6 +34,7 @@ class CustomFilter < ApplicationRecord belongs_to :account has_many :keywords, class_name: 'CustomFilterKeyword', foreign_key: :custom_filter_id, inverse_of: :custom_filter, dependent: :destroy + has_many :statuses, class_name: 'CustomFilterStatus', foreign_key: :custom_filter_id, inverse_of: :custom_filter, dependent: :destroy accepts_nested_attributes_for :keywords, reject_if: :all_blank, allow_destroy: true validates :title, :context, presence: true @@ -62,8 +63,10 @@ class CustomFilter < ApplicationRecord def self.cached_filters_for(account_id) active_filters = Rails.cache.fetch("filters:v3:#{account_id}") do + filters_hash = {} + scope = CustomFilterKeyword.includes(:custom_filter).where(custom_filter: { account_id: account_id }).where(Arel.sql('expires_at IS NULL OR expires_at > NOW()')) - scope.to_a.group_by(&:custom_filter).map do |filter, keywords| + scope.to_a.group_by(&:custom_filter).each do |filter, keywords| keywords.map! do |keyword| if keyword.whole_word sb = /\A[[:word:]]/.match?(keyword.keyword) ? '\b' : '' @@ -74,13 +77,34 @@ class CustomFilter < ApplicationRecord /#{Regexp.escape(keyword.keyword)}/i end end - [filter, { keywords: Regexp.union(keywords) }] + + filters_hash[filter.id] = { keywords: Regexp.union(keywords), filter: filter } + end.to_h + + scope = CustomFilterStatus.includes(:custom_filter).where(custom_filter: { account_id: account_id }).where(Arel.sql('expires_at IS NULL OR expires_at > NOW()')) + scope.to_a.group_by(&:custom_filter).each do |filter, statuses| + filters_hash[filter.id] ||= { filter: filter } + filters_hash[filter.id].merge!(status_ids: statuses.map(&:status_id)) end + + filters_hash.values.map { |cache| [cache.delete(:filter), cache] } end.to_a active_filters.select { |custom_filter, _| !custom_filter.expired? } end + def self.apply_cached_filters(cached_filters, status) + cached_filters.filter_map do |filter, rules| + match = rules[:keywords].match(status.proper.searchable_text) if rules[:keywords].present? + keyword_matches = [match.to_s] unless match.nil? + + status_matches = [status.id, status.reblog_of_id].compact & rules[:status_ids] if rules[:status_ids].present? + + next if keyword_matches.blank? && status_matches.blank? + FilterResultPresenter.new(filter: filter, keyword_matches: keyword_matches, status_matches: status_matches) + end + end + def prepare_cache_invalidation! @should_invalidate_cache = true end diff --git a/app/models/custom_filter_status.rb b/app/models/custom_filter_status.rb new file mode 100644 index 00000000000..b6bea139431 --- /dev/null +++ b/app/models/custom_filter_status.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: custom_filter_statuses +# +# id :bigint(8) not null, primary key +# custom_filter_id :bigint(8) not null +# status_id :bigint(8) default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class CustomFilterStatus < ApplicationRecord + belongs_to :custom_filter + belongs_to :status + + validates :status, uniqueness: { scope: :custom_filter } + validate :validate_status_access + + before_save :prepare_cache_invalidation! + before_destroy :prepare_cache_invalidation! + after_commit :invalidate_cache! + + private + + def validate_status_access + errors.add(:status_id, :invalid) unless StatusPolicy.new(custom_filter.account, status).show? + end + + def prepare_cache_invalidation! + custom_filter.prepare_cache_invalidation! + end + + def invalidate_cache! + custom_filter.invalidate_cache! + end +end diff --git a/app/models/form/status_filter_batch_action.rb b/app/models/form/status_filter_batch_action.rb new file mode 100644 index 00000000000..d87bd5cc4d8 --- /dev/null +++ b/app/models/form/status_filter_batch_action.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class Form::StatusFilterBatchAction + include ActiveModel::Model + include AccountableConcern + include Authorization + + attr_accessor :current_account, :type, + :status_filter_ids, :filter_id + + def save! + process_action! + end + + private + + def status_filters + filter = current_account.custom_filters.find(filter_id) + filter.statuses.where(id: status_filter_ids) + end + + def process_action! + return if status_filter_ids.empty? + + case type + when 'remove' + handle_remove! + end + end + + def handle_remove! + status_filters.destroy_all + end +end diff --git a/app/presenters/filter_result_presenter.rb b/app/presenters/filter_result_presenter.rb index 677225f5ec2..1e9e8f3c19a 100644 --- a/app/presenters/filter_result_presenter.rb +++ b/app/presenters/filter_result_presenter.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true class FilterResultPresenter < ActiveModelSerializers::Model - attributes :filter, :keyword_matches + attributes :filter, :keyword_matches, :status_matches end diff --git a/app/presenters/status_relationships_presenter.rb b/app/presenters/status_relationships_presenter.rb index d7ffb1954af..be818a2de79 100644 --- a/app/presenters/status_relationships_presenter.rb +++ b/app/presenters/status_relationships_presenter.rb @@ -33,12 +33,7 @@ class StatusRelationshipsPresenter active_filters = CustomFilter.cached_filters_for(current_account_id) @filters_map = statuses.each_with_object({}) do |status, h| - filter_matches = active_filters.filter_map do |filter, rules| - next if rules[:keywords].blank? - - match = rules[:keywords].match(status.proper.searchable_text) - FilterResultPresenter.new(filter: filter, keyword_matches: [match.to_s]) unless match.nil? - end + filter_matches = CustomFilter.apply_cached_filters(active_filters, status) unless filter_matches.empty? h[status.id] = filter_matches diff --git a/app/serializers/rest/filter_result_serializer.rb b/app/serializers/rest/filter_result_serializer.rb index 0ef4db79a87..54ead2f1f14 100644 --- a/app/serializers/rest/filter_result_serializer.rb +++ b/app/serializers/rest/filter_result_serializer.rb @@ -3,4 +3,9 @@ class REST::FilterResultSerializer < ActiveModel::Serializer belongs_to :filter, serializer: REST::FilterSerializer has_many :keyword_matches + has_many :status_matches + + def status_matches + object.status_matches&.map(&:to_s) + end end diff --git a/app/serializers/rest/filter_serializer.rb b/app/serializers/rest/filter_serializer.rb index 98d7edb175a..8816dd8076b 100644 --- a/app/serializers/rest/filter_serializer.rb +++ b/app/serializers/rest/filter_serializer.rb @@ -3,6 +3,7 @@ class REST::FilterSerializer < ActiveModel::Serializer attributes :id, :title, :context, :expires_at, :filter_action has_many :keywords, serializer: REST::FilterKeywordSerializer, if: :rules_requested? + has_many :statuses, serializer: REST::FilterStatusSerializer, if: :rules_requested? def id object.id.to_s diff --git a/app/serializers/rest/filter_status_serializer.rb b/app/serializers/rest/filter_status_serializer.rb new file mode 100644 index 00000000000..6bcbaa249c6 --- /dev/null +++ b/app/serializers/rest/filter_status_serializer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class REST::FilterStatusSerializer < ActiveModel::Serializer + attributes :id, :status_id + + def id + object.id.to_s + end + + def status_id + object.status_id.to_s + end +end diff --git a/app/views/filters/_filter.html.haml b/app/views/filters/_filter.html.haml index 2ab014081c8..9993ad2ee8e 100644 --- a/app/views/filters/_filter.html.haml +++ b/app/views/filters/_filter.html.haml @@ -22,6 +22,15 @@ - keywords = filter.keywords.map(&:keyword) - keywords = keywords.take(5) + ['…'] if keywords.size > 5 # TODO = keywords.join(', ') + - unless filter.statuses.empty? + %li.permissions-list__item + .permissions-list__item__icon + = fa_icon('comment') + .permissions-list__item__text + .permissions-list__item__text__title + = t('filters.index.statuses', count: filter.statuses.size) + .permissions-list__item__text__type + = t('filters.index.statuses_long', count: filter.statuses.size) .announcements-list__item__action-bar .announcements-list__item__meta diff --git a/app/views/filters/_filter_fields.html.haml b/app/views/filters/_filter_fields.html.haml index 1a52faa7af5..c58978f5a35 100644 --- a/app/views/filters/_filter_fields.html.haml +++ b/app/views/filters/_filter_fields.html.haml @@ -14,6 +14,13 @@ %hr.spacer/ +- unless f.object.statuses.empty? + %h4= t('filters.edit.statuses') + + %p.muted-hint= t('filters.edit.statuses_hint_html', path: filter_statuses_path(f.object)) + + %hr.spacer/ + %h4= t('filters.edit.keywords') .table-wrapper diff --git a/app/views/filters/statuses/_status_filter.html.haml b/app/views/filters/statuses/_status_filter.html.haml new file mode 100644 index 00000000000..ba1170cf92a --- /dev/null +++ b/app/views/filters/statuses/_status_filter.html.haml @@ -0,0 +1,37 @@ +- status = status_filter.status.proper + +.batch-table__row + %label.batch-table__row__select.batch-checkbox + = f.check_box :status_filter_ids, { multiple: true, include_hidden: false }, status_filter.id + .batch-table__row__content + .status__content>< + - if status.spoiler_text.blank? + = prerender_custom_emojis(status_content_format(status), status.emojis) + - else + %details< + %summary>< + %strong> Content warning: #{prerender_custom_emojis(h(status.spoiler_text), status.emojis)} + = prerender_custom_emojis(status_content_format(status), status.emojis) + + - status.ordered_media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name + + .detailed-status__meta + = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'name-tag', target: '_blank', rel: 'noopener noreferrer' do + = image_tag(status.account.avatar.url, width: 15, height: 15, alt: display_name(status.account), class: 'avatar') + .username= status.account.acct + · + = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener noreferrer' do + %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) + - if status.edited? + · + = t('statuses.edited_at_html', date: content_tag(:time, l(status.edited_at), datetime: status.edited_at.iso8601, title: l(status.edited_at), class: 'formatted')) + · + = fa_visibility_icon(status) + = t("statuses.visibilities.#{status.visibility}") + - if status.sensitive? + · + = fa_icon('eye-slash fw') + = t('stream_entries.sensitive_content') diff --git a/app/views/filters/statuses/index.html.haml b/app/views/filters/statuses/index.html.haml new file mode 100644 index 00000000000..886de58fa06 --- /dev/null +++ b/app/views/filters/statuses/index.html.haml @@ -0,0 +1,38 @@ +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +- content_for :page_title do + = t('filters.statuses.index.title') + \- + = @filter.title + +.filters + .back-link + = link_to edit_filter_path(@filter) do + = fa_icon 'chevron-left fw' + = t('filters.statuses.back_to_filter') + +%p.hint= t('filters.statuses.index.hint') + +%hr.spacer/ + += form_for(@status_filter_batch_action, url: batch_filter_statuses_path(@filter.id)) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - Admin::StatusFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + - unless @status_filters.empty? + = f.button safe_join([fa_icon('times'), t('filters.statuses.batch.remove')]), name: :remove, class: 'table-action-link', type: :submit + .batch-table__body + - if @status_filters.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'status_filter', collection: @status_filters, locals: { f: f } + += paginate @status_filters diff --git a/config/locales/en.yml b/config/locales/en.yml index 2cd4f45ac2f..596cc1a284c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1181,6 +1181,8 @@ en: edit: add_keyword: Add keyword keywords: Keywords + statuses: Individual posts + statuses_hint_html: This filter applies to select individual posts regardless of whether they match the keywords below. You can review these posts and remove them from the filter by clicking here. title: Edit filter errors: deprecated_api_multiple_keywords: These parameters cannot be changed from this application because they apply to more than one filter keyword. Use a more recent application or the web interface. @@ -1194,10 +1196,23 @@ en: keywords: one: "%{count} keyword" other: "%{count} keywords" + statuses: + one: "%{count} post" + other: "%{count} posts" + statuses_long: + one: "%{count} individual post hidden" + other: "%{count} individual posts hidden" title: Filters new: save: Save new filter title: Add new filter + statuses: + back_to_filter: Back to filter + batch: + remove: Remove from filter + index: + hint: This filter applies to select individual posts regardless of other criteria. You can add more posts to this filter from the Web interface. + title: Filtered posts footer: developers: Developers more: More… diff --git a/config/routes.rb b/config/routes.rb index 7dc9f391db2..dff0add3ae0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -177,7 +177,14 @@ Rails.application.routes.draw do resources :tags, only: [:show] resources :emojis, only: [:show] resources :invites, only: [:index, :create, :destroy] - resources :filters, except: [:show] + resources :filters, except: [:show] do + resources :statuses, only: [:index], controller: 'filters/statuses' do + collection do + post :batch + end + end + end + resource :relationships, only: [:show, :update] resource :statuses_cleanup, controller: :statuses_cleanup, only: [:show, :update] @@ -448,12 +455,14 @@ Rails.application.routes.draw do resources :trends, only: [:index], controller: 'trends/tags' resources :filters, only: [:index, :create, :show, :update, :destroy] do resources :keywords, only: [:index, :create], controller: 'filters/keywords' + resources :statuses, only: [:index, :create], controller: 'filters/statuses' end resources :endorsements, only: [:index] resources :markers, only: [:index, :create] namespace :filters do resources :keywords, only: [:show, :update, :destroy] + resources :statuses, only: [:show, :destroy] end namespace :apps do diff --git a/db/migrate/20220808101323_create_custom_filter_statuses.rb b/db/migrate/20220808101323_create_custom_filter_statuses.rb new file mode 100644 index 00000000000..52f70374913 --- /dev/null +++ b/db/migrate/20220808101323_create_custom_filter_statuses.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class CreateCustomFilterStatuses < ActiveRecord::Migration[6.1] + def change + create_table :custom_filter_statuses do |t| + t.belongs_to :custom_filter, foreign_key: { on_delete: :cascade }, null: false + t.belongs_to :status, foreign_key: { on_delete: :cascade }, null: false + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 2263dc7d7cd..15ab2e85e78 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_07_14_171049) do +ActiveRecord::Schema.define(version: 2022_08_08_101323) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -348,6 +348,15 @@ ActiveRecord::Schema.define(version: 2022_07_14_171049) do t.index ["custom_filter_id"], name: "index_custom_filter_keywords_on_custom_filter_id" end + create_table "custom_filter_statuses", force: :cascade do |t| + t.bigint "custom_filter_id", null: false + t.bigint "status_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["custom_filter_id"], name: "index_custom_filter_statuses_on_custom_filter_id" + t.index ["status_id"], name: "index_custom_filter_statuses_on_status_id" + end + create_table "custom_filters", force: :cascade do |t| t.bigint "account_id" t.datetime "expires_at" @@ -1113,6 +1122,8 @@ ActiveRecord::Schema.define(version: 2022_07_14_171049) do add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade add_foreign_key "custom_filter_keywords", "custom_filters", on_delete: :cascade + add_foreign_key "custom_filter_statuses", "custom_filters", on_delete: :cascade + add_foreign_key "custom_filter_statuses", "statuses", on_delete: :cascade add_foreign_key "custom_filters", "accounts", on_delete: :cascade add_foreign_key "devices", "accounts", on_delete: :cascade add_foreign_key "devices", "oauth_access_tokens", column: "access_token_id", on_delete: :cascade diff --git a/spec/controllers/api/v1/filters/statuses_controller_spec.rb b/spec/controllers/api/v1/filters/statuses_controller_spec.rb new file mode 100644 index 00000000000..3b2399dd898 --- /dev/null +++ b/spec/controllers/api/v1/filters/statuses_controller_spec.rb @@ -0,0 +1,116 @@ +require 'rails_helper' + +RSpec.describe Api::V1::Filters::StatusesController, type: :controller do + render_views + + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + let(:filter) { Fabricate(:custom_filter, account: user.account) } + let(:other_user) { Fabricate(:user) } + let(:other_filter) { Fabricate(:custom_filter, account: other_user.account) } + + before do + allow(controller).to receive(:doorkeeper_token) { token } + end + + describe 'GET #index' do + let(:scopes) { 'read:filters' } + let!(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter) } + + it 'returns http success' do + get :index, params: { filter_id: filter.id } + expect(response).to have_http_status(200) + end + + context "when trying to access another's user filters" do + it 'returns http not found' do + get :index, params: { filter_id: other_filter.id } + expect(response).to have_http_status(404) + end + end + end + + describe 'POST #create' do + let(:scopes) { 'write:filters' } + let(:filter_id) { filter.id } + let!(:status) { Fabricate(:status) } + + before do + post :create, params: { filter_id: filter_id, status_id: status.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns a status filter' do + json = body_as_json + expect(json[:status_id]).to eq status.id.to_s + end + + it 'creates a status filter' do + filter = user.account.custom_filters.first + expect(filter).to_not be_nil + expect(filter.statuses.pluck(:status_id)).to eq [status.id] + end + + context "when trying to add to another another's user filters" do + let(:filter_id) { other_filter.id } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end + + describe 'GET #show' do + let(:scopes) { 'read:filters' } + let!(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter) } + + before do + get :show, params: { id: status_filter.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns expected data' do + json = body_as_json + expect(json[:status_id]).to eq status_filter.status_id.to_s + end + + context "when trying to access another user's filter keyword" do + let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: other_filter) } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end + + describe 'DELETE #destroy' do + let(:scopes) { 'write:filters' } + let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter) } + + before do + delete :destroy, params: { id: status_filter.id } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'removes the filter' do + expect { status_filter.reload }.to raise_error ActiveRecord::RecordNotFound + end + + context "when trying to update another user's filter keyword" do + let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: other_filter) } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end +end diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb index 4d104a198db..24810a5d27c 100644 --- a/spec/controllers/api/v1/statuses_controller_spec.rb +++ b/spec/controllers/api/v1/statuses_controller_spec.rb @@ -47,6 +47,33 @@ RSpec.describe Api::V1::StatusesController, type: :controller do end end + context 'when post is explicitly filtered' do + let(:status) { Fabricate(:status, text: 'hello world') } + + before do + filter = user.account.custom_filters.create!(phrase: 'filter1', context: %w(home), action: :hide) + filter.statuses.create!(status_id: status.id) + end + + it 'returns http success' do + get :show, params: { id: status.id } + expect(response).to have_http_status(200) + end + + it 'returns filter information' do + get :show, params: { id: status.id } + json = body_as_json + expect(json[:filtered][0]).to include({ + filter: a_hash_including({ + id: user.account.custom_filters.first.id.to_s, + title: 'filter1', + filter_action: 'hide', + }), + status_matches: [status.id.to_s], + }) + end + end + context 'when reblog includes filtered terms' do let(:status) { Fabricate(:status, reblog: Fabricate(:status, text: 'this toot is about that banned word')) } diff --git a/spec/fabricators/custom_filter_status_fabricator.rb b/spec/fabricators/custom_filter_status_fabricator.rb new file mode 100644 index 00000000000..d082b81c5ee --- /dev/null +++ b/spec/fabricators/custom_filter_status_fabricator.rb @@ -0,0 +1,4 @@ +Fabricator(:custom_filter_status) do + custom_filter + status +end diff --git a/spec/presenters/status_relationships_presenter_spec.rb b/spec/presenters/status_relationships_presenter_spec.rb index 5cd4929a63c..eaab922fd97 100644 --- a/spec/presenters/status_relationships_presenter_spec.rb +++ b/spec/presenters/status_relationships_presenter_spec.rb @@ -94,5 +94,32 @@ RSpec.describe StatusRelationshipsPresenter do expect(matched_filters[0].keyword_matches).to eq ['irrelevant'] end end + + context 'when post includes filtered individual statuses' do + let(:statuses) { [Fabricate(:status, text: 'hello world'), Fabricate(:status, reblog: Fabricate(:status, text: 'this toot is about an irrelevant word'))] } + let(:options) { {} } + + before do + filter = Account.find(current_account_id).custom_filters.create!(phrase: 'filter1', context: %w(home), action: :hide) + filter.statuses.create!(status_id: statuses[0].id) + filter.statuses.create!(status_id: statuses[1].reblog_of_id) + end + + it 'sets @filters_map to filter top-level status' do + matched_filters = presenter.filters_map[statuses[0].id] + expect(matched_filters.size).to eq 1 + + expect(matched_filters[0].filter.title).to eq 'filter1' + expect(matched_filters[0].status_matches).to eq [statuses[0].id] + end + + it 'sets @filters_map to filter reblogged status' do + matched_filters = presenter.filters_map[statuses[1].reblog_of_id] + expect(matched_filters.size).to eq 1 + + expect(matched_filters[0].filter.title).to eq 'filter1' + expect(matched_filters[0].status_matches).to eq [statuses[1].reblog_of_id] + end + end end end From afb8bc97d08c2738da7873ca42fea68e4672d65b Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Aug 2022 04:29:00 +0200 Subject: [PATCH 10/22] Fix quickly switching notification filters resulting in empty or incorrect list (#18960) --- app/javascript/mastodon/actions/notifications.js | 6 +++--- app/javascript/mastodon/reducers/notifications.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js index 3c42f71da32..7f62e6c0420 100644 --- a/app/javascript/mastodon/actions/notifications.js +++ b/app/javascript/mastodon/actions/notifications.js @@ -141,13 +141,13 @@ const excludeTypesFromFilter = filter => { const noOp = () => {}; -export function expandNotifications({ maxId } = {}, done = noOp) { +export function expandNotifications({ maxId, forceLoad } = {}, done = noOp) { return (dispatch, getState) => { const activeFilter = getState().getIn(['settings', 'notifications', 'quickFilter', 'active']); const notifications = getState().get('notifications'); const isLoadingMore = !!maxId; - if (notifications.get('isLoading')) { + if (notifications.get('isLoading') && !forceLoad) { done(); return; } @@ -243,7 +243,7 @@ export function setFilter (filterType) { path: ['notifications', 'quickFilter', 'active'], value: filterType, }); - dispatch(expandNotifications()); + dispatch(expandNotifications({ forceLoad: true })); dispatch(saveSettings()); }; }; diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js index 4563118eaa3..eb34edb6331 100644 --- a/app/javascript/mastodon/reducers/notifications.js +++ b/app/javascript/mastodon/reducers/notifications.js @@ -41,7 +41,7 @@ const initialState = ImmutableMap({ lastReadId: '0', readMarkerId: '0', isTabVisible: true, - isLoading: false, + isLoading: 0, browserSupport: false, browserPermission: 'default', }); @@ -115,7 +115,7 @@ const expandNormalizedNotifications = (state, notifications, next, isLoadingRece } } - mutable.set('isLoading', false); + mutable.update('isLoading', (nbLoading) => nbLoading - 1); }); }; @@ -214,9 +214,9 @@ export default function notifications(state = initialState, action) { case NOTIFICATIONS_LOAD_PENDING: return state.update('items', list => state.get('pendingItems').concat(list.take(40))).set('pendingItems', ImmutableList()).set('unread', 0); case NOTIFICATIONS_EXPAND_REQUEST: - return state.set('isLoading', true); + return state.update('isLoading', (nbLoading) => nbLoading + 1); case NOTIFICATIONS_EXPAND_FAIL: - return state.set('isLoading', false); + return state.update('isLoading', (nbLoading) => nbLoading - 1); case NOTIFICATIONS_FILTER_SET: return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', true); case NOTIFICATIONS_SCROLL_TOP: From 19f2f35b33e4dacc43df2335ae89ec6af8bb8cb0 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Aug 2022 04:29:20 +0200 Subject: [PATCH 11/22] Fix filtering on context mismatch (#18956) From ba745ca99a4ce4b953e11776827aabb68d621e79 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Aug 2022 04:30:53 +0200 Subject: [PATCH 12/22] Fix media modal link button (#18877) Fixes regression from #18697 --- app/javascript/mastodon/components/icon_button.js | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/app/javascript/mastodon/components/icon_button.js b/app/javascript/mastodon/components/icon_button.js index 81743a1dbe1..47945c475f3 100644 --- a/app/javascript/mastodon/components/icon_button.js +++ b/app/javascript/mastodon/components/icon_button.js @@ -131,17 +131,9 @@ export default class IconButton extends React.PureComponent { ); - if (href) { - return ( - + if (href && !this.prop) { + contents = ( + {contents} ); From 03241d884eb93c59695b4bad98c1725bdc544824 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Aug 2022 04:31:10 +0200 Subject: [PATCH 13/22] Add option for EMAIL_DOMAIN_DENYLIST/EMAIL_DOMAIN_ALLOWLIST to apply after confirmation (#18642) Fixes #18620 --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/user.rb b/app/models/user.rb index 60abaf77e69..9833300cd0a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -94,7 +94,7 @@ class User < ApplicationRecord validates :invite_request, presence: true, on: :create, if: :invite_text_required? validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale? - validates_with BlacklistedEmailValidator, if: -> { !confirmed? } + validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? } validates_with EmailMxValidator, if: :validate_email_dns? validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create From 8479a4b0c1079fc354164842c439f2f87e9eab92 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Aug 2022 04:32:45 +0200 Subject: [PATCH 14/22] Fix tests with regards to upcoming immutable PKeys in OpenSSL 3 (#18464) From af9c9936dd8c87746c9afa128483a93ad21a8b7e Mon Sep 17 00:00:00 2001 From: Arya K <73596856+gi-yt@users.noreply.github.com> Date: Thu, 25 Aug 2022 08:07:09 +0530 Subject: [PATCH 15/22] Fix I2P HTTPS redirect (#18929) --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 514c08cff5b..f41a0f19716 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -47,7 +47,7 @@ Rails.application.configure do config.force_ssl = true config.ssl_options = { redirect: { - exclude: -> request { request.path.start_with?('/health') || request.headers["Host"].end_with?('.onion') } + exclude: -> request { request.path.start_with?('/health') || request.headers["Host"].end_with?('.onion') || request.headers["Host"].end_with?('.i2p') } } } From 66b8abf218a87e65fab8a7fae6fc6ea73c41d750 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 25 Aug 2022 11:37:40 +0900 Subject: [PATCH 16/22] Fix case where boolean was passed to onFilter on StatusActionBar (#18923) --- app/javascript/mastodon/components/status.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index d36311c6772..6fc132bf504 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -516,7 +516,7 @@ class Status extends ImmutablePureComponent { {media} - +
From 5d70a16a1417e53b0c6cc97def9688fda21f337c Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 25 Aug 2022 11:38:01 +0900 Subject: [PATCH 17/22] Fix action type for unfollowHashtag (#18924) --- app/javascript/mastodon/actions/tags.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/javascript/mastodon/actions/tags.js b/app/javascript/mastodon/actions/tags.js index 216e5b54133..37e79d4cbab 100644 --- a/app/javascript/mastodon/actions/tags.js +++ b/app/javascript/mastodon/actions/tags.js @@ -75,18 +75,18 @@ export const unfollowHashtag = name => (dispatch, getState) => { }; export const unfollowHashtagRequest = name => ({ - type: HASHTAG_FETCH_REQUEST, + type: HASHTAG_UNFOLLOW_REQUEST, name, }); export const unfollowHashtagSuccess = (name, tag) => ({ - type: HASHTAG_FETCH_SUCCESS, + type: HASHTAG_UNFOLLOW_SUCCESS, name, tag, }); export const unfollowHashtagFail = (name, error) => ({ - type: HASHTAG_FETCH_FAIL, + type: HASHTAG_UNFOLLOW_FAIL, name, error, }); From 42ff4dce412752a05c2e3b990c93b08cb46ffd88 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Thu, 25 Aug 2022 11:38:34 +0900 Subject: [PATCH 18/22] Use type="color" on badge color input field (#18825) This informs browser to use interactive color picker --- app/views/admin/roles/_form.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/admin/roles/_form.html.haml b/app/views/admin/roles/_form.html.haml index 9beaf619fa5..31f78f2405d 100644 --- a/app/views/admin/roles/_form.html.haml +++ b/app/views/admin/roles/_form.html.haml @@ -13,7 +13,7 @@ = f.input :position, wrapper: :with_label, input_html: { max: current_user.role.position - 1 } .fields-group - = f.input :color, wrapper: :with_label, input_html: { placeholder: '#000000' } + = f.input :color, wrapper: :with_label, input_html: { placeholder: '#000000', type: 'color' } %hr.spacer/ From 63a5514b29d44520058260cfb64c9fbf256e366a Mon Sep 17 00:00:00 2001 From: Alex Nordlund Date: Thu, 25 Aug 2022 04:39:11 +0200 Subject: [PATCH 19/22] Allow S3 to use an existing secret (#18997) --- chart/templates/deployment-web.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/chart/templates/deployment-web.yaml b/chart/templates/deployment-web.yaml index 5e22ca53932..ab722c77b1c 100644 --- a/chart/templates/deployment-web.yaml +++ b/chart/templates/deployment-web.yaml @@ -70,6 +70,18 @@ spec: key: redis-password - name: "PORT" value: {{ .Values.mastodon.web.port | quote }} + {{- if (and .Values.mastodon.s3.enabled .Values.mastodon.s3.existingSecret) }} + - name: "AWS_SECRET_ACCESS_KEY" + valueFrom: + secretKeyRef: + name: {{ .Values.mastodon.s3.existingSecret }} + key: AWS_SECRET_ACCESS_KEY + - name: "AWS_ACCESS_KEY_ID" + valueFrom: + secretKeyRef: + name: {{ .Values.mastodon.s3.existingSecret }} + key: AWS_ACCESS_KEY_ID + {{- end -}} {{- if (not .Values.mastodon.s3.enabled) }} volumeMounts: - name: assets From e682975afd9d476eaf938dfe753debfc20d4e603 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Thu, 25 Aug 2022 11:40:17 +0900 Subject: [PATCH 20/22] Add '--days' option to tootctl media refresh (#18425) * Add '--days' option to tootctl media refresh * Fix undefined scope --- lib/mastodon/media_cli.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 36ca71844f0..4904cc5eb99 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -187,6 +187,7 @@ module Mastodon option :account, type: :string option :domain, type: :string option :status, type: :numeric + option :days, type: :numeric option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false @@ -204,6 +205,8 @@ module Mastodon Use the --domain option to download attachments from a specific domain. + Use the --days option to limit attachments created within days. + By default, attachments that are believed to be already downloaded will not be re-downloaded. To force re-download of every URL, use --force. DESC @@ -224,10 +227,16 @@ module Mastodon scope = MediaAttachment.where(account_id: account.id) elsif options[:domain] scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain])) + elsif options[:days].present? + scope = MediaAttachment.remote else exit(1) end + if options[:days].present? + scope = scope.where('id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) + end + processed, aggregate = parallelize_with_progress(scope) do |media_attachment| next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) next if DomainBlock.reject_media?(media_attachment.account.domain) From 1165943968f8c79cfaccf30c392b14b4930d68e6 Mon Sep 17 00:00:00 2001 From: James Smith <119005+jgsmith@users.noreply.github.com> Date: Wed, 24 Aug 2022 22:40:38 -0400 Subject: [PATCH 21/22] Mark job pods not to use Istio's envoy sidecar (#18415) * Mark job pods not to use Istio's envoy sidecar Istio injects sidecars into pods to implement mTLS between pods. Jobs usually don't know about this, so they don't signal the Envoy process to stop when the job finishes. Since at least one process is running in the pod, Kubernetes doesn't consider the job to be completed, so it lingers. By adding the `sidecar.istio.io/inject` annotation set to `"false"`, we let Istio know that it should not inject the sidecar. If Istio is not installed, then this has no impact. * Support arbitrary job annotations in the Helm chart Rather than focus on Istio, this allows arbitrary annotations for job pods. * Add in-line documentation for pod/job annotations --- chart/templates/cronjob-media-remove.yaml | 4 ++++ chart/templates/job-assets-precompile.yaml | 4 ++++ chart/templates/job-chewy-upgrade.yaml | 4 ++++ chart/templates/job-create-admin.yaml | 4 ++++ chart/templates/job-db-migrate.yaml | 4 ++++ chart/values.yaml | 6 ++++++ 6 files changed, 26 insertions(+) diff --git a/chart/templates/cronjob-media-remove.yaml b/chart/templates/cronjob-media-remove.yaml index 726e100cf8d..160aee20421 100644 --- a/chart/templates/cronjob-media-remove.yaml +++ b/chart/templates/cronjob-media-remove.yaml @@ -12,6 +12,10 @@ spec: template: metadata: name: {{ include "mastodon.fullname" . }}-media-remove + {{- with .Values.jobAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} spec: restartPolicy: OnFailure {{- if (not .Values.mastodon.s3.enabled) }} diff --git a/chart/templates/job-assets-precompile.yaml b/chart/templates/job-assets-precompile.yaml index 4aa8d1407c4..faa51a20d9d 100644 --- a/chart/templates/job-assets-precompile.yaml +++ b/chart/templates/job-assets-precompile.yaml @@ -12,6 +12,10 @@ spec: template: metadata: name: {{ include "mastodon.fullname" . }}-assets-precompile + {{- with .Values.jobAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: Never {{- if (not .Values.mastodon.s3.enabled) }} diff --git a/chart/templates/job-chewy-upgrade.yaml b/chart/templates/job-chewy-upgrade.yaml index 16b4f75a7db..ae6fb38e129 100644 --- a/chart/templates/job-chewy-upgrade.yaml +++ b/chart/templates/job-chewy-upgrade.yaml @@ -13,6 +13,10 @@ spec: template: metadata: name: {{ include "mastodon.fullname" . }}-chewy-upgrade + {{- with .Values.jobAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: Never {{- if (not .Values.mastodon.s3.enabled) }} diff --git a/chart/templates/job-create-admin.yaml b/chart/templates/job-create-admin.yaml index 486c0c35726..659c00671fc 100644 --- a/chart/templates/job-create-admin.yaml +++ b/chart/templates/job-create-admin.yaml @@ -13,6 +13,10 @@ spec: template: metadata: name: {{ include "mastodon.fullname" . }}-create-admin + {{- with .Values.jobAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: Never {{- if (not .Values.mastodon.s3.enabled) }} diff --git a/chart/templates/job-db-migrate.yaml b/chart/templates/job-db-migrate.yaml index 41ece64a2a1..8e4f70dfb1d 100644 --- a/chart/templates/job-db-migrate.yaml +++ b/chart/templates/job-db-migrate.yaml @@ -12,6 +12,10 @@ spec: template: metadata: name: {{ include "mastodon.fullname" . }}-db-migrate + {{- with .Values.jobAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: Never {{- if (not .Values.mastodon.s3.enabled) }} diff --git a/chart/values.yaml b/chart/values.yaml index bd723567f0f..4b18a9dfa5e 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -281,8 +281,14 @@ serviceAccount: # If not set and create is true, a name is generated using the fullname template name: "" +# Kubernetes manages pods for jobs and pods for deployments differently, so you might +# need to apply different annotations to the two different sets of pods. The annotations +# set with podAnnotations will be added to all deployment-managed pods. podAnnotations: {} +# The annotations set with jobAnnotations will be added to all job pods. +jobAnnotations: {} + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little From 861b35dd54d266bc0a40b3cacb28e5b82ff6faaa Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Thu, 25 Aug 2022 11:41:14 +0900 Subject: [PATCH 22/22] Support "http_hidden_proxy" ENV var for hidden service only proxy (#18427) * Support "http_hidden_proxy" ENV var for hidden service only proxy * Fallback to http_proxy if http_hidden_proxy is not set --- app/lib/request.rb | 18 +++++++++++++++--- config/initializers/http_client_proxy.rb | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/app/lib/request.rb b/app/lib/request.rb index 4289da9333d..f5123d776ab 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -31,7 +31,7 @@ class Request @url = Addressable::URI.parse(url).normalize @http_client = options.delete(:http_client) @options = options.merge(socket_class: use_proxy? ? ProxySocket : Socket) - @options = @options.merge(Rails.configuration.x.http_client_proxy) if use_proxy? + @options = @options.merge(proxy_url) if use_proxy? @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? @@ -141,11 +141,23 @@ class Request end def use_proxy? - Rails.configuration.x.http_client_proxy.present? + proxy_url.present? + end + + def proxy_url + if hidden_service? && Rails.configuration.x.http_client_hidden_proxy.present? + Rails.configuration.x.http_client_hidden_proxy + else + Rails.configuration.x.http_client_proxy + end end def block_hidden_service? - !Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match?(@url.host) + !Rails.configuration.x.access_to_hidden_service && hidden_service? + end + + def hidden_service? + /\.(onion|i2p)$/.match?(@url.host) end module ClientLimit diff --git a/config/initializers/http_client_proxy.rb b/config/initializers/http_client_proxy.rb index 7a9b7b86d7d..b29e9edd750 100644 --- a/config/initializers/http_client_proxy.rb +++ b/config/initializers/http_client_proxy.rb @@ -18,5 +18,22 @@ Rails.application.configure do }.compact end + if ENV['http_hidden_proxy'].present? + proxy = URI.parse(ENV['http_hidden_proxy']) + + raise "Unsupported proxy type: #{proxy.scheme}" unless %w(http https).include? proxy.scheme + raise "No proxy host" unless proxy.host + + host = proxy.host + host = host[1...-1] if host[0] == '[' # for IPv6 address + + config.x.http_client_hidden_proxy[:proxy] = { + proxy_address: host, + proxy_port: proxy.port, + proxy_username: proxy.user, + proxy_password: proxy.password, + }.compact + end + config.x.access_to_hidden_service = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true' end