From 34626b8291db22fafaa7963496bc304f2725ae58 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 25 Jul 2024 06:37:48 -0400 Subject: [PATCH 1/7] Update schema to reflect full `db:migrate` run results (#31128) --- db/schema.rb | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index 66da12f713..453f1f7b89 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -194,8 +194,8 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_13_171909) do t.integer "avatar_storage_schema_version" t.integer "header_storage_schema_version" t.string "devices_url" - t.integer "suspension_origin" t.datetime "sensitized_at", precision: nil + t.integer "suspension_origin" t.boolean "trendable" t.datetime "reviewed_at", precision: nil t.datetime "requested_review_at", precision: nil @@ -579,12 +579,12 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_13_171909) do end create_table "ip_blocks", force: :cascade do |t| - t.datetime "created_at", precision: nil, null: false - t.datetime "updated_at", precision: nil, null: false - t.datetime "expires_at", precision: nil t.inet "ip", default: "0.0.0.0", null: false t.integer "severity", default: 0, null: false + t.datetime "expires_at", precision: nil t.text "comment", default: "", null: false + t.datetime "created_at", precision: nil, null: false + t.datetime "updated_at", precision: nil, null: false t.index ["ip"], name: "index_ip_blocks_on_ip", unique: true end @@ -1421,9 +1421,9 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_13_171909) do add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true create_view "user_ips", sql_definition: <<-SQL - SELECT t0.user_id, - t0.ip, - max(t0.used_at) AS used_at + SELECT user_id, + ip, + max(used_at) AS used_at FROM ( SELECT users.id AS user_id, users.sign_up_ip AS ip, users.created_at AS used_at @@ -1440,7 +1440,7 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_13_171909) do login_activities.created_at FROM login_activities WHERE (login_activities.success = true)) t0 - GROUP BY t0.user_id, t0.ip; + GROUP BY user_id, ip; SQL create_view "account_summaries", materialized: true, sql_definition: <<-SQL SELECT accounts.id AS account_id, @@ -1461,9 +1461,9 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_13_171909) do add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true create_view "global_follow_recommendations", materialized: true, sql_definition: <<-SQL - SELECT t0.account_id, - sum(t0.rank) AS rank, - array_agg(t0.reason) AS reason + SELECT account_id, + sum(rank) AS rank, + array_agg(reason) AS reason FROM ( SELECT account_summaries.account_id, ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank, 'most_followed'::text AS reason @@ -1487,8 +1487,8 @@ ActiveRecord::Schema[7.1].define(version: 2024_07_13_171909) do WHERE (follow_recommendation_suppressions.account_id = statuses.account_id))))) GROUP BY account_summaries.account_id HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0 - GROUP BY t0.account_id - ORDER BY (sum(t0.rank)) DESC; + GROUP BY account_id + ORDER BY (sum(rank)) DESC; SQL add_index "global_follow_recommendations", ["account_id"], name: "index_global_follow_recommendations_on_account_id", unique: true From e881a5967149e34046f91d3f4445ebb457348b93 Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 25 Jul 2024 10:18:24 -0400 Subject: [PATCH 2/7] Add `User.unconfirmed` scope, reduce factories in `scheduler/user_cleanup` spec (#31063) --- app/models/user.rb | 1 + .../scheduler/user_cleanup_scheduler.rb | 2 +- .../scheduler/user_cleanup_scheduler_spec.rb | 30 ++++++++++--------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 8bc0b23ce8..7285456926 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -117,6 +117,7 @@ class User < ApplicationRecord scope :pending, -> { where(approved: false) } scope :approved, -> { where(approved: true) } scope :confirmed, -> { where.not(confirmed_at: nil) } + scope :unconfirmed, -> { where(confirmed_at: nil) } scope :enabled, -> { where(disabled: false) } scope :disabled, -> { where(disabled: true) } scope :active, -> { confirmed.signed_in_recently.account_not_suspended } diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb index 74abc23701..9f58d9225b 100644 --- a/app/workers/scheduler/user_cleanup_scheduler.rb +++ b/app/workers/scheduler/user_cleanup_scheduler.rb @@ -16,7 +16,7 @@ class Scheduler::UserCleanupScheduler private def clean_unconfirmed_accounts! - User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', UNCONFIRMED_ACCOUNTS_MAX_AGE_DAYS.days.ago).reorder(nil).find_in_batches do |batch| + User.unconfirmed.where(confirmation_sent_at: ..UNCONFIRMED_ACCOUNTS_MAX_AGE_DAYS.days.ago).reorder(nil).find_in_batches do |batch| # We have to do it separately because of missing database constraints AccountModerationNote.where(target_account_id: batch.map(&:account_id)).delete_all Account.where(id: batch.map(&:account_id)).delete_all diff --git a/spec/workers/scheduler/user_cleanup_scheduler_spec.rb b/spec/workers/scheduler/user_cleanup_scheduler_spec.rb index c3940901d4..7952f2c146 100644 --- a/spec/workers/scheduler/user_cleanup_scheduler_spec.rb +++ b/spec/workers/scheduler/user_cleanup_scheduler_spec.rb @@ -12,29 +12,31 @@ describe Scheduler::UserCleanupScheduler do describe '#perform' do before do - # Need to update the already-existing users because their initialization overrides confirmation_sent_at + # Update already-existing users because initialization overrides `confirmation_sent_at` new_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: Time.now.utc) old_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: 10.days.ago) confirmed_user.update!(confirmed_at: 1.day.ago) end - it 'deletes the old unconfirmed user, their account, and the moderation note' do + it 'deletes the old unconfirmed user and metadata while preserving confirmed user and newer unconfirmed user' do expect { subject.perform } - .to change { User.exists?(old_unconfirmed_user.id) }.from(true).to(false) - .and change { Account.exists?(old_unconfirmed_user.account_id) }.from(true).to(false) - expect { moderation_note.reload }.to raise_error(ActiveRecord::RecordNotFound) + .to change { User.exists?(old_unconfirmed_user.id) } + .from(true).to(false) + .and change { Account.exists?(old_unconfirmed_user.account_id) } + .from(true).to(false) + expect { moderation_note.reload } + .to raise_error(ActiveRecord::RecordNotFound) + expect_preservation_of(new_unconfirmed_user) + expect_preservation_of(confirmed_user) end - it 'does not delete the new unconfirmed user or their account' do - subject.perform - expect(User.exists?(new_unconfirmed_user.id)).to be true - expect(Account.exists?(new_unconfirmed_user.account_id)).to be true - end + private - it 'does not delete the confirmed user or their account' do - subject.perform - expect(User.exists?(confirmed_user.id)).to be true - expect(Account.exists?(confirmed_user.account_id)).to be true + def expect_preservation_of(user) + expect(User.exists?(user.id)) + .to be true + expect(Account.exists?(user.account_id)) + .to be true end end end From 337a7fcf30b7253830633dfe962d4411b6b2a370 Mon Sep 17 00:00:00 2001 From: Adam Niedzielski Date: Thu, 25 Jul 2024 16:24:19 +0200 Subject: [PATCH 3/7] =?UTF-8?q?Fix=20=C3=9F=20bug=20in=20regexp=20for=20me?= =?UTF-8?q?ntions=20and=20tags=20(#31122)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/account.rb | 2 +- app/models/tag.rb | 2 +- spec/models/account_spec.rb | 8 ++++++++ spec/models/tag_spec.rb | 8 ++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/app/models/account.rb b/app/models/account.rb index 8a990bb831..23ff07c769 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -68,7 +68,7 @@ class Account < ApplicationRecord INSTANCE_ACTOR_ID = -99 USERNAME_RE = /[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?/i - MENTION_RE = %r{(? Date: Thu, 25 Jul 2024 10:28:08 -0400 Subject: [PATCH 4/7] Improve failure message on pagination matcher (#31146) --- spec/support/matchers/api_pagination.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/support/matchers/api_pagination.rb b/spec/support/matchers/api_pagination.rb index 81e27e44b8..f7d552b242 100644 --- a/spec/support/matchers/api_pagination.rb +++ b/spec/support/matchers/api_pagination.rb @@ -7,7 +7,7 @@ RSpec::Matchers.define :include_pagination_headers do |links| end.all? end - failure_message do |header| - "expected that #{header} would have the same values as #{links}." + failure_message do |response| + "expected that #{response.headers['Link']} would have the same values as #{links}." end end From fbf95216c55adac8af18bc709b4e3f456641013a Mon Sep 17 00:00:00 2001 From: Adam Niedzielski Date: Thu, 25 Jul 2024 16:51:44 +0200 Subject: [PATCH 5/7] Fix LinkCrawlWorker NoMethodError: undefined method `[]' for nil (#31144) --- app/lib/link_details_extractor.rb | 2 +- spec/lib/link_details_extractor_spec.rb | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index d81f4a3062..bd78aef7a9 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -101,7 +101,7 @@ class LinkDetailsExtractor end def json - @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {} + @json ||= root_array(Oj.load(@data)).compact.find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {} end end diff --git a/spec/lib/link_details_extractor_spec.rb b/spec/lib/link_details_extractor_spec.rb index 2a4df70a8b..b1e5cedced 100644 --- a/spec/lib/link_details_extractor_spec.rb +++ b/spec/lib/link_details_extractor_spec.rb @@ -129,6 +129,24 @@ RSpec.describe LinkDetailsExtractor do include_examples 'structured data' end + context 'with the first tag is null' do + let(:html) { <<~HTML } + + + + + + + + HTML + + include_examples 'structured data' + end + context 'with preceding block of unsupported LD+JSON' do let(:html) { <<~HTML } From 5f5373397fd6d1bfde907c078174de6ca14d925e Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 25 Jul 2024 10:59:56 -0400 Subject: [PATCH 6/7] Remove duplicate `omniauth_only?` helper method (#31066) --- app/controllers/application_controller.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 66e0f7e305..62e3355ae6 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -20,7 +20,6 @@ class ApplicationController < ActionController::Base helper_method :current_theme helper_method :single_user_mode? helper_method :use_seamless_external_login? - helper_method :omniauth_only? helper_method :sso_account_settings helper_method :limited_federation_mode? helper_method :body_class_string @@ -137,10 +136,6 @@ class ApplicationController < ActionController::Base Devise.pam_authentication || Devise.ldap_authentication end - def omniauth_only? - ENV['OMNIAUTH_ONLY'] == 'true' - end - def sso_account_settings ENV.fetch('SSO_ACCOUNT_SETTINGS', nil) end From 887e64efd4abbf3980e008c7a5441b44fbd6c766 Mon Sep 17 00:00:00 2001 From: Adam Niedzielski Date: Thu, 25 Jul 2024 17:02:58 +0200 Subject: [PATCH 7/7] Allow @ at the end of an URL (#31124) --- config/initializers/twitter_regex.rb | 2 +- spec/lib/text_formatter_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb index 9ca4eb0da9..b932e97e2f 100644 --- a/config/initializers/twitter_regex.rb +++ b/config/initializers/twitter_regex.rb @@ -9,7 +9,7 @@ module Twitter::TwitterText class Regex REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}<>()?]/iou - REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}()?!*"'「」<>;:=,.$%\[\]~&|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou + REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}()?!*"'「」<>;:=,.$%\[\]~&|]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou REGEXEN[:valid_url_balanced_parens] = / \( (?: diff --git a/spec/lib/text_formatter_spec.rb b/spec/lib/text_formatter_spec.rb index 8b922c018b..bde17bb79c 100644 --- a/spec/lib/text_formatter_spec.rb +++ b/spec/lib/text_formatter_spec.rb @@ -224,6 +224,14 @@ RSpec.describe TextFormatter do end end + context 'when given a URL with trailing @ symbol' do + let(:text) { 'https://gta.fandom.com/wiki/TW@ Content' } + + it 'matches the full URL' do + expect(subject).to include 'href="https://gta.fandom.com/wiki/TW@"' + end + end + context 'when given a URL containing unsafe code (XSS attack, visible part)' do let(:text) { 'http://example.com/bb' }