From fab85ca7fb993c5af4ac88d1211c785c5a46504e Mon Sep 17 00:00:00 2001 From: kouhai Date: Mon, 15 Apr 2024 00:33:47 -0700 Subject: [PATCH] th: prototype automod --- app/lib/activitypub/activity/create.rb | 1 + .../activitypub/process_account_service.rb | 5 + app/services/report_service.rb | 2 + config/application.rb | 14 ++ lib/treehouse/automod.rb | 151 ++++++++++++++++++ spec/fabricators/user_fabricator.rb | 4 + spec/fabricators/user_role_fabricator.rb | 8 + spec/lib/activitypub/activity/create_spec.rb | 84 ++++++++++ .../process_account_service_spec.rb | 110 +++++++++++++ 9 files changed, 379 insertions(+) create mode 100644 lib/treehouse/automod.rb diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index f091f0d201..7d06cbd73d 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -86,6 +86,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity @status = Status.create!(@params) attach_tags(@status) end + return if Treehouse::Automod.process_status!(@status) resolve_thread(@status) fetch_replies(@status) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index b667e97f4d..35323f2af1 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'digest' + class ActivityPub::ProcessAccountService < BaseService include JsonLdHelper include DomainControlHelper @@ -90,6 +92,9 @@ class ActivityPub::ProcessAccountService < BaseService set_immediate_protocol_attributes! set_fetchable_key! unless @account.suspended? && @account.suspension_origin_local? set_immediate_attributes! unless @account.suspended? + + Treehouse::Automod.process_account!(@account) + set_fetchable_attributes! unless @options[:only_key] || @account.suspended? @account.save_with_optional_media! diff --git a/app/services/report_service.rb b/app/services/report_service.rb index dea6df7b0a..ae22b0d7ca 100644 --- a/app/services/report_service.rb +++ b/app/services/report_service.rb @@ -40,6 +40,7 @@ class ReportService < BaseService end def notify_staff! + return if @options[:th_skip_notify_staff] return if @report.unresolved_siblings? User.those_who_can(:manage_reports).includes(:account).find_each do |u| @@ -65,6 +66,7 @@ class ReportService < BaseService end def forward? + return false if @options[:th_skip_forward] !@target_account.local? && ActiveModel::Type::Boolean.new.cast(@options[:forward]) end diff --git a/config/application.rb b/config/application.rb index 5aca74fd1c..95412f4107 100644 --- a/config/application.rb +++ b/config/application.rb @@ -56,6 +56,10 @@ require_relative '../lib/active_record/with_recursive' require_relative '../lib/arel/union_parenthesizing' require_relative '../lib/simple_navigation/item_extensions' +require_relative '../lib/treehouse/automod' + +Dotenv::Railtie.load + Bundler.require(:pam_authentication) if ENV['PAM_ENABLED'] == 'true' module Mastodon @@ -121,5 +125,15 @@ module Mastodon Devise::FailureApp.include AbstractController::Callbacks Devise::FailureApp.include Localized end + + config.x.th_automod.automod_account_username = ENV['TH_STAFF_ACCOUNT'] + config.x.th_automod.account_service_heuristic_auto_suspend_active = ENV.fetch('TH_ACCOUNT_SERVICE_HEURISTIC_AUTO_SUSPEND', '') == 'that-one-spammer' + config.x.th_automod.mention_spam_heuristic_auto_limit_active = ENV.fetch('TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE', '') == 'can-spam' + config.x.th_automod.mention_spam_threshold = + begin + value = ENV.fetch('TH_MENTION_SPAM_THRESHOLD', '0').to_i + value == 0 ? Float::INFINITY : value + end + config.x.th_automod.min_account_age_threshold = ENV.fetch('TH_MIN_ACCOUNT_AGE_THRESHOLD', '2').to_i.days end end diff --git a/lib/treehouse/automod.rb b/lib/treehouse/automod.rb new file mode 100644 index 0000000000..f2cbb132ba --- /dev/null +++ b/lib/treehouse/automod.rb @@ -0,0 +1,151 @@ +module Treehouse + module Automod + COMMENT_HEADER = <<~EOS + Tracking Report - automatically created by TreehouseAutomod + EOS + + WARNING_TEXT = <<~EOS + Tracking Infraction - automatically created by TreehouseAutomod + EOS + + def self.silence_with_tracking_report!(account, status_ids: [], explanation: "") + account.save! + + self.file_tracking_report!(account, status_ids: status_ids, type: 'silence') unless account.suspension_origin == "local" + end + + def self.suspend_with_tracking_report!(account, status_ids: [], explanation: "") + account.save! + + self.file_tracking_report!(account, status_ids: status_ids, type: 'suspend') unless account.suspension_origin == "local" + end + + def self.file_tracking_report!(target_account, status_ids: [], explanation: "", type: 'suspend') + reporter = self.staff_account + return if reporter.nil? + + # status_ids is broken because of validation + report = ReportService.new.call( + reporter, + target_account, + { + status_ids: status_ids, + comment: explanation.blank? ? COMMENT_HEADER : "#{COMMENT_HEADER}\n\n#{EXPLANATION}", + th_skip_notify_staff: true, + th_skip_forward: true, + } + ) + report.save! + report.assign_to_self!(reporter) + + account_action = Admin::AccountAction.new( + type: type, + report_id: report.id, + target_account: target_account, + current_account: reporter, + send_email_notification: false, + text: WARNING_TEXT, + ) + account_action.save! + + Admin::ActionLog.create( + account: reporter, + action: account_action, + target: target_account, + ) + + report.resolve!(reporter) + end + + def self.staff_account + username = Rails.configuration.x.th_automod.automod_account_username + Account.find_local(username) unless username.blank? + end + + def self.process_status!(status) + ActivityPubActivityCreateExt.process!(status) + end + + def self.process_account!(account) + AccountServiceExt.process!(account) + end + + module ActivityPubActivityCreateExt + EXPLANATION = <<~EOS + This account was automatically suspended by TreehouseAutomod, an unsupported feature. + + Currently, the account-only heuristic should only automatically suspend accounts with one specific username and display name. + + If this action is unexpected, please unset TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE. + EOS + + def self.is_spam?(status) + return false unless Rails.configuration.x.th_automod.mention_spam_heuristic_auto_limit_active + account = status.account + minimal_effort = account.note.blank? && account.avatar_remote_url.blank? && account.header_remote_url.blank? + return false if (account.local? || + account.local_followers_count > 0 || + !minimal_effort) + + # minimal effort account, check mentions and account-known age + has_mention_spam = status.mentions.size >= Rails.configuration.x.th_automod.mention_spam_threshold + is_new_account = account.created_at > (Time.now - Rails.configuration.x.th_automod.min_account_age_threshold) + + has_mention_spam && is_new_account + end + + # check if the status should be considered spam + # @return true if the status was reported and the account was infracted + def self.process!(status) + return false unless self.is_spam?(status) + return true if status.account.silenced? + + Automod.silence_with_tracking_report!(status.account, explanation: EXPLANATION) + + true + end + end + + module AccountServiceExt + # hardcoded for now + # md5 because they don't deserve more mentions + HEURISTIC_NAMES = { + "0116a9deace3289b7092e945ef5ca0a5" => Set["57d3d0b932cc9cd01be6b2f4e82c1a4a"], + } + # probably mathematically impossible to collide, but just in case... + HEURISTIC_MAX_LEN = 16 + + EXPLANATION = <<~EOS + This account was automatically suspended by TreehouseAutomod, an unsupported feature. + + Currently, the account-only heuristic should only automatically suspend accounts with one specific username and display name. + + If this action is unexpected, please unset TH_HEURISTIC_AUTO_SUSPEND. + EOS + + # @return true if the account was infracted + def self.process!(account) + return false unless heuristic_auto_suspend?(account) + + Automod.suspend_with_tracking_report!(account, explanation: EXPLANATION) unless account.suspension_origin == "local" + + true + end + + def self.matches_evil_hash?(account) + username_md5 = Digest::MD5.hexdigest(account.username) + display_name_md5 = Digest::MD5.hexdigest(account.display_name) + + HEURISTIC_NAMES[username_md5].include?(display_name_md5) + end + + def self.heuristic_auto_suspend?(account) + return false unless Rails.configuration.x.th_automod.account_service_heuristic_auto_suspend_active + + return unless account.username.length < HEURISTIC_MAX_LEN && account.display_name.length < HEURISTIC_MAX_LEN + + self.matches_evil_hash?(account) + end + end + end +end diff --git a/spec/fabricators/user_fabricator.rb b/spec/fabricators/user_fabricator.rb index 0df7caea60..538bb377de 100644 --- a/spec/fabricators/user_fabricator.rb +++ b/spec/fabricators/user_fabricator.rb @@ -13,3 +13,7 @@ Fabricator(:user) do current_sign_in_at { Time.zone.now } agreement true end + +Fabricator(:moderator_user, :from => :user) do + role { Fabricate(:moderator_role) } +end diff --git a/spec/fabricators/user_role_fabricator.rb b/spec/fabricators/user_role_fabricator.rb index d443227605..cd220de45c 100644 --- a/spec/fabricators/user_role_fabricator.rb +++ b/spec/fabricators/user_role_fabricator.rb @@ -5,3 +5,11 @@ Fabricator(:user_role) do color '' permissions 0 end + +Fabricator(:moderator_role, :from => :user_role) do + name 'fake moderator' + permissions UserRole::Flags::DEFAULT | + UserRole::Flags::CATEGORIES[:moderation] + .map { |p| UserRole::FLAGS[p] } + .reduce(&:|) +end diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index fbaf672b5a..841ad1f958 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -1158,5 +1158,89 @@ RSpec.describe ActivityPub::Activity::Create do expect(sender.statuses.count).to eq 0 end end + + context 'with automod active' do + subject { described_class.new(json, sender, delivery: true) } + + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor', created_at: created_at) } + let(:created_at) { Time.now } + let(:min_account_age_threshold) { 1.day } + + let(:recipient_a) { Fabricate(:account) } + let(:recipient_b) { Fabricate(:account) } + let(:staff_user) { Fabricate(:moderator_user) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + cc: ActivityPub::TagManager.instance.uri_for(recipient_a), + tag: recipients.map do |recipient| + { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(recipient), + } + end, + } + end + + before do + allow(Rails.configuration.x.th_automod).to receive(:automod_account_username).and_return(staff_user.account.username) + allow(Rails.configuration.x.th_automod).to receive(:mention_spam_heuristic_auto_limit_active).and_return(true) + allow(Rails.configuration.x.th_automod).to receive(:mention_spam_threshold).and_return(2) + allow(Rails.configuration.x.th_automod).to receive(:min_account_age_threshold).and_return(min_account_age_threshold) + allow(subject).to receive(:distribute) + allow(sender).to receive(:silence!).and_call_original + subject.perform + end + + shared_examples 'automod activates' do + it 'silences the sender' do + expect(sender).to have_received(:silence!) + expect(sender.silenced?).to be_truthy + end + + it 'skips distribution' do + expect(subject).not_to have_received(:distribute) + end + + it 'files a tracking report' do + expect(sender.previous_strikes_count).to be_truthy + end + end + + shared_examples 'automod does not activate' do + it 'does not silence the sender' do + expect(sender.silenced?).to be_falsy + end + + it 'does not file a tracking report' do + expect(sender.reports.empty?).to be_truthy + end + end + + context 'and spammy message' do + let(:recipients) { [recipient_a, recipient_b] } + + context 'and old account' do + let(:created_at) { Time.now - min_account_age_threshold - 1.hour } + + include_examples 'automod does not activate' + end + + context 'and new account' do + let(:created_at) { Time.now - min_account_age_threshold + 1.hour } + + include_examples 'automod activates' + end + end + + context 'and hammy message' do + let(:recipients) { [recipient_a] } + + include_examples 'automod does not activate' + end + end end end diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 4fbb527b39..53c8fd1916 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -231,6 +231,115 @@ RSpec.describe ActivityPub::ProcessAccountService do end end + context 'treehouse automod' do + subject { described_class.new.call(account_username, 'foo.test', payload) } + let(:account_username) { 'evil' } + let(:account_display_name) { 'evil display name' } + let(:account_payload_suspended) { false } + + let(:staff_user) { Fabricate(:moderator_user) } + let(:automod_account_username) { staff_user.account.username } + + let(:payload) do + { + id: 'https://foo.test', + type: 'Actor', + inbox: 'https://foo.test/inbox', + suspended: account_payload_suspended, + name: account_display_name, + }.with_indifferent_access + end + + let(:name_hash_hash) do + { + # 'evil' => 'evil display name' + '4034a346ccee15292d823416f7510a2f' => Set['225e44a7c4a792ee22a4ada2032da7cd'] + } + end + + before do + allow(Rails.configuration.x.th_automod).to receive(:account_service_heuristic_auto_suspend_active).and_return(true) + allow(Rails.configuration.x.th_automod).to receive(:automod_account_username).and_return(automod_account_username) + + stub_const('::Treehouse::Automod::AccountServiceExt::HEURISTIC_NAMES', name_hash_hash) + stub_const('::Treehouse::Automod::AccountServiceExt::HEURISTIC_MAX_LEN', 20) + end + + context 'new account' do + context 'heuristic matching' do + it 'suspends the user locally' do + expect(subject.suspended?).to be_truthy + expect(subject.suspension_origin_local?).to be_truthy + end + end + + context 'heuristic not matching' do + let(:account_display_name) { '' } + it 'does nothing' do + expect(subject.suspended?).to be_falsy + end + end + end + + context 'existing account' do + let!(:account) { Fabricate(:account, username: account_username, domain: 'foo.test', display_name: account_display_name) } + + before do + allow(Admin::SuspensionWorker).to receive(:perform_async) + end + + context 'heuristic matching' do + it 'suspends the user locally' do + expect(subject.suspended?).to be_truthy + expect(subject.suspension_origin_local?).to be_truthy + end + end + + context 'heuristic not matching' do + let(:account_display_name) { 'not evil display name' } + + it 'does nothing' do + expect(subject.suspended?).to be_falsy + end + + context 'suspended locally' do + before do + account.suspend!(origin: :local) + end + + it 'does nothing' do + expect(subject.suspended?).to be_truthy + end + end + end + end + + context 'tracking report' do + let(:automod_account_username) { 'automod_test' } + + let!(:automod_user_role) { Fabricate(:user_role, name: 'Automod', permissions: UserRole::FLAGS[:administrator]) } + + let!(:automod_account) do + account = Fabricate(:account, username: automod_account_username) + account.user.role_id = automod_user_role.id + account.user.save! + account + end + + it 'creates report' do + expect(subject.targeted_reports.empty?).to be_falsy + + report = Report.find_by(target_account_id: subject.id, account_id: automod_account.id, assigned_account_id: automod_account.id) + expect(report.comment.starts_with?('Tracking Report - automatically created by TreehouseAutomod')).to be_truthy + end + + it 'creates account action' do + subject + expect(Admin::ActionLog.find_by(account_id: automod_account.id, target_id: subject.id)).not_to be nil + end + end + end + private def create_some_remote_accounts @@ -240,4 +349,5 @@ RSpec.describe ActivityPub::ProcessAccountService do def create_fewer_than_rate_limit_accounts change(Account.remote, :count).by_at_most(5) end + end