th: prototype automod

app/lib/activitypub/activity/create.rb

@@ -86,6 +86,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       @status = Status.create!(@params)
       attach_tags(@status)
     end
+    return if Treehouse::Automod.process_status!(@status)
 
     resolve_thread(@status)
     fetch_replies(@status)

app/services/activitypub/process_account_service.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 class ActivityPub::ProcessAccountService < BaseService
   include JsonLdHelper
   include DomainControlHelper
@@ -90,6 +92,9 @@ class ActivityPub::ProcessAccountService < BaseService
     set_immediate_protocol_attributes!
     set_fetchable_key! unless @account.suspended? && @account.suspension_origin_local?
     set_immediate_attributes! unless @account.suspended?
+
+    Treehouse::Automod.process_account!(@account)
+
     set_fetchable_attributes! unless @options[:only_key] || @account.suspended?
 
     @account.save_with_optional_media!

app/services/report_service.rb

@@ -40,6 +40,7 @@ class ReportService < BaseService
   end
 
   def notify_staff!
+    return if @options[:th_skip_notify_staff]
     return if @report.unresolved_siblings?
 
     User.those_who_can(:manage_reports).includes(:account).find_each do |u|
@@ -65,6 +66,7 @@ class ReportService < BaseService
   end
 
   def forward?
+    return false if @options[:th_skip_forward]
     !@target_account.local? && ActiveModel::Type::Boolean.new.cast(@options[:forward])
   end
 

config/application.rb

@@ -56,6 +56,10 @@ require_relative '../lib/active_record/with_recursive'
 require_relative '../lib/arel/union_parenthesizing'
 require_relative '../lib/simple_navigation/item_extensions'
 
+require_relative '../lib/treehouse/automod'
+
+Dotenv::Railtie.load
+
 Bundler.require(:pam_authentication) if ENV['PAM_ENABLED'] == 'true'
 
 module Mastodon
@@ -121,5 +125,15 @@ module Mastodon
       Devise::FailureApp.include AbstractController::Callbacks
       Devise::FailureApp.include Localized
     end
+
+    config.x.th_automod.automod_account_username = ENV['TH_STAFF_ACCOUNT']
+    config.x.th_automod.account_service_heuristic_auto_suspend_active = ENV.fetch('TH_ACCOUNT_SERVICE_HEURISTIC_AUTO_SUSPEND', '') == 'that-one-spammer'
+    config.x.th_automod.mention_spam_heuristic_auto_limit_active = ENV.fetch('TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE', '') == 'can-spam'
+    config.x.th_automod.mention_spam_threshold =
+      begin
+        value = ENV.fetch('TH_MENTION_SPAM_THRESHOLD', '0').to_i
+        value == 0 ? Float::INFINITY : value
+      end
+    config.x.th_automod.min_account_age_threshold = ENV.fetch('TH_MIN_ACCOUNT_AGE_THRESHOLD', '2').to_i.days
   end
 end

lib/treehouse/automod.rb

@@ -0,0 +1,151 @@
+module Treehouse
+  module Automod
+    COMMENT_HEADER = <<~EOS
+      Tracking Report - automatically created by TreehouseAutomod
+    EOS
+
+    WARNING_TEXT = <<~EOS
+      Tracking Infraction - automatically created by TreehouseAutomod
+    EOS
+
+    def self.silence_with_tracking_report!(account, status_ids: [], explanation: "")
+      account.save!
+
+      self.file_tracking_report!(account, status_ids: status_ids, type: 'silence') unless account.suspension_origin == "local"
+    end
+
+    def self.suspend_with_tracking_report!(account, status_ids: [], explanation: "")
+      account.save!
+
+      self.file_tracking_report!(account, status_ids: status_ids, type: 'suspend') unless account.suspension_origin == "local"
+    end
+
+    def self.file_tracking_report!(target_account, status_ids: [], explanation: "", type: 'suspend')
+      reporter = self.staff_account
+      return if reporter.nil?
+
+      # status_ids is broken because of validation
+      report = ReportService.new.call(
+        reporter,
+        target_account,
+        {
+          status_ids: status_ids,
+          comment: explanation.blank? ? COMMENT_HEADER : "#{COMMENT_HEADER}\n\n#{EXPLANATION}",
+          th_skip_notify_staff: true,
+          th_skip_forward: true,
+        }
+      )
+      report.save!
+      report.assign_to_self!(reporter)
+
+      account_action = Admin::AccountAction.new(
+        type: type,
+        report_id: report.id,
+        target_account: target_account,
+        current_account: reporter,
+        send_email_notification: false,
+        text: WARNING_TEXT,
+      )
+      account_action.save!
+
+      Admin::ActionLog.create(
+        account: reporter,
+        action: account_action,
+        target: target_account,
+      )
+
+      report.resolve!(reporter)
+    end
+
+    def self.staff_account
+      username = Rails.configuration.x.th_automod.automod_account_username
+      Account.find_local(username) unless username.blank?
+    end
+
+    def self.process_status!(status)
+      ActivityPubActivityCreateExt.process!(status)
+    end
+
+    def self.process_account!(account)
+      AccountServiceExt.process!(account)
+    end
+
+    module ActivityPubActivityCreateExt
+      EXPLANATION = <<~EOS
+        This account was automatically suspended by TreehouseAutomod, an unsupported feature.
+
+        Currently, the account-only heuristic should only automatically suspend accounts with one specific username and display name.
+
+        If this action is unexpected, please unset TH_MENTION_SPAM_HEURISTIC_AUTO_LIMIT_ACTIVE.
+      EOS
+
+      def self.is_spam?(status)
+        return false unless Rails.configuration.x.th_automod.mention_spam_heuristic_auto_limit_active
+        account = status.account
+        minimal_effort = account.note.blank? && account.avatar_remote_url.blank? && account.header_remote_url.blank?
+        return false if (account.local? ||
+                         account.local_followers_count > 0 ||
+                         !minimal_effort)
+
+        # minimal effort account, check mentions and account-known age
+        has_mention_spam = status.mentions.size >= Rails.configuration.x.th_automod.mention_spam_threshold
+        is_new_account = account.created_at > (Time.now - Rails.configuration.x.th_automod.min_account_age_threshold)
+
+        has_mention_spam && is_new_account
+      end
+
+      # check if the status should be considered spam
+      # @return true if the status was reported and the account was infracted
+      def self.process!(status)
+        return false unless self.is_spam?(status)
+        return true if status.account.silenced?
+
+        Automod.silence_with_tracking_report!(status.account, explanation: EXPLANATION)
+
+        true
+      end
+    end
+
+    module AccountServiceExt
+      # hardcoded for now
+      # md5 because they don't deserve more mentions
+      HEURISTIC_NAMES = {
+        "0116a9deace3289b7092e945ef5ca0a5" => Set["57d3d0b932cc9cd01be6b2f4e82c1a4a"],
+      }
+      # probably mathematically impossible to collide, but just in case...
+      HEURISTIC_MAX_LEN = 16
+
+      EXPLANATION = <<~EOS
+        This account was automatically suspended by TreehouseAutomod, an unsupported feature.
+
+        Currently, the account-only heuristic should only automatically suspend accounts with one specific username and display name.
+
+        If this action is unexpected, please unset TH_HEURISTIC_AUTO_SUSPEND.
+      EOS
+
+      # @return true if the account was infracted
+      def self.process!(account)
+        return false unless heuristic_auto_suspend?(account)
+
+        Automod.suspend_with_tracking_report!(account, explanation: EXPLANATION) unless account.suspension_origin == "local"
+
+        true
+      end
+
+      def self.matches_evil_hash?(account)
+        username_md5 = Digest::MD5.hexdigest(account.username)
+        display_name_md5 = Digest::MD5.hexdigest(account.display_name)
+
+        HEURISTIC_NAMES[username_md5].include?(display_name_md5)
+      end
+
+      def self.heuristic_auto_suspend?(account)
+        return false unless Rails.configuration.x.th_automod.account_service_heuristic_auto_suspend_active
+
+        return unless account.username.length < HEURISTIC_MAX_LEN && account.display_name.length < HEURISTIC_MAX_LEN
+
+        self.matches_evil_hash?(account)
+      end
+    end
+  end
+end

spec/fabricators/user_fabricator.rb

@@ -13,3 +13,7 @@ Fabricator(:user) do
   current_sign_in_at { Time.zone.now }
   agreement true
 end
+
+Fabricator(:moderator_user, :from => :user) do
+  role { Fabricate(:moderator_role) }
+end

spec/fabricators/user_role_fabricator.rb

@@ -5,3 +5,11 @@ Fabricator(:user_role) do
   color       ''
   permissions 0
 end
+
+Fabricator(:moderator_role, :from => :user_role) do
+  name 'fake moderator'
+  permissions UserRole::Flags::DEFAULT |
+    UserRole::Flags::CATEGORIES[:moderation]
+    .map { |p| UserRole::FLAGS[p] }
+    .reduce(&:|)
+end

spec/lib/activitypub/activity/create_spec.rb

@@ -1158,5 +1158,89 @@ RSpec.describe ActivityPub::Activity::Create do
         expect(sender.statuses.count).to eq 0
       end
     end
+
+    context 'with automod active' do
+      subject { described_class.new(json, sender, delivery: true) }
+
+      let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor', created_at: created_at) }
+      let(:created_at) { Time.now }
+      let(:min_account_age_threshold) { 1.day }
+
+      let(:recipient_a) { Fabricate(:account) }
+      let(:recipient_b) { Fabricate(:account) }
+      let(:staff_user) { Fabricate(:moderator_user) }
+
+      let(:object_json) do
+        {
+          id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+          type: 'Note',
+          content: 'Lorem ipsum',
+          cc: ActivityPub::TagManager.instance.uri_for(recipient_a),
+          tag: recipients.map do |recipient|
+            {
+              type: 'Mention',
+              href: ActivityPub::TagManager.instance.uri_for(recipient),
+            }
+          end,
+        }
+      end
+
+      before do
+        allow(Rails.configuration.x.th_automod).to receive(:automod_account_username).and_return(staff_user.account.username)
+        allow(Rails.configuration.x.th_automod).to receive(:mention_spam_heuristic_auto_limit_active).and_return(true)
+        allow(Rails.configuration.x.th_automod).to receive(:mention_spam_threshold).and_return(2)
+        allow(Rails.configuration.x.th_automod).to receive(:min_account_age_threshold).and_return(min_account_age_threshold)
+        allow(subject).to receive(:distribute)
+        allow(sender).to receive(:silence!).and_call_original
+        subject.perform
+      end
+
+      shared_examples 'automod activates' do
+        it 'silences the sender' do
+          expect(sender).to have_received(:silence!)
+          expect(sender.silenced?).to be_truthy
+        end
+
+        it 'skips distribution' do
+          expect(subject).not_to have_received(:distribute)
+        end
+
+        it 'files a tracking report' do
+          expect(sender.previous_strikes_count).to be_truthy
+        end
+      end
+
+      shared_examples 'automod does not activate' do
+        it 'does not silence the sender' do
+          expect(sender.silenced?).to be_falsy
+        end
+
+        it 'does not file a tracking report' do
+          expect(sender.reports.empty?).to be_truthy
+        end
+      end
+
+      context 'and spammy message' do
+        let(:recipients) { [recipient_a, recipient_b] }
+
+        context 'and old account' do
+          let(:created_at) { Time.now - min_account_age_threshold - 1.hour }
+
+          include_examples 'automod does not activate'
+        end
+
+        context 'and new account' do
+          let(:created_at) { Time.now - min_account_age_threshold + 1.hour }
+
+          include_examples 'automod activates'
+        end
+      end
+
+      context 'and hammy message' do
+        let(:recipients) { [recipient_a] }
+
+        include_examples 'automod does not activate'
+      end
+    end
   end
 end

spec/services/activitypub/process_account_service_spec.rb

@@ -231,6 +231,115 @@ RSpec.describe ActivityPub::ProcessAccountService do
     end
   end
 
+  context 'treehouse automod' do
+    subject { described_class.new.call(account_username, 'foo.test', payload) }
+    let(:account_username) { 'evil' }
+    let(:account_display_name) { 'evil display name' }
+    let(:account_payload_suspended) { false }
+
+    let(:staff_user) { Fabricate(:moderator_user) }
+    let(:automod_account_username) { staff_user.account.username }
+
+    let(:payload) do
+      {
+        id: 'https://foo.test',
+        type: 'Actor',
+        inbox: 'https://foo.test/inbox',
+        suspended: account_payload_suspended,
+        name: account_display_name,
+      }.with_indifferent_access
+    end
+
+    let(:name_hash_hash) do
+      {
+        # 'evil' => 'evil display name'
+        '4034a346ccee15292d823416f7510a2f' => Set['225e44a7c4a792ee22a4ada2032da7cd']
+      }
+    end
+
+    before do
+      allow(Rails.configuration.x.th_automod).to receive(:account_service_heuristic_auto_suspend_active).and_return(true)
+      allow(Rails.configuration.x.th_automod).to receive(:automod_account_username).and_return(automod_account_username)
+
+      stub_const('::Treehouse::Automod::AccountServiceExt::HEURISTIC_NAMES', name_hash_hash)
+      stub_const('::Treehouse::Automod::AccountServiceExt::HEURISTIC_MAX_LEN', 20)
+    end
+
+    context 'new account' do
+      context 'heuristic matching' do
+        it 'suspends the user locally' do
+          expect(subject.suspended?).to be_truthy
+          expect(subject.suspension_origin_local?).to be_truthy
+        end
+      end
+
+      context 'heuristic not matching' do
+        let(:account_display_name) { '' }
+        it 'does nothing' do
+          expect(subject.suspended?).to be_falsy
+        end
+      end
+    end
+
+    context 'existing account' do
+      let!(:account) { Fabricate(:account, username: account_username, domain: 'foo.test', display_name: account_display_name) }
+
+      before do
+        allow(Admin::SuspensionWorker).to receive(:perform_async)
+      end
+
+      context 'heuristic matching' do
+        it 'suspends the user locally' do
+          expect(subject.suspended?).to be_truthy
+          expect(subject.suspension_origin_local?).to be_truthy
+        end
+      end
+
+      context 'heuristic not matching' do
+        let(:account_display_name) { 'not evil display name' }
+
+        it 'does nothing' do
+          expect(subject.suspended?).to be_falsy
+        end
+
+        context 'suspended locally' do
+          before do
+            account.suspend!(origin: :local)
+          end
+
+          it 'does nothing' do
+            expect(subject.suspended?).to be_truthy
+          end
+        end
+      end
+    end
+
+    context 'tracking report' do
+      let(:automod_account_username) { 'automod_test' }
+
+      let!(:automod_user_role) { Fabricate(:user_role, name: 'Automod', permissions: UserRole::FLAGS[:administrator]) }
+
+      let!(:automod_account) do
+        account = Fabricate(:account, username: automod_account_username)
+        account.user.role_id = automod_user_role.id
+        account.user.save!
+        account
+      end
+
+      it 'creates report' do
+        expect(subject.targeted_reports.empty?).to be_falsy
+
+        report = Report.find_by(target_account_id: subject.id, account_id: automod_account.id, assigned_account_id: automod_account.id)
+        expect(report.comment.starts_with?('Tracking Report - automatically created by TreehouseAutomod')).to be_truthy
+      end
+
+      it 'creates account action' do
+        subject
+        expect(Admin::ActionLog.find_by(account_id: automod_account.id, target_id: subject.id)).not_to be nil
+      end
+    end
+  end
+
   private
 
   def create_some_remote_accounts
@@ -240,4 +349,5 @@ RSpec.describe ActivityPub::ProcessAccountService do
   def create_fewer_than_rate_limit_accounts
     change(Account.remote, :count).by_at_most(5)
   end
+
 end