treehouse
/
mastodon
21
8
Fork 5
Code Issues 32 Pull Requests Projects 1 Releases Wiki Activity

Fix unbounded recursion in account discovery (#22025) 

* Fix trying to fetch posts from other users when fetching featured posts

* Rate-limit discovery of new subdomains

* Put a limit on recursively discovering new accounts
lolsob-rspec
Claire 2022-12-07 00:15:24 +01:00 committed by GitHub
parent 5c93b9fbcd
commit aec7de494f
12 changed files with 147 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile
View File

@ -66,6 +66,7 @@ gem 'oj', '~> 3.13'
gem 'ox', '~> 2.14'
gem 'parslet'
gem 'posix-spawn'
gem 'public_suffix', '~> 5.0'
gem 'pundit', '~> 2.2'
gem 'premailer-rails'
gem 'rack-attack', '~> 6.6'

1
Gemfile.lock
View File

@ -819,6 +819,7 @@ DEPENDENCIES
private_address_check (~> 0.5)
pry-byebug (~> 3.10)
pry-rails (~> 0.3)
public_suffix (~> 5.0)
puma (~> 5.6)
pundit (~> 2.2)
rack (~> 2.2.4)

2
app/lib/activitypub/activity/create.rb
View File

@ -222,7 +222,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
return if tag['href'].blank?
account = account_from_uri(tag['href'])
account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
account = ActivityPub::FetchRemoteAccountService.new.call(tag['href'], request_id: @options[:request_id]) if account.nil?
return if account.nil?

4
app/lib/activitypub/activity/update.rb
View File

@ -18,7 +18,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
def update_account
return reject_payload! if @account.uri != object_uri
ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true)
ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true, request_id: @options[:request_id])
end
def update_status
@ -28,6 +28,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
return if @status.nil?
ActivityPub::ProcessStatusUpdateService.new.call(@status, @object)
ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id])
end
end

15
app/models/concerns/domain_materializable.rb
View File

@ -3,11 +3,24 @@
module DomainMaterializable
extend ActiveSupport::Concern
include Redisable
included do
after_create_commit :refresh_instances_view
end
def refresh_instances_view
Instance.refresh unless domain.nil? || Instance.where(domain: domain).exists?
return if domain.nil? || Instance.exists?(domain: domain)
Instance.refresh
count_unique_subdomains!
end
def count_unique_subdomains!
second_and_top_level_domain = PublicSuffix.domain(domain, ignore_private: true)
with_redis do |redis|
redis.pfadd("unique_subdomains_for:#{second_and_top_level_domain}", domain)
redis.expire("unique_subdomains_for:#{second_and_top_level_domain}", 1.minute.seconds)
end
end
end

4
app/services/activitypub/fetch_featured_collection_service.rb
View File

@ -46,9 +46,9 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
next unless item.is_a?(String) || item['type'] == 'Note'
uri = value_or_id(item)
next if ActivityPub::TagManager.instance.local_uri?(uri)
next if ActivityPub::TagManager.instance.local_uri?(uri) || invalid_origin?(uri)
status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower)
status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower, expected_actor_uri: @account.uri, request_id: @options[:request_id])
next unless status&.account_id == @account.id
status.id

2
app/services/activitypub/fetch_remote_account_service.rb
View File

@ -2,7 +2,7 @@
class ActivityPub::FetchRemoteAccountService < ActivityPub::FetchRemoteActorService
# Does a WebFinger roundtrip on each call, unless `only_key` is true
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true)
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
actor = super
return actor if actor.nil? || actor.is_a?(Account)

4
app/services/activitypub/fetch_remote_actor_service.rb
View File

@ -10,7 +10,7 @@ class ActivityPub::FetchRemoteActorService < BaseService
SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
# Does a WebFinger roundtrip on each call, unless `only_key` is true
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true)
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
return if domain_not_allowed?(uri)
return ActivityPub::TagManager.instance.uri_to_actor(uri) if ActivityPub::TagManager.instance.local_uri?(uri)
@ -35,7 +35,7 @@ class ActivityPub::FetchRemoteActorService < BaseService
check_webfinger! unless only_key
ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key)
ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key, request_id: request_id)
rescue Error => e
Rails.logger.debug "Fetching actor #{uri} failed: #{e.message}"
raise unless suppress_errors

8
app/services/activitypub/fetch_remote_status_service.rb
View File

@ -4,7 +4,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService
include JsonLdHelper
# Should be called when uri has already been checked for locality
def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil)
def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
@request_id = request_id
@json = begin
if prefetched_body.nil?
fetch_resource(uri, id, on_behalf_of)
@ -30,6 +31,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
end
return if activity_json.nil? || object_uri.nil? || !trustworthy_attribution?(@json['id'], actor_uri)
return if expected_actor_uri.present? && actor_uri != expected_actor_uri
return ActivityPub::TagManager.instance.uri_to_resource(object_uri, Status) if ActivityPub::TagManager.instance.local_uri?(object_uri)
actor = account_from_uri(actor_uri)
@ -40,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
# activity as an update rather than create
activity_json['type'] = 'Update' if equals_or_includes_any?(activity_json['type'], %w(Create)) && Status.where(uri: object_uri, account_id: actor.id).exists?
ActivityPub::Activity.factory(activity_json, actor).perform
ActivityPub::Activity.factory(activity_json, actor, request_id: request_id).perform
end
private
@ -52,7 +54,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
def account_from_uri(uri)
actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true) if actor.nil? || actor.possibly_stale?
actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true, request_id: @request_id) if actor.nil? || actor.possibly_stale?
actor
end

25
app/services/activitypub/process_account_service.rb
View File

@ -6,6 +6,9 @@ class ActivityPub::ProcessAccountService < BaseService
include Redisable
include Lockable
SUBDOMAINS_RATELIMIT = 10
DISCOVERIES_PER_REQUEST = 400
# Should be called with confirmed valid JSON
# and WebFinger-resolved username and domain
def call(username, domain, json, options = {})
@ -15,9 +18,12 @@ class ActivityPub::ProcessAccountService < BaseService
@json = json
@uri = @json['id']
@username = username
@domain = domain
@domain = TagManager.instance.normalize_domain(domain)
@collections = {}
# The key does not need to be unguessable, it just needs to be somewhat unique
@options[:request_id] ||= "#{Time.now.utc.to_i}-#{username}@#{domain}"
with_lock("process_account:#{@uri}") do
@account = Account.remote.find_by(uri: @uri) if @options[:only_key]
@account ||= Account.find_remote(@username, @domain)
@ -25,7 +31,18 @@ class ActivityPub::ProcessAccountService < BaseService
@old_protocol = @account&.protocol
@suspension_changed = false
create_account if @account.nil?
if @account.nil?
with_redis do |redis|
return nil if redis.pfcount("unique_subdomains_for:#{PublicSuffix.domain(@domain, ignore_private: true)}") >= SUBDOMAINS_RATELIMIT
discoveries = redis.incr("discovery_per_request:#{@options[:request_id]}")
redis.expire("discovery_per_request:#{@options[:request_id]}", 5.minutes.seconds)
return nil if discoveries > DISCOVERIES_PER_REQUEST
end
create_account
end
update_account
process_tags
@ -150,7 +167,7 @@ class ActivityPub::ProcessAccountService < BaseService
end
def check_featured_collection!
ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'hashtag' => @json['featuredTags'].blank? })
ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'hashtag' => @json['featuredTags'].blank?, 'request_id' => @options[:request_id] })
end
def check_featured_tags_collection!
@ -254,7 +271,7 @@ class ActivityPub::ProcessAccountService < BaseService
def moved_account
account = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account)
account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true)
account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true, request_id: @options[:request_id])
account
end

5
app/services/activitypub/process_status_update_service.rb
View File

@ -5,7 +5,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
include Redisable
include Lockable
def call(status, json)
def call(status, json, request_id: nil)
raise ArgumentError, 'Status has unsaved changes' if status.changed?
@json = json
@ -15,6 +15,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
@account = status.account
@media_attachments_changed = false
@poll_changed = false
@request_id = request_id
# Only native types can be updated at the moment
return @status if !expected_type? || already_updated_more_recently?
@ -191,7 +192,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
next if href.blank?
account = ActivityPub::TagManager.instance.uri_to_resource(href, Account)
account ||= ActivityPub::FetchRemoteAccountService.new.call(href)
account ||= ActivityPub::FetchRemoteAccountService.new.call(href, request_id: @request_id)
next if account.nil?

94
spec/services/activitypub/process_account_service_spec.rb
View File

@ -109,4 +109,98 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
end
end
end
context 'discovering many subdomains in a short timeframe' do
before do
stub_const 'ActivityPub::ProcessAccountService::SUBDOMAINS_RATELIMIT', 5
end
let(:subject) do
8.times do |i|
domain = "test#{i}.testdomain.com"
json = {
id: "https://#{domain}/users/1",
type: 'Actor',
inbox: "https://#{domain}/inbox",
}.with_indifferent_access
described_class.new.call('alice', domain, json)
end
end
it 'creates at least some accounts' do
expect { subject }.to change { Account.remote.count }.by_at_least(2)
end
it 'creates no more account than the limit allows' do
expect { subject }.to change { Account.remote.count }.by_at_most(5)
end
end
context 'accounts referencing other accounts' do
before do
stub_const 'ActivityPub::ProcessAccountService::DISCOVERIES_PER_REQUEST', 5
end
let(:payload) do
{
'@context': ['https://www.w3.org/ns/activitystreams'],
id: 'https://foo.test/users/1',
type: 'Person',
inbox: 'https://foo.test/inbox',
featured: 'https://foo.test/users/1/featured',
preferredUsername: 'user1',
}.with_indifferent_access
end
before do
8.times do |i|
actor_json = {
'@context': ['https://www.w3.org/ns/activitystreams'],
id: "https://foo.test/users/#{i}",
type: 'Person',
inbox: 'https://foo.test/inbox',
featured: "https://foo.test/users/#{i}/featured",
preferredUsername: "user#{i}",
}.with_indifferent_access
status_json = {
'@context': ['https://www.w3.org/ns/activitystreams'],
id: "https://foo.test/users/#{i}/status",
attributedTo: "https://foo.test/users/#{i}",
type: 'Note',
content: "@user#{i + 1} test",
tag: [
{
type: 'Mention',
href: "https://foo.test/users/#{i + 1}",
name: "@user#{i + 1 }",
}
],
to: [ 'as:Public', "https://foo.test/users/#{i + 1}" ]
}.with_indifferent_access
featured_json = {
'@context': ['https://www.w3.org/ns/activitystreams'],
id: "https://foo.test/users/#{i}/featured",
type: 'OrderedCollection',
totelItems: 1,
orderedItems: [status_json],
}.with_indifferent_access
webfinger = {
subject: "acct:user#{i}@foo.test",
links: [{ rel: 'self', href: "https://foo.test/users/#{i}" }],
}.with_indifferent_access
stub_request(:get, "https://foo.test/users/#{i}").to_return(status: 200, body: actor_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
stub_request(:get, "https://foo.test/users/#{i}/featured").to_return(status: 200, body: featured_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
stub_request(:get, "https://foo.test/users/#{i}/status").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' })
stub_request(:get, "https://foo.test/.well-known/webfinger?resource=acct:user#{i}@foo.test").to_return(body: webfinger.to_json, headers: { 'Content-Type': 'application/jrd+json' })
end
end
it 'creates at least some accounts' do
expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_least(2)
end
it 'creates no more account than the limit allows' do
expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_most(5)
end
end
end