From 8f6c67bfdeddd1c2c1085067e3dc549fb53f6ff4 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 8 Mar 2022 09:14:39 +0100 Subject: [PATCH] Fix performance of account timelines (#17709) * Fix performance of account timelines * Various fixes and improvements * Fix duplicate results being returned Co-authored-by: Claire * Fix grouping for pinned statuses scope Co-authored-by: Claire --- .../activitypub/outboxes_controller.rb | 2 +- .../api/v1/accounts/statuses_controller.rb | 41 +--- app/models/account_statuses_filter.rb | 134 ++++++++++ app/models/status.rb | 22 -- spec/models/account_statuses_filter_spec.rb | 229 ++++++++++++++++++ spec/models/status_spec.rb | 53 ---- 6 files changed, 366 insertions(+), 115 deletions(-) create mode 100644 app/models/account_statuses_filter.rb create mode 100644 spec/models/account_statuses_filter_spec.rb diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb index b2aab56a56..cd3992502d 100644 --- a/app/controllers/activitypub/outboxes_controller.rb +++ b/app/controllers/activitypub/outboxes_controller.rb @@ -62,7 +62,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController return unless page_requested? @statuses = cache_collection_paginated_by_id( - @account.statuses.permitted_for(@account, signed_request_account), + AccountStatusesFilter.new(@account, signed_request_account).results, Status, LIMIT, params_slice(:max_id, :min_id, :since_id) diff --git a/app/controllers/api/v1/accounts/statuses_controller.rb b/app/controllers/api/v1/accounts/statuses_controller.rb index 2c027ea769..38c9f5a20d 100644 --- a/app/controllers/api/v1/accounts/statuses_controller.rb +++ b/app/controllers/api/v1/accounts/statuses_controller.rb @@ -22,53 +22,16 @@ class Api::V1::Accounts::StatusesController < Api::BaseController end def cached_account_statuses - statuses = truthy_param?(:pinned) ? pinned_scope : permitted_account_statuses - - statuses.merge!(only_media_scope) if truthy_param?(:only_media) - statuses.merge!(no_replies_scope) if truthy_param?(:exclude_replies) - statuses.merge!(no_reblogs_scope) if truthy_param?(:exclude_reblogs) - statuses.merge!(hashtag_scope) if params[:tagged].present? - cache_collection_paginated_by_id( - statuses, + AccountStatusesFilter.new(@account, current_account, params).results, Status, limit_param(DEFAULT_STATUSES_LIMIT), params_slice(:max_id, :since_id, :min_id) ) end - def permitted_account_statuses - @account.statuses.permitted_for(@account, current_account) - end - - def only_media_scope - Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id) - end - - def pinned_scope - @account.pinned_statuses.permitted_for(@account, current_account) - end - - def no_replies_scope - Status.without_replies - end - - def no_reblogs_scope - Status.without_reblogs - end - - def hashtag_scope - tag = Tag.find_normalized(params[:tagged]) - - if tag - Status.tagged_with(tag.id) - else - Status.none - end - end - def pagination_params(core_params) - params.slice(:limit, :only_media, :exclude_replies).permit(:limit, :only_media, :exclude_replies).merge(core_params) + params.slice(:limit, *AccountStatusesFilter::KEYS).permit(:limit, *AccountStatusesFilter::KEYS).merge(core_params) end def insert_pagination_headers diff --git a/app/models/account_statuses_filter.rb b/app/models/account_statuses_filter.rb new file mode 100644 index 0000000000..211f414787 --- /dev/null +++ b/app/models/account_statuses_filter.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +class AccountStatusesFilter + KEYS = %i( + pinned + tagged + only_media + exclude_replies + exclude_reblogs + ).freeze + + attr_reader :params, :account, :current_account + + def initialize(account, current_account, params = {}) + @account = account + @current_account = current_account + @params = params + end + + def results + scope = initial_scope + + scope.merge!(pinned_scope) if pinned? + scope.merge!(only_media_scope) if only_media? + scope.merge!(no_replies_scope) if exclude_replies? + scope.merge!(no_reblogs_scope) if exclude_reblogs? + scope.merge!(hashtag_scope) if tagged? + + scope + end + + private + + def initial_scope + if suspended? + Status.none + elsif anonymous? + account.statuses.where(visibility: %i(public unlisted)) + elsif author? + account.statuses.all # NOTE: #merge! does not work without the #all + elsif blocked? + Status.none + else + filtered_scope + end + end + + def filtered_scope + scope = account.statuses.left_outer_joins(:mentions) + + scope.merge!(scope.where(visibility: follower? ? %i(public unlisted private) : %i(public unlisted)).or(scope.where(mentions: { account_id: current_account.id })).group(Status.arel_table[:id])) + scope.merge!(filtered_reblogs_scope) if reblogs_may_occur? + + scope + end + + def filtered_reblogs_scope + Status.left_outer_joins(:reblog).where(reblog_of_id: nil).or(Status.where.not(reblogs_statuses: { account_id: current_account.excluded_from_timeline_account_ids })) + end + + def only_media_scope + Status.joins(:media_attachments).merge(account.media_attachments.reorder(nil)).group(Status.arel_table[:id]) + end + + def no_replies_scope + Status.without_replies + end + + def no_reblogs_scope + Status.without_reblogs + end + + def pinned_scope + account.pinned_statuses.group(Status.arel_table[:id], StatusPin.arel_table[:created_at]) + end + + def hashtag_scope + tag = Tag.find_normalized(params[:tagged]) + + if tag + Status.tagged_with(tag.id) + else + Status.none + end + end + + def suspended? + account.suspended? + end + + def anonymous? + current_account.nil? + end + + def author? + current_account.id == account.id + end + + def blocked? + account.blocking?(current_account) || (current_account.domain.present? && account.domain_blocking?(current_account.domain)) + end + + def follower? + current_account.following?(account) + end + + def reblogs_may_occur? + !exclude_reblogs? && !only_media? && !tagged? + end + + def pinned? + truthy_param?(:pinned) + end + + def only_media? + truthy_param?(:only_media) + end + + def exclude_replies? + truthy_param?(:exclude_replies) + end + + def exclude_reblogs? + truthy_param?(:exclude_reblogs) + end + + def tagged? + params[:tagged].present? + end + + def truthy_param?(key) + ActiveModel::Type::Boolean.new.cast(params[key]) + end +end diff --git a/app/models/status.rb b/app/models/status.rb index 60dde50459..af3e645dc3 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -345,28 +345,6 @@ class Status < ApplicationRecord end end - def permitted_for(target_account, account) - visibility = [:public, :unlisted] - - if account.nil? - where(visibility: visibility) - elsif target_account.blocking?(account) || (account.domain.present? && target_account.domain_blocking?(account.domain)) # get rid of blocked peeps - none - elsif account.id == target_account.id # author can see own stuff - all - else - # followers can see followers-only stuff, but also things they are mentioned in. - # non-followers can see everything that isn't private/direct, but can see stuff they are mentioned in. - visibility.push(:private) if account.following?(target_account) - - scope = left_outer_joins(:reblog) - - scope.where(visibility: visibility) - .or(scope.where(id: account.mentions.select(:status_id))) - .merge(scope.where(reblog_of_id: nil).or(scope.where.not(reblogs_statuses: { account_id: account.excluded_from_timeline_account_ids }))) - end - end - def from_text(text) return [] if text.blank? diff --git a/spec/models/account_statuses_filter_spec.rb b/spec/models/account_statuses_filter_spec.rb new file mode 100644 index 0000000000..03f0ffeb06 --- /dev/null +++ b/spec/models/account_statuses_filter_spec.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe AccountStatusesFilter do + let(:account) { Fabricate(:account) } + let(:current_account) { nil } + let(:params) { {} } + + subject { described_class.new(account, current_account, params) } + + def status!(visibility) + Fabricate(:status, account: account, visibility: visibility) + end + + def status_with_tag!(visibility, tag) + Fabricate(:status, account: account, visibility: visibility, tags: [tag]) + end + + def status_with_parent!(visibility) + Fabricate(:status, account: account, visibility: visibility, thread: Fabricate(:status)) + end + + def status_with_reblog!(visibility) + Fabricate(:status, account: account, visibility: visibility, reblog: Fabricate(:status)) + end + + def status_with_mention!(visibility, mentioned_account = nil) + Fabricate(:status, account: account, visibility: visibility).tap do |status| + Fabricate(:mention, status: status, account: mentioned_account || Fabricate(:account)) + end + end + + def status_with_media_attachment!(visibility) + Fabricate(:status, account: account, visibility: visibility).tap do |status| + Fabricate(:media_attachment, account: account, status: status) + end + end + + describe '#results' do + let(:tag) { Fabricate(:tag) } + + before do + status!(:public) + status!(:unlisted) + status!(:private) + status_with_parent!(:public) + status_with_reblog!(:public) + status_with_tag!(:public, tag) + status_with_mention!(:direct) + status_with_media_attachment!(:public) + end + + shared_examples 'filter params' do + context 'with only_media param' do + let(:params) { { only_media: true } } + + it 'returns only statuses with media' do + expect(subject.results.all?(&:with_media?)).to be true + end + end + + context 'with tagged param' do + let(:params) { { tagged: tag.name } } + + it 'returns only statuses with tag' do + expect(subject.results.all? { |s| s.tags.include?(tag) }).to be true + end + end + + context 'with exclude_replies param' do + let(:params) { { exclude_replies: true } } + + it 'returns only statuses that are not replies' do + expect(subject.results.none?(&:reply?)).to be true + end + end + + context 'with exclude_reblogs param' do + let(:params) { { exclude_reblogs: true } } + + it 'returns only statuses that are not reblogs' do + expect(subject.results.none?(&:reblog?)).to be true + end + end + end + + context 'when accessed anonymously' do + let(:current_account) { nil } + let(:direct_status) { nil } + + it 'returns only public statuses' do + expect(subject.results.pluck(:visibility).uniq).to match_array %w(unlisted public) + end + + it 'returns public replies' do + expect(subject.results.pluck(:in_reply_to_id)).to_not be_empty + end + + it 'returns public reblogs' do + expect(subject.results.pluck(:reblog_of_id)).to_not be_empty + end + + it_behaves_like 'filter params' + end + + context 'when accessed with a blocked account' do + let(:current_account) { Fabricate(:account) } + + before do + account.block!(current_account) + end + + it 'returns nothing' do + expect(subject.results.to_a).to be_empty + end + end + + context 'when accessed by self' do + let(:current_account) { account } + + it 'returns everything' do + expect(subject.results.pluck(:visibility).uniq).to match_array %w(direct private unlisted public) + end + + it 'returns replies' do + expect(subject.results.pluck(:in_reply_to_id)).to_not be_empty + end + + it 'returns reblogs' do + expect(subject.results.pluck(:reblog_of_id)).to_not be_empty + end + + it_behaves_like 'filter params' + end + + context 'when accessed by a follower' do + let(:current_account) { Fabricate(:account) } + + before do + current_account.follow!(account) + end + + it 'returns private statuses' do + expect(subject.results.pluck(:visibility).uniq).to match_array %w(private unlisted public) + end + + it 'returns replies' do + expect(subject.results.pluck(:in_reply_to_id)).to_not be_empty + end + + it 'returns reblogs' do + expect(subject.results.pluck(:reblog_of_id)).to_not be_empty + end + + context 'when there is a direct status mentioning the non-follower' do + let!(:direct_status) { status_with_mention!(:direct, current_account) } + + it 'returns the direct status' do + expect(subject.results.pluck(:id)).to include(direct_status.id) + end + end + + it_behaves_like 'filter params' + end + + context 'when accessed by a non-follower' do + let(:current_account) { Fabricate(:account) } + + it 'returns only public statuses' do + expect(subject.results.pluck(:visibility).uniq).to match_array %w(unlisted public) + end + + it 'returns public replies' do + expect(subject.results.pluck(:in_reply_to_id)).to_not be_empty + end + + it 'returns public reblogs' do + expect(subject.results.pluck(:reblog_of_id)).to_not be_empty + end + + context 'when there is a private status mentioning the non-follower' do + let!(:private_status) { status_with_mention!(:private, current_account) } + + it 'returns the private status' do + expect(subject.results.pluck(:id)).to include(private_status.id) + end + end + + context 'when blocking a reblogged account' do + let(:reblog) { status_with_reblog!('public') } + + before do + current_account.block!(reblog.reblog.account) + end + + it 'does not return reblog of blocked account' do + expect(subject.results.pluck(:id)).to_not include(reblog.id) + end + end + + context 'when muting a reblogged account' do + let(:reblog) { status_with_reblog!('public') } + + before do + current_account.mute!(reblog.reblog.account) + end + + it 'does not return reblog of muted account' do + expect(subject.results.pluck(:id)).to_not include(reblog.id) + end + end + + context 'when blocked by a reblogged account' do + let(:reblog) { status_with_reblog!('public') } + + before do + reblog.reblog.account.block!(current_account) + end + + it 'does not return reblog of blocked-by account' do + expect(subject.results.pluck(:id)).to_not include(reblog.id) + end + end + + it_behaves_like 'filter params' + end + end +end diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index 67af6d06d1..130f4d03fa 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -348,59 +348,6 @@ RSpec.describe Status, type: :model do end end - describe '.permitted_for' do - subject { described_class.permitted_for(target_account, account).pluck(:visibility) } - - let(:target_account) { alice } - let(:account) { bob } - let!(:public_status) { Fabricate(:status, account: target_account, visibility: 'public') } - let!(:unlisted_status) { Fabricate(:status, account: target_account, visibility: 'unlisted') } - let!(:private_status) { Fabricate(:status, account: target_account, visibility: 'private') } - - let!(:direct_status) do - Fabricate(:status, account: target_account, visibility: 'direct').tap do |status| - Fabricate(:mention, status: status, account: account) - end - end - - let!(:other_direct_status) do - Fabricate(:status, account: target_account, visibility: 'direct').tap do |status| - Fabricate(:mention, status: status) - end - end - - context 'given nil' do - let(:account) { nil } - let(:direct_status) { nil } - it { is_expected.to eq(%w(unlisted public)) } - end - - context 'given blocked account' do - before do - target_account.block!(account) - end - - it { is_expected.to be_empty } - end - - context 'given same account' do - let(:account) { target_account } - it { is_expected.to eq(%w(direct direct private unlisted public)) } - end - - context 'given followed account' do - before do - account.follow!(target_account) - end - - it { is_expected.to eq(%w(direct private unlisted public)) } - end - - context 'given unfollowed account' do - it { is_expected.to eq(%w(direct unlisted public)) } - end - end - describe 'before_validation' do it 'sets account being replied to correctly over intermediary nodes' do first_status = Fabricate(:status, account: bob)