diff --git a/.rubocop.yml b/.rubocop.yml index 2af0f59bbe0..68634e9e33a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -32,10 +32,11 @@ Layout/EmptyLineAfterGuardClause: Layout/EmptyLinesAroundAttributeAccessor: Enabled: true +Layout/FirstHashElementIndentation: + EnforcedStyle: consistent + Layout/HashAlignment: Enabled: false - # EnforcedHashRocketStyle: table - # EnforcedColonStyle: table Layout/SpaceAroundMethodCallOperator: Enabled: true diff --git a/app/controllers/admin/trends/links/preview_card_providers_controller.rb b/app/controllers/admin/trends/links/preview_card_providers_controller.rb index 2c26e03f361..40a466cd630 100644 --- a/app/controllers/admin/trends/links/preview_card_providers_controller.rb +++ b/app/controllers/admin/trends/links/preview_card_providers_controller.rb @@ -5,11 +5,11 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll authorize :preview_card_provider, :index? @preview_card_providers = filtered_preview_card_providers.page(params[:page]) - @form = Form::PreviewCardProviderBatch.new + @form = Trends::PreviewCardProviderBatch.new end def batch - @form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) + @form = Trends::PreviewCardProviderBatch.new(trends_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') @@ -20,15 +20,15 @@ class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseControll private def filtered_preview_card_providers - PreviewCardProviderFilter.new(filter_params).results + Trends::PreviewCardProviderFilter.new(filter_params).results end def filter_params - params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS) + params.slice(:page, *Trends::PreviewCardProviderFilter::KEYS).permit(:page, *Trends::PreviewCardProviderFilter::KEYS) end - def form_preview_card_provider_batch_params - params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) + def trends_preview_card_provider_batch_params + params.require(:trends_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) end def action_from_button diff --git a/app/controllers/admin/trends/links_controller.rb b/app/controllers/admin/trends/links_controller.rb index 619b37deb1d..434eec5feea 100644 --- a/app/controllers/admin/trends/links_controller.rb +++ b/app/controllers/admin/trends/links_controller.rb @@ -5,11 +5,11 @@ class Admin::Trends::LinksController < Admin::BaseController authorize :preview_card, :index? @preview_cards = filtered_preview_cards.page(params[:page]) - @form = Form::PreviewCardBatch.new + @form = Trends::PreviewCardBatch.new end def batch - @form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) + @form = Trends::PreviewCardBatch.new(trends_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') @@ -20,26 +20,26 @@ class Admin::Trends::LinksController < Admin::BaseController private def filtered_preview_cards - PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results + Trends::PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results end def filter_params - params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS) + params.slice(:page, *Trends::PreviewCardFilter::KEYS).permit(:page, *Trends::PreviewCardFilter::KEYS) end - def form_preview_card_batch_params - params.require(:form_preview_card_batch).permit(:action, preview_card_ids: []) + def trends_preview_card_batch_params + params.require(:trends_preview_card_batch).permit(:action, preview_card_ids: []) end def action_from_button if params[:approve] 'approve' - elsif params[:approve_all] - 'approve_all' + elsif params[:approve_providers] + 'approve_providers' elsif params[:reject] 'reject' - elsif params[:reject_all] - 'reject_all' + elsif params[:reject_providers] + 'reject_providers' end end end diff --git a/app/controllers/admin/trends/statuses_controller.rb b/app/controllers/admin/trends/statuses_controller.rb new file mode 100644 index 00000000000..7662427387d --- /dev/null +++ b/app/controllers/admin/trends/statuses_controller.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class Admin::Trends::StatusesController < Admin::BaseController + def index + authorize :status, :index? + + @statuses = filtered_statuses.page(params[:page]) + @form = Trends::StatusBatch.new + end + + def batch + @form = Trends::StatusBatch.new(trends_status_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_statuses_path(filter_params) + end + + private + + def filtered_statuses + Trends::StatusFilter.new(filter_params.with_defaults(trending: 'all')).results.includes(:account, :media_attachments, :active_mentions) + end + + def filter_params + params.slice(:page, *Trends::StatusFilter::KEYS).permit(:page, *Trends::StatusFilter::KEYS) + end + + def trends_status_batch_params + params.require(:trends_status_batch).permit(:action, status_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:approve_accounts] + 'approve_accounts' + elsif params[:reject] + 'reject' + elsif params[:reject_accounts] + 'reject_accounts' + end + end +end diff --git a/app/controllers/admin/trends/tags_controller.rb b/app/controllers/admin/trends/tags_controller.rb index 91ff33d406d..f4d1ec0d1f6 100644 --- a/app/controllers/admin/trends/tags_controller.rb +++ b/app/controllers/admin/trends/tags_controller.rb @@ -5,11 +5,11 @@ class Admin::Trends::TagsController < Admin::BaseController authorize :tag, :index? @tags = filtered_tags.page(params[:page]) - @form = Form::TagBatch.new + @form = Trends::TagBatch.new end def batch - @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) + @form = Trends::TagBatch.new(trends_tag_batch_params.merge(current_account: current_account, action: action_from_button)) @form.save rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.accounts.no_account_selected') @@ -20,15 +20,15 @@ class Admin::Trends::TagsController < Admin::BaseController private def filtered_tags - TagFilter.new(filter_params).results + Trends::TagFilter.new(filter_params).results end def filter_params - params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) + params.slice(:page, *Trends::TagFilter::KEYS).permit(:page, *Trends::TagFilter::KEYS) end - def form_tag_batch_params - params.require(:form_tag_batch).permit(:action, tag_ids: []) + def trends_tag_batch_params + params.require(:trends_tag_batch).permit(:action, tag_ids: []) end def action_from_button diff --git a/app/controllers/api/v1/admin/trends/links_controller.rb b/app/controllers/api/v1/admin/trends/links_controller.rb new file mode 100644 index 00000000000..63b3d9358e0 --- /dev/null +++ b/app/controllers/api/v1/admin/trends/links_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Api::V1::Admin::Trends::LinksController < Api::BaseController + protect_from_forgery with: :exception + + before_action -> { authorize_if_got_token! :'admin:read' } + before_action :require_staff! + before_action :set_links + + def index + render json: @links, each_serializer: REST::Trends::LinkSerializer + end + + private + + def set_links + @links = Trends.links.query.limit(limit_param(10)) + end +end diff --git a/app/controllers/api/v1/admin/trends/statuses_controller.rb b/app/controllers/api/v1/admin/trends/statuses_controller.rb new file mode 100644 index 00000000000..86633cc7430 --- /dev/null +++ b/app/controllers/api/v1/admin/trends/statuses_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Api::V1::Admin::Trends::StatusesController < Api::BaseController + protect_from_forgery with: :exception + + before_action -> { authorize_if_got_token! :'admin:read' } + before_action :require_staff! + before_action :set_statuses + + def index + render json: @statuses, each_serializer: REST::StatusSerializer + end + + private + + def set_statuses + @statuses = cache_collection(Trends.statuses.query.limit(limit_param(DEFAULT_STATUSES_LIMIT)), Status) + end +end diff --git a/app/controllers/api/v1/admin/trends/tags_controller.rb b/app/controllers/api/v1/admin/trends/tags_controller.rb index 4815af31ea9..5cc4c269d71 100644 --- a/app/controllers/api/v1/admin/trends/tags_controller.rb +++ b/app/controllers/api/v1/admin/trends/tags_controller.rb @@ -14,6 +14,6 @@ class Api::V1::Admin::Trends::TagsController < Api::BaseController private def set_tags - @tags = Trends.tags.get(false, limit_param(10)) + @tags = Trends.tags.query.limit(limit_param(10)) end end diff --git a/app/controllers/api/v1/trends/links_controller.rb b/app/controllers/api/v1/trends/links_controller.rb index 1c3ab1e1c2e..ad20e7f8be9 100644 --- a/app/controllers/api/v1/trends/links_controller.rb +++ b/app/controllers/api/v1/trends/links_controller.rb @@ -12,10 +12,14 @@ class Api::V1::Trends::LinksController < Api::BaseController def set_links @links = begin if Setting.trends - Trends.links.get(true, limit_param(10)) + links_from_trends else [] end end end + + def links_from_trends + Trends.links.query.allowed.in_locale(content_locale).limit(limit_param(10)) + end end diff --git a/app/controllers/api/v1/trends/statuses_controller.rb b/app/controllers/api/v1/trends/statuses_controller.rb new file mode 100644 index 00000000000..d4ec97ae5f1 --- /dev/null +++ b/app/controllers/api/v1/trends/statuses_controller.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class Api::V1::Trends::StatusesController < Api::BaseController + before_action :set_statuses + + def index + render json: @statuses, each_serializer: REST::StatusSerializer + end + + private + + def set_statuses + @statuses = begin + if Setting.trends + cache_collection(statuses_from_trends, Status) + else + [] + end + end + end + + def statuses_from_trends + scope = Trends.statuses.query.allowed.in_locale(content_locale) + scope = scope.filtered_for(current_account) if user_signed_in? + scope.limit(limit_param(DEFAULT_STATUSES_LIMIT)) + end +end diff --git a/app/controllers/api/v1/trends/tags_controller.rb b/app/controllers/api/v1/trends/tags_controller.rb index 947b53de234..1334b72d25c 100644 --- a/app/controllers/api/v1/trends/tags_controller.rb +++ b/app/controllers/api/v1/trends/tags_controller.rb @@ -12,7 +12,7 @@ class Api::V1::Trends::TagsController < Api::BaseController def set_tags @tags = begin if Setting.trends - Trends.tags.get(true, limit_param(10)) + Trends.tags.query.allowed.limit(limit_param(10)) else [] end diff --git a/app/controllers/concerns/localized.rb b/app/controllers/concerns/localized.rb index 173316800fa..ede299d5a4d 100644 --- a/app/controllers/concerns/localized.rb +++ b/app/controllers/concerns/localized.rb @@ -27,4 +27,8 @@ module Localized def available_locale_or_nil(locale_name) locale_name.to_sym if locale_name.present? && I18n.available_locales.include?(locale_name.to_sym) end + + def content_locale + @content_locale ||= I18n.locale.to_s.split(/[_-]/).first + end end diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb index 907529b3723..140fc73ede4 100644 --- a/app/helpers/admin/filter_helper.rb +++ b/app/helpers/admin/filter_helper.rb @@ -5,9 +5,10 @@ module Admin::FilterHelper AccountFilter::KEYS, CustomEmojiFilter::KEYS, ReportFilter::KEYS, - TagFilter::KEYS, - PreviewCardProviderFilter::KEYS, - PreviewCardFilter::KEYS, + Trends::TagFilter::KEYS, + Trends::PreviewCardProviderFilter::KEYS, + Trends::PreviewCardFilter::KEYS, + Trends::StatusFilter::KEYS, InstanceFilter::KEYS, InviteFilter::KEYS, RelationshipFilter::KEYS, diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb index 3a65af68695..f22cc6d2845 100644 --- a/app/helpers/languages_helper.rb +++ b/app/helpers/languages_helper.rb @@ -242,6 +242,6 @@ module LanguagesHelper end def valid_locale?(locale) - SUPPORTED_LOCALES.key?(locale.to_sym) + locale.present? && SUPPORTED_LOCALES.key?(locale.to_sym) end end diff --git a/app/javascript/styles/mastodon/accounts.scss b/app/javascript/styles/mastodon/accounts.scss index 485fe4a9dc1..215774a192f 100644 --- a/app/javascript/styles/mastodon/accounts.scss +++ b/app/javascript/styles/mastodon/accounts.scss @@ -331,7 +331,8 @@ } .batch-table__row--muted .pending-account__header, -.batch-table__row--muted .accounts-table { +.batch-table__row--muted .accounts-table, +.batch-table__row--muted .name-tag { &, a, strong { @@ -339,6 +340,10 @@ } } +.batch-table__row--muted .name-tag .avatar { + opacity: 0.5; +} + .batch-table__row--muted .accounts-table { tbody td.accounts-table__extra, &__count, @@ -352,7 +357,8 @@ } .batch-table__row--attention .pending-account__header, -.batch-table__row--attention .accounts-table { +.batch-table__row--attention .accounts-table, +.batch-table__row--attention .name-tag { &, a, strong { diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss index 36bc07a72b1..1f7e71776e4 100644 --- a/app/javascript/styles/mastodon/tables.scss +++ b/app/javascript/styles/mastodon/tables.scss @@ -210,6 +210,7 @@ a.table-action-link { &__content { padding-top: 12px; padding-bottom: 16px; + overflow: hidden; &--unpadded { padding: 0; @@ -296,3 +297,9 @@ a.table-action-link { } } } + +.one-liner { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 12fad8da407..7cd5a41e8e9 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -23,8 +23,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity visibility: visibility_from_audience ) - Trends.tags.register(@status) - Trends.links.register(@status) + Trends.register!(@status) distribute end diff --git a/app/lib/activitypub/activity/like.rb b/app/lib/activitypub/activity/like.rb index c065f01f860..ebbda15b9f6 100644 --- a/app/lib/activitypub/activity/like.rb +++ b/app/lib/activitypub/activity/like.rb @@ -7,6 +7,8 @@ class ActivityPub::Activity::Like < ActivityPub::Activity return if original_status.nil? || !original_status.account.local? || delete_arrived_first?(@json['id']) || @account.favourited?(original_status) favourite = original_status.favourites.create!(account: @account) + NotifyService.new.call(original_status.account, :favourite, favourite) + Trends.statuses.register(original_status) end end diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index a9d00c000dd..f416977d850 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -35,25 +35,18 @@ class AdminMailer < ApplicationMailer end end - def new_trending_tags(recipient, tags) - @tags = tags - @me = recipient - @instance = Rails.configuration.x.local_domain - @lowest_trending_tag = Trends.tags.get(true, Trends.tags.options[:review_threshold]).last + def new_trends(recipient, links, tags, statuses) + @links = links + @lowest_trending_link = Trends.links.query.allowed.limit(Trends.links.options[:review_threshold]).last + @tags = tags + @lowest_trending_tag = Trends.tags.query.allowed.limit(Trends.tags.options[:review_threshold]).last + @statuses = statuses + @lowest_trending_status = Trends.statuses.query.allowed.limit(Trends.statuses.options[:review_threshold]).last + @me = recipient + @instance = Rails.configuration.x.local_domain locale_for_account(@me) do - mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance) - end - end - - def new_trending_links(recipient, links) - @links = links - @me = recipient - @instance = Rails.configuration.x.local_domain - @lowest_trending_link = Trends.links.get(true, Trends.links.options[:review_threshold]).last - - locale_for_account(@me) do - mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance) + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance) end end end diff --git a/app/models/account.rb b/app/models/account.rb index 2ad45feda09..dfdf9045ff9 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -40,13 +40,15 @@ # also_known_as :string is an Array # silenced_at :datetime # suspended_at :datetime -# trust_level :integer # hide_collections :boolean # avatar_storage_schema_version :integer # header_storage_schema_version :integer # devices_url :string # suspension_origin :integer # sensitized_at :datetime +# trendable :boolean +# reviewed_at :datetime +# requested_review_at :datetime # class Account < ApplicationRecord @@ -56,6 +58,7 @@ class Account < ApplicationRecord remote_url salmon_url hub_url + trust_level ) USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i @@ -74,11 +77,6 @@ class Account < ApplicationRecord include DomainMaterializable include AccountMerging - TRUST_LEVELS = { - untrusted: 0, - trusted: 1, - }.freeze - enum protocol: [:ostatus, :activitypub] enum suspension_origin: [:local, :remote], _prefix: true @@ -202,10 +200,6 @@ class Account < ApplicationRecord last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago end - def trust_level - self[:trust_level] || 0 - end - def refresh! ResolveAccountService.new.call(acct) unless local? end @@ -388,6 +382,22 @@ class Account < ApplicationRecord @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/" end + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def requires_review_notification? + requires_review? && !requested_review? + end + class Field < ActiveModelSerializers::Model attributes :name, :value, :verified_at, :account diff --git a/app/models/status.rb b/app/models/status.rb index 96e41b1d328..adb92ef9164 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -268,6 +268,18 @@ class Status < ApplicationRecord update_status_stat!(key => [public_send(key) - 1, 0].max) end + def trendable? + if attributes['trendable'].nil? + account.trendable? + else + attributes['trendable'] + end + end + + def requires_review_notification? + attributes['trendable'].nil? && account.requires_review_notification? + end + after_create_commit :increment_counter_caches after_destroy_commit :decrement_counter_caches diff --git a/app/models/trends.rb b/app/models/trends.rb index 7dd3a9c87ea..f8864e55f55 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -13,15 +13,37 @@ module Trends @tags ||= Trends::Tags.new end + def self.statuses + @statuses ||= Trends::Statuses.new + end + + def self.register!(status) + [links, tags, statuses].each { |trend_type| trend_type.register(status) } + end + def self.refresh! - [links, tags].each(&:refresh) + [links, tags, statuses].each(&:refresh) end def self.request_review! - [links, tags].each(&:request_review) if enabled? + return unless enabled? + + links_requiring_review = links.request_review + tags_requiring_review = tags.request_review + statuses_requiring_review = statuses.request_review + + return if links_requiring_review.empty? && tags_requiring_review.empty? && statuses_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trends(user.account, links_requiring_review, tags_requiring_review, statuses_requiring_review).deliver_later! if user.allows_trends_review_emails? + end end def self.enabled? Setting.trends end + + def self.available_locales + @available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq + end end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index b767dcb1aca..7ed13228dfd 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -2,6 +2,7 @@ class Trends::Base include Redisable + include LanguagesHelper class_attribute :default_options @@ -32,8 +33,8 @@ class Trends::Base raise NotImplementedError end - def get(*) - raise NotImplementedError + def query + Trends::Query.new(key_prefix, klass) end def score(id) @@ -72,6 +73,21 @@ class Trends::Base redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 end + # @param [Integer] id + # @param [Float] score + # @param [Hash] subsets + def add_to_and_remove_from_subsets(id, score, subsets = {}) + subsets.each_key do |subset| + key = [key_prefix, subset].compact.join(':') + + if score.positive? && subsets[subset] + redis.zadd(key, score, id) + else + redis.zrem(key, id) + end + end + end + private def used_key(at_time) diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index a0d65138bae..62308e7067b 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -4,8 +4,8 @@ class Trends::Links < Trends::Base PREFIX = 'trending_links' self.default_options = { - threshold: 15, - review_threshold: 10, + threshold: 5, + review_threshold: 3, max_score_cooldown: 2.days.freeze, max_score_halflife: 8.hours.freeze, } @@ -27,12 +27,6 @@ class Trends::Links < Trends::Base record_used_id(preview_card.id, at_time) end - def get(allowed, limit) - preview_card_ids = currently_trending_ids(allowed, limit) - preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) - preview_card_ids.map { |id| preview_cards[id] }.compact - end - def refresh(at_time = Time.now.utc) preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) calculate_scores(preview_cards, at_time) @@ -42,7 +36,7 @@ class Trends::Links < Trends::Base def request_review preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) - preview_cards_requiring_review = preview_cards.filter_map do |preview_card| + preview_cards.filter_map do |preview_card| next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? if preview_card.provider.nil? @@ -53,12 +47,6 @@ class Trends::Links < Trends::Base preview_card end - - return if preview_cards_requiring_review.empty? - - User.staff.includes(:account).find_each do |user| - AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? - end end protected @@ -67,6 +55,10 @@ class Trends::Links < Trends::Base PREFIX end + def klass + PreviewCard + end + private def calculate_scores(preview_cards, at_time) @@ -96,17 +88,27 @@ class Trends::Links < Trends::Base decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - if decaying_score.zero? - redis.zrem("#{PREFIX}:all", preview_card.id) - redis.zrem("#{PREFIX}:allowed", preview_card.id) - else - redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) + add_to_and_remove_from_subsets(preview_card.id, decaying_score, { + all: true, + allowed: preview_card.trendable?, + }) - if preview_card.trendable? - redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) - else - redis.zrem("#{PREFIX}:allowed", preview_card.id) - end + next unless valid_locale?(preview_card.language) + + add_to_and_remove_from_subsets(preview_card.id, decaying_score, { + "all:#{preview_card.language}" => true, + "allowed:#{preview_card.language}" => preview_card.trendable?, + }) + end + + # Clean up localized sets by calculating the intersection with the main + # set. We do this instead of just deleting the localized sets to avoid + # having moments where the API returns empty results + + redis.pipelined do + Trends.available_locales.each do |locale| + redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') + redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max') end end end diff --git a/app/models/form/preview_card_batch.rb b/app/models/trends/preview_card_batch.rb similarity index 82% rename from app/models/form/preview_card_batch.rb rename to app/models/trends/preview_card_batch.rb index 5f6e6522a26..b1d6829101d 100644 --- a/app/models/form/preview_card_batch.rb +++ b/app/models/trends/preview_card_batch.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Form::PreviewCardBatch +class Trends::PreviewCardBatch include ActiveModel::Model include Authorization @@ -10,12 +10,12 @@ class Form::PreviewCardBatch case action when 'approve' approve! - when 'approve_all' - approve_all! + when 'approve_providers' + approve_providers! when 'reject' reject! - when 'reject_all' - reject_all! + when 'reject_providers' + reject_providers! end end @@ -30,13 +30,13 @@ class Form::PreviewCardBatch end def approve! - preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.each { |preview_card| authorize(preview_card, :review?) } preview_cards.update_all(trendable: true) end - def approve_all! + def approve_providers! preview_card_providers.each do |provider| - authorize(provider, :update?) + authorize(provider, :review?) provider.update(trendable: true, reviewed_at: action_time) end @@ -45,13 +45,13 @@ class Form::PreviewCardBatch end def reject! - preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.each { |preview_card| authorize(preview_card, :review?) } preview_cards.update_all(trendable: false) end - def reject_all! + def reject_providers! preview_card_providers.each do |provider| - authorize(provider, :update?) + authorize(provider, :review?) provider.update(trendable: false, reviewed_at: action_time) end diff --git a/app/models/preview_card_filter.rb b/app/models/trends/preview_card_filter.rb similarity index 52% rename from app/models/preview_card_filter.rb rename to app/models/trends/preview_card_filter.rb index 8dda9989cea..25add58c896 100644 --- a/app/models/preview_card_filter.rb +++ b/app/models/trends/preview_card_filter.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true -class PreviewCardFilter +class Trends::PreviewCardFilter KEYS = %i( trending + locale ).freeze attr_reader :params @@ -15,7 +16,7 @@ class PreviewCardFilter scope = PreviewCard.unscoped params.each do |key, value| - next if key.to_s == 'page' + next if %w(page locale).include?(key.to_s) scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end @@ -35,19 +36,11 @@ class PreviewCardFilter end def trending_scope(value) - ids = begin - case value.to_s - when 'allowed' - Trends.links.currently_trending_ids(true, -1) - else - Trends.links.currently_trending_ids(false, -1) - end - end + scope = Trends.links.query - if ids.empty? - PreviewCard.none - else - PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') - end + scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? + scope = scope.allowed if value == 'allowed' + + scope.to_arel end end diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/trends/preview_card_provider_batch.rb similarity index 91% rename from app/models/form/preview_card_provider_batch.rb rename to app/models/trends/preview_card_provider_batch.rb index e6ab3d8fa39..062720c814e 100644 --- a/app/models/form/preview_card_provider_batch.rb +++ b/app/models/trends/preview_card_provider_batch.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Form::PreviewCardProviderBatch +class Trends::PreviewCardProviderBatch include ActiveModel::Model include Authorization @@ -22,12 +22,12 @@ class Form::PreviewCardProviderBatch end def approve! - preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.each { |provider| authorize(provider, :review?) } preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) end def reject! - preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.each { |provider| authorize(provider, :review?) } preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) end end diff --git a/app/models/preview_card_provider_filter.rb b/app/models/trends/preview_card_provider_filter.rb similarity index 95% rename from app/models/preview_card_provider_filter.rb rename to app/models/trends/preview_card_provider_filter.rb index 1e90d3c9daa..abfdd07e887 100644 --- a/app/models/preview_card_provider_filter.rb +++ b/app/models/trends/preview_card_provider_filter.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class PreviewCardProviderFilter +class Trends::PreviewCardProviderFilter KEYS = %i( status ).freeze diff --git a/app/models/trends/query.rb b/app/models/trends/query.rb new file mode 100644 index 00000000000..64a4c0c1fb5 --- /dev/null +++ b/app/models/trends/query.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +class Trends::Query + include Redisable + include Enumerable + + attr_reader :prefix, :klass, :loaded + + alias loaded? loaded + + def initialize(prefix, klass) + @prefix = prefix + @klass = klass + @records = [] + @loaded = false + @allowed = false + @limit = -1 + @offset = 0 + end + + def allowed! + @allowed = true + self + end + + def allowed + clone.allowed! + end + + def in_locale!(value) + @locale = value + self + end + + def in_locale(value) + clone.in_locale!(value) + end + + def offset!(value) + @offset = value + self + end + + def offset(value) + clone.offset!(value) + end + + def limit!(value) + @limit = value + self + end + + def limit(value) + clone.limit!(value) + end + + def records + load + @records + end + + delegate :each, :empty?, :first, :last, to: :records + + def to_ary + records.dup + end + + alias to_a to_ary + + def to_arel + tmp_ids = ids + + if tmp_ids.empty? + klass.none + else + klass.joins("join unnest(array[#{tmp_ids.join(',')}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id").reorder('x.ordering') + end + end + + private + + def key + [@prefix, @allowed ? 'allowed' : 'all', @locale].compact.join(':') + end + + def load + unless loaded? + @records = perform_queries + @loaded = true + end + + self + end + + def ids + redis.zrevrange(key, @offset, @limit.positive? ? @limit - 1 : @limit).map(&:to_i) + end + + def perform_queries + apply_scopes(to_arel).to_a + end + + def apply_scopes(scope) + scope + end +end diff --git a/app/models/trends/status_batch.rb b/app/models/trends/status_batch.rb new file mode 100644 index 00000000000..78d93bed445 --- /dev/null +++ b/app/models/trends/status_batch.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Trends::StatusBatch + include ActiveModel::Model + include Authorization + + attr_accessor :status_ids, :action, :current_account + + def save + case action + when 'approve' + approve! + when 'approve_accounts' + approve_accounts! + when 'reject' + reject! + when 'reject_accounts' + reject_accounts! + end + end + + private + + def statuses + @statuses ||= Status.where(id: status_ids) + end + + def status_accounts + @status_accounts ||= Account.where(id: statuses.map(&:account_id).uniq) + end + + def approve! + statuses.each { |status| authorize(status, :review?) } + statuses.update_all(trendable: true) + end + + def approve_accounts! + status_accounts.each do |account| + authorize(account, :review?) + account.update(trendable: true, reviewed_at: action_time) + end + + # Reset any individual overrides + statuses.update_all(trendable: nil) + end + + def reject! + statuses.each { |status| authorize(status, :review?) } + statuses.update_all(trendable: false) + end + + def reject_accounts! + status_accounts.each do |account| + authorize(account, :review?) + account.update(trendable: false, reviewed_at: action_time) + end + + # Reset any individual overrides + statuses.update_all(trendable: nil) + end + + def action_time + @action_time ||= Time.now.utc + end +end diff --git a/app/models/trends/status_filter.rb b/app/models/trends/status_filter.rb new file mode 100644 index 00000000000..7c453e33938 --- /dev/null +++ b/app/models/trends/status_filter.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class Trends::StatusFilter + KEYS = %i( + trending + locale + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = Status.unscoped.kept + + params.each do |key, value| + next if %w(page locale).include?(key.to_s) + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'trending' + trending_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def trending_scope(value) + scope = Trends.statuses.query + + scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? + scope = scope.allowed if value == 'allowed' + + scope.to_arel + end +end diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb new file mode 100644 index 00000000000..e785413ec47 --- /dev/null +++ b/app/models/trends/statuses.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +class Trends::Statuses < Trends::Base + PREFIX = 'trending_statuses' + + self.default_options = { + threshold: 5, + review_threshold: 3, + score_halflife: 2.hours.freeze, + } + + class Query < Trends::Query + def filtered_for!(account) + @account = account + self + end + + def filtered_for(account) + clone.filtered_for!(account) + end + + private + + def apply_scopes(scope) + scope.includes(:account) + end + + def perform_queries + return super if @account.nil? + + statuses = super + account_ids = statuses.map(&:account_id) + account_domains = statuses.map(&:account_domain) + + preloaded_relations = { + blocking: Account.blocking_map(account_ids, @account.id), + blocked_by: Account.blocked_by_map(account_ids, @account.id), + muting: Account.muting_map(account_ids, @account.id), + following: Account.following_map(account_ids, @account.id), + domain_blocking_by_domain: Account.domain_blocking_map_by_domain(account_domains, @account.id), + } + + statuses.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? } + end + end + + def register(status, at_time = Time.now.utc) + add(status.proper, status.account_id, at_time) if eligible?(status) + end + + def add(status, _account_id, at_time = Time.now.utc) + # We rely on the total reblogs and favourites count, so we + # don't record which account did the what and when here + + record_used_id(status.id, at_time) + end + + def query + Query.new(key_prefix, klass) + end + + def refresh(at_time = Time.now.utc) + statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments) + calculate_scores(statuses, at_time) + trim_older_items + end + + def request_review + statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account) + + statuses.filter_map do |status| + next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification? + + status.account.touch(:requested_review_at) + status + end + end + + protected + + def key_prefix + PREFIX + end + + def klass + Status + end + + private + + def eligible?(status) + original_status = status.proper + + original_status.public_visibility? && + original_status.account.discoverable? && !original_status.account.silenced? && + original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply? + end + + def calculate_scores(statuses, at_time) + redis.pipelined do + statuses.each do |status| + expected = 1.0 + observed = (status.reblogs_count + status.favourites_count).to_f + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) + + add_to_and_remove_from_subsets(status.id, decaying_score, { + all: true, + allowed: status.trendable? && status.account.discoverable?, + }) + + next unless valid_locale?(status.language) + + add_to_and_remove_from_subsets(status.id, decaying_score, { + "all:#{status.language}" => true, + "allowed:#{status.language}" => status.trendable? && status.account.discoverable?, + }) + end + + # Clean up localized sets by calculating the intersection with the main + # set. We do this instead of just deleting the localized sets to avoid + # having moments where the API returns empty results + + Trends.available_locales.each do |locale| + redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') + redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:all"], aggregate: 'max') + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/form/tag_batch.rb b/app/models/trends/tag_batch.rb similarity index 81% rename from app/models/form/tag_batch.rb rename to app/models/trends/tag_batch.rb index b9330745fed..16ee08c065e 100644 --- a/app/models/form/tag_batch.rb +++ b/app/models/trends/tag_batch.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Form::TagBatch +class Trends::TagBatch include ActiveModel::Model include Authorization @@ -22,12 +22,12 @@ class Form::TagBatch end def approve! - tags.each { |tag| authorize(tag, :update?) } + tags.each { |tag| authorize(tag, :review?) } tags.update_all(trendable: true, reviewed_at: action_time) end def reject! - tags.each { |tag| authorize(tag, :update?) } + tags.each { |tag| authorize(tag, :review?) } tags.update_all(trendable: false, reviewed_at: action_time) end diff --git a/app/models/tag_filter.rb b/app/models/trends/tag_filter.rb similarity index 76% rename from app/models/tag_filter.rb rename to app/models/trends/tag_filter.rb index ecdb52503e2..3b142efc450 100644 --- a/app/models/tag_filter.rb +++ b/app/models/trends/tag_filter.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class TagFilter +class Trends::TagFilter KEYS = %i( trending status @@ -42,13 +42,7 @@ class TagFilter end def trending_scope - ids = Trends.tags.currently_trending_ids(false, -1) - - if ids.empty? - Tag.none - else - Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering') - end + Trends.tags.query.to_arel end def status_scope(value) diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb index 2ea4550df6a..3caa58815d2 100644 --- a/app/models/trends/tags.rb +++ b/app/models/trends/tags.rb @@ -5,7 +5,7 @@ class Trends::Tags < Trends::Base self.default_options = { threshold: 5, - review_threshold: 10, + review_threshold: 3, max_score_cooldown: 2.days.freeze, max_score_halflife: 4.hours.freeze, } @@ -29,27 +29,15 @@ class Trends::Tags < Trends::Base trim_older_items end - def get(allowed, limit) - tag_ids = currently_trending_ids(allowed, limit) - tags = Tag.where(id: tag_ids).index_by(&:id) - tag_ids.map { |id| tags[id] }.compact - end - def request_review tags = Tag.where(id: currently_trending_ids(false, -1)) - tags_requiring_review = tags.filter_map do |tag| + tags.filter_map do |tag| next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? tag.touch(:requested_review_at) tag end - - return if tags_requiring_review.empty? - - User.staff.includes(:account).find_each do |user| - AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? - end end protected @@ -58,6 +46,10 @@ class Trends::Tags < Trends::Base PREFIX end + def klass + Tag + end + private def calculate_scores(tags, at_time) @@ -87,18 +79,10 @@ class Trends::Tags < Trends::Base decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - if decaying_score.zero? - redis.zrem("#{PREFIX}:all", tag.id) - redis.zrem("#{PREFIX}:allowed", tag.id) - else - redis.zadd("#{PREFIX}:all", decaying_score, tag.id) - - if tag.trendable? - redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) - else - redis.zrem("#{PREFIX}:allowed", tag.id) - end - end + add_to_and_remove_from_subsets(tag.id, decaying_score, { + all: true, + allowed: tag.trendable?, + }) end end diff --git a/app/models/user.rb b/app/models/user.rb index 517254a910d..bbf850d84db 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -269,7 +269,7 @@ class User < ApplicationRecord settings.notification_emails['appeal'] end - def allows_trending_tag_emails? + def allows_trends_review_emails? settings.notification_emails['trending_tag'] end diff --git a/app/policies/account_policy.rb b/app/policies/account_policy.rb index 46237e45c10..cc23771e7a9 100644 --- a/app/policies/account_policy.rb +++ b/app/policies/account_policy.rb @@ -68,4 +68,8 @@ class AccountPolicy < ApplicationPolicy def unblock_email? staff? end + + def review? + staff? + end end diff --git a/app/policies/preview_card_policy.rb b/app/policies/preview_card_policy.rb index 4f485d7fc61..0410987e46c 100644 --- a/app/policies/preview_card_policy.rb +++ b/app/policies/preview_card_policy.rb @@ -5,7 +5,7 @@ class PreviewCardPolicy < ApplicationPolicy staff? end - def update? + def review? staff? end end diff --git a/app/policies/preview_card_provider_policy.rb b/app/policies/preview_card_provider_policy.rb index 598d54a5e28..44d2ad5cfc5 100644 --- a/app/policies/preview_card_provider_policy.rb +++ b/app/policies/preview_card_provider_policy.rb @@ -5,7 +5,7 @@ class PreviewCardProviderPolicy < ApplicationPolicy staff? end - def update? + def review? staff? end end diff --git a/app/policies/status_policy.rb b/app/policies/status_policy.rb index 6e9b840db6b..400f1ec7910 100644 --- a/app/policies/status_policy.rb +++ b/app/policies/status_policy.rb @@ -41,6 +41,10 @@ class StatusPolicy < ApplicationPolicy staff? || owned? end + def review? + staff? + end + private def requires_mention? diff --git a/app/policies/tag_policy.rb b/app/policies/tag_policy.rb index aaf70fcabc4..bdfcec0c96c 100644 --- a/app/policies/tag_policy.rb +++ b/app/policies/tag_policy.rb @@ -12,4 +12,8 @@ class TagPolicy < ApplicationPolicy def update? staff? end + + def review? + staff? + end end diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index a572a7c597d..a2d535d2627 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -220,21 +220,23 @@ class DeleteAccountService < BaseService return unless keep_account_record? - @account.silenced_at = nil - @account.suspended_at = @options[:suspended_at] || Time.now.utc - @account.suspension_origin = :local - @account.locked = false - @account.memorial = false - @account.discoverable = false - @account.display_name = '' - @account.note = '' - @account.fields = [] - @account.statuses_count = 0 - @account.followers_count = 0 - @account.following_count = 0 - @account.moved_to_account = nil - @account.also_known_as = [] - @account.trust_level = :untrusted + @account.silenced_at = nil + @account.suspended_at = @options[:suspended_at] || Time.now.utc + @account.suspension_origin = :local + @account.locked = false + @account.memorial = false + @account.discoverable = false + @account.trendable = false + @account.display_name = '' + @account.note = '' + @account.fields = [] + @account.statuses_count = 0 + @account.followers_count = 0 + @account.following_count = 0 + @account.moved_to_account = nil + @account.reviewed_at = nil + @account.requested_review_at = nil + @account.also_known_as = [] @account.avatar.destroy @account.header.destroy @account.save! diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb index a0ab3b4b755..0ca0081b48a 100644 --- a/app/services/favourite_service.rb +++ b/app/services/favourite_service.rb @@ -17,6 +17,8 @@ class FavouriteService < BaseService favourite = Favourite.create!(account: account, status: status) + Trends.statuses.register(status) + create_notification(favourite) bump_potential_friendship(account, status) diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 2d1265f1072..7d2981709b2 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -30,8 +30,7 @@ class ReblogService < BaseService reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit]) - Trends.tags.register(reblog) - Trends.links.register(reblog) + Trends.register!(reblog) DistributionWorker.perform_async(reblog.id) ActivityPub::DistributionWorker.perform_async(reblog.id) diff --git a/app/views/admin/custom_emojis/_custom_emoji.html.haml b/app/views/admin/custom_emojis/_custom_emoji.html.haml index 526c844e925..41f3975cf45 100644 --- a/app/views/admin/custom_emojis/_custom_emoji.html.haml +++ b/app/views/admin/custom_emojis/_custom_emoji.html.haml @@ -3,7 +3,7 @@ = f.check_box :custom_emoji_ids, { multiple: true, include_hidden: false }, custom_emoji.id .batch-table__row__content.batch-table__row__content--with-image .batch-table__row__content__image - = custom_emoji_tag(custom_emoji, animate = current_account&.user&.setting_auto_play_gif) + = custom_emoji_tag(custom_emoji, current_account&.user&.setting_auto_play_gif) .batch-table__row__content__text %samp= ":#{custom_emoji.shortcode}:" diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml index 272681864a9..ebc4a2c6b86 100644 --- a/app/views/admin/follow_recommendations/show.html.haml +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -9,12 +9,14 @@ %hr.spacer/ = form_tag admin_follow_recommendations_path, method: 'GET', class: 'simple_form' do + - RelationshipFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + .filters .filter-subset.filter-subset--with-select %strong= t('admin.follow_recommendations.language') .input.select.optional - = select_tag :language, options_for_select(I18n.available_locales.map { |key| key.to_s.split(/[_-]/).first.to_sym }.uniq.map { |key| [standard_locale_name(key), key]}, @language) - + = select_tag :language, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, @language) .filter-subset %strong= t('admin.follow_recommendations.status') %ul diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml index 240ae722b58..79f3513d318 100644 --- a/app/views/admin/trends/links/index.html.haml +++ b/app/views/admin/trends/links/index.html.haml @@ -4,23 +4,29 @@ - content_for :header_tags do = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' -.filters - .filter-subset - %strong= t('admin.trends.trending') - %ul - %li= filter_link_to t('generic.all'), trending: nil - %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' - .back-link - = link_to admin_trends_links_preview_card_providers_path do - = t('admin.trends.preview_card_providers.title') - = fa_icon 'chevron-right fw' += form_tag admin_trends_links_path, method: 'GET', class: 'simple_form' do + - Trends::PreviewCardFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? -%hr.spacer/ + .filters + .filter-subset.filter-subset--with-select + %strong= t('admin.follow_recommendations.language') + .input.select.optional + = select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key] }, params[:locale]), include_blank: true + .filter-subset + %strong= t('admin.trends.trending') + %ul + %li= filter_link_to t('generic.all'), trending: nil + %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' + .back-link + = link_to admin_trends_links_preview_card_providers_path do + = t('admin.trends.preview_card_providers.title') + = fa_icon 'chevron-right fw' = form_for(@form, url: batch_admin_trends_links_path) do |f| = hidden_field_tag :page, params[:page] || 1 - - PreviewCardFilter::KEYS.each do |key| + - Trends::PreviewCardFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? .batch-table @@ -29,9 +35,9 @@ = check_box_tag :batch_checkbox_all, nil, false .batch-table__toolbar__actions = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_providers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_providers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } .batch-table__body - if @preview_cards.empty? = nothing_here 'nothing-here--under-tabs' diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml index eac6e641fe5..b793499477b 100644 --- a/app/views/admin/trends/links/preview_card_providers/index.html.haml +++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml @@ -23,7 +23,7 @@ = form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f| = hidden_field_tag :page, params[:page] || 1 - - PreviewCardProviderFilter::KEYS.each do |key| + - Trends::PreviewCardProviderFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? .batch-table.optional diff --git a/app/views/admin/trends/statuses/_status.html.haml b/app/views/admin/trends/statuses/_status.html.haml new file mode 100644 index 00000000000..c99ee5d60ee --- /dev/null +++ b/app/views/admin/trends/statuses/_status.html.haml @@ -0,0 +1,30 @@ +.batch-table__row{ class: [status.account.requires_review? && 'batch-table__row--attention', !status.account.requires_review? && !status.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :status_ids, { multiple: true, include_hidden: false }, status.id + + .batch-table__row__content.pending-account__header + .one-liner + = admin_account_link_to status.account + + = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank', class: 'emojify', rel: 'noopener noreferrer' do + = one_line_preview(status) + + - status.media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name + + = t('admin.trends.statuses.shared_by', count: status.reblogs_count + status.favourites_count, friendly_count: friendly_number_to_human(status.reblogs_count + status.favourites_count)) + + - if status.account.domain.present? + • + = status.account.domain + - if status.language.present? + • + = standard_locale_name(status.language) + - if status.trendable? && (rank = Trends.statuses.rank(status.id)) + • + %abbr{ title: t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) + - elsif status.account.requires_review? + • + = t('admin.trends.pending_review') diff --git a/app/views/admin/trends/statuses/index.html.haml b/app/views/admin/trends/statuses/index.html.haml new file mode 100644 index 00000000000..34768826286 --- /dev/null +++ b/app/views/admin/trends/statuses/index.html.haml @@ -0,0 +1,43 @@ +- content_for :page_title do + = t('admin.trends.statuses.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + += form_tag admin_trends_statuses_path, method: 'GET', class: 'simple_form' do + - Trends::StatusFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .filters + .filter-subset.filter-subset--with-select + %strong= t('admin.follow_recommendations.language') + .input.select.optional + = select_tag :locale, options_for_select(Trends.available_locales.map { |key| [standard_locale_name(key), key]}, params[:locale]), include_blank: true + .filter-subset + %strong= t('admin.trends.trending') + %ul + %li= filter_link_to t('generic.all'), trending: nil + %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' + += form_for(@form, url: batch_admin_trends_statuses_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - Trends::StatusFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.statuses.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('check'), t('admin.trends.statuses.allow_account')]), name: :approve_accounts, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.statuses.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.statuses.disallow_account')]), name: :reject_accounts, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @statuses.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'status', collection: @statuses, locals: { f: f } + += paginate @statuses diff --git a/app/views/admin/trends/tags/index.html.haml b/app/views/admin/trends/tags/index.html.haml index 8df0a9920b4..8a2f785bc21 100644 --- a/app/views/admin/trends/tags/index.html.haml +++ b/app/views/admin/trends/tags/index.html.haml @@ -13,12 +13,10 @@ %li= filter_link_to t('admin.trends.rejected'), status: 'rejected' %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review' -%hr.spacer/ - = form_for(@form, url: batch_admin_trends_tags_path) do |f| = hidden_field_tag :page, params[:page] || 1 - - TagFilter::KEYS.each do |key| + - Trends::TagFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? .batch-table.optional diff --git a/app/views/admin_mailer/_new_trending_links.text.erb b/app/views/admin_mailer/_new_trending_links.text.erb new file mode 100644 index 00000000000..405926fddc3 --- /dev/null +++ b/app/views/admin_mailer/_new_trending_links.text.erb @@ -0,0 +1,14 @@ +<%= raw t('admin_mailer.new_trends.new_trending_links.title') %> + +<% @links.each do |link| %> +- <%= link.title %> • <%= link.url %> + <%= raw t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> +<% end %> + +<% if @lowest_trending_link %> +<%= raw t('admin_mailer.new_trends.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2), rank: Trends.links.options[:review_threshold]) %> +<% else %> +<%= raw t('admin_mailer.new_trends.new_trending_links.no_approved_links') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> diff --git a/app/views/admin_mailer/_new_trending_statuses.text.erb b/app/views/admin_mailer/_new_trending_statuses.text.erb new file mode 100644 index 00000000000..8d11a80c2ec --- /dev/null +++ b/app/views/admin_mailer/_new_trending_statuses.text.erb @@ -0,0 +1,14 @@ +<%= raw t('admin_mailer.new_trends.new_trending_statuses.title') %> + +<% @statuses.each do |status| %> +- <%= ActivityPub::TagManager.instance.url_for(status) %> + <%= raw t('admin.trends.tags.current_score', score: Trends.statuses.score(status.id).round(2)) %> +<% end %> + +<% if @lowest_trending_status %> +<%= raw t('admin_mailer.new_trends.new_trending_statuses.requirements', lowest_status_url: ActivityPub::TagManager.instance.url_for(@lowest_trending_status), lowest_status_score: Trends.statuses.score(@lowest_trending_status.id).round(2), rank: Trends.statuses.options[:review_threshold]) %> +<% else %> +<%= raw t('admin_mailer.new_trends.new_trending_statuses.no_approved_statuses') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_statuses_url %> diff --git a/app/views/admin_mailer/_new_trending_tags.text.erb b/app/views/admin_mailer/_new_trending_tags.text.erb new file mode 100644 index 00000000000..49fe8430960 --- /dev/null +++ b/app/views/admin_mailer/_new_trending_tags.text.erb @@ -0,0 +1,14 @@ +<%= raw t('admin_mailer.new_trends.new_trending_tags.title') %> + +<% @tags.each do |tag| %> +- #<%= tag.name %> + <%= raw t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> +<% end %> + +<% if @lowest_trending_tag %> +<%= raw t('admin_mailer.new_trends.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2), rank: Trends.tags.options[:review_threshold]) %> +<% else %> +<%= raw t('admin_mailer.new_trends.new_trending_tags.no_approved_tags') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %> diff --git a/app/views/admin_mailer/new_trending_links.text.erb b/app/views/admin_mailer/new_trending_links.text.erb deleted file mode 100644 index 51789aca5cf..00000000000 --- a/app/views/admin_mailer/new_trending_links.text.erb +++ /dev/null @@ -1,16 +0,0 @@ -<%= raw t('application_mailer.salutation', name: display_name(@me)) %> - -<%= raw t('admin_mailer.new_trending_links.body') %> - -<% @links.each do |link| %> -- <%= link.title %> • <%= link.url %> - <%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> -<% end %> - -<% if @lowest_trending_link %> -<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %> -<% else %> -<%= t('admin_mailer.new_trending_links.no_approved_links') %> -<% end %> - -<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> diff --git a/app/views/admin_mailer/new_trending_tags.text.erb b/app/views/admin_mailer/new_trending_tags.text.erb deleted file mode 100644 index 9ea31fa7ce5..00000000000 --- a/app/views/admin_mailer/new_trending_tags.text.erb +++ /dev/null @@ -1,16 +0,0 @@ -<%= raw t('application_mailer.salutation', name: display_name(@me)) %> - -<%= raw t('admin_mailer.new_trending_tags.body') %> - -<% @tags.each do |tag| %> -- #<%= tag.name %> - <%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> -<% end %> - -<% if @lowest_trending_tag %> -<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %> -<% else %> -<%= t('admin_mailer.new_trending_tags.no_approved_tags') %> -<% end %> - -<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(status: 'pending_review') %> diff --git a/app/views/admin_mailer/new_trends.text.erb b/app/views/admin_mailer/new_trends.text.erb new file mode 100644 index 00000000000..13b2968461e --- /dev/null +++ b/app/views/admin_mailer/new_trends.text.erb @@ -0,0 +1,13 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trends.body') %> + +<% unless @links.empty? %> +<%= render 'new_trending_links' %> +<% end %> +<% unless @tags.empty? %> +<%= render 'new_trending_tags' unless @tags.empty? %> +<% end %> +<% unless @statuses.empty? %> +<%= render 'new_trending_statuses' unless @statuses.empty? %> +<% end %> diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml index 6826c3b5876..e97c493fe86 100644 --- a/app/views/application/_sidebar.html.haml +++ b/app/views/application/_sidebar.html.haml @@ -6,7 +6,7 @@ %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html') - if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - - trends = Trends.tags.get(true, 3) + - trends = Trends.tags.query.allowed.limit(3) - unless trends.empty? .endorsements-widget.trends-widget diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb index 084619cbd93..57f78170e48 100644 --- a/app/workers/scheduler/follow_recommendations_scheduler.rb +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -18,7 +18,7 @@ class Scheduler::FollowRecommendationsScheduler fallback_recommendations = FollowRecommendation.order(rank: :desc).limit(SET_SIZE) - I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq.each do |locale| + Trends.available_locales.each do |locale| recommendations = begin if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist FollowRecommendation.localized(locale).order(rank: :desc).limit(SET_SIZE).map { |recommendation| [recommendation.account_id, recommendation.rank] } @@ -49,11 +49,11 @@ class Scheduler::FollowRecommendationsScheduler end end - redis.pipelined do - redis.del(key(locale)) + redis.multi do |multi| + multi.del(key(locale)) recommendations.each do |(account_id, rank)| - redis.zadd(key(locale), rank, account_id) + multi.zadd(key(locale), rank, account_id) end end end diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 6ffe12ae06f..c24146da415 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -7,7 +7,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/status.rb", - "line": 104, + "line": 105, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "result.joins(\"INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}\")", "render_path": null, @@ -20,6 +20,26 @@ "confidence": "Weak", "note": "" }, + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "30dfe36e87fe1b8f239df9a33d576e44a9863f73b680198d4713be6540ae61d3", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/trends/query.rb", + "line": 60, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "klass.joins(\"join unnest(array[#{ids.join(\",\")}]) with ordinality as x (id, ordering) on #{klass.table_name}.id = x.id\")", + "render_path": null, + "location": { + "type": "method", + "class": "Trends::Query", + "method": "to_arel" + }, + "user_input": "ids.join(\",\")", + "confidence": "Weak", + "note": "" + }, { "warning_type": "Redirect", "warning_code": 18, @@ -100,26 +120,6 @@ "confidence": "High", "note": "" }, - { - "warning_type": "SQL Injection", - "warning_code": 0, - "fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd", - "check_name": "SQL", - "message": "Possible SQL injection", - "file": "app/models/preview_card_filter.rb", - "line": 50, - "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")", - "render_path": null, - "location": { - "type": "method", - "class": "PreviewCardFilter", - "method": "trending_scope" - }, - "user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")", - "confidence": "Medium", - "note": "" - }, { "warning_type": "Cross-Site Scripting", "warning_code": 2, @@ -134,7 +134,7 @@ { "type": "template", "name": "admin/disputes/appeals/index", - "line": 16, + "line": 20, "file": "app/views/admin/disputes/appeals/index.html.haml", "rendered": { "name": "admin/disputes/appeals/_appeal", @@ -170,26 +170,6 @@ "confidence": "High", "note": "" }, - { - "warning_type": "SQL Injection", - "warning_code": 0, - "fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e", - "check_name": "SQL", - "message": "Possible SQL injection", - "file": "app/models/tag_filter.rb", - "line": 50, - "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")", - "render_path": null, - "location": { - "type": "method", - "class": "TagFilter", - "method": "trending_scope" - }, - "user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")", - "confidence": "Medium", - "note": "" - }, { "warning_type": "Cross-Site Scripting", "warning_code": 4, @@ -204,7 +184,7 @@ { "type": "template", "name": "admin/trends/links/index", - "line": 39, + "line": 45, "file": "app/views/admin/trends/links/index.html.haml", "rendered": { "name": "admin/trends/links/_preview_card", @@ -241,6 +221,6 @@ "note": "" } ], - "updated": "2022-02-13 02:24:12 +0100", + "updated": "2022-02-15 03:48:53 +0100", "brakeman_version": "5.2.1" } diff --git a/config/locales/en.yml b/config/locales/en.yml index f045174a9b7..60c29154073 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -787,6 +787,15 @@ en: rejected: Links from this publisher won't trend title: Publishers rejected: Rejected + statuses: + allow: Allow post + allow_account: Allow author + disallow: Disallow post + disallow_account: Disallow author + shared_by: + one: Shared or favourited one time + other: Shared and favourited %{friendly_count} times + title: Trending posts tags: current_score: Current score %{score} dashboard: @@ -835,16 +844,21 @@ en: body: "%{reporter} has reported %{target}" body_remote: Someone from %{domain} has reported %{target} subject: New report for %{instance} (#%{id}) - new_trending_links: - body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated. - no_approved_links: There are currently no approved trending links. - requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}. - subject: New trending links up for review on %{instance} - new_trending_tags: - body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:' - no_approved_tags: There are currently no approved trending hashtags. - requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.' - subject: New trending hashtags up for review on %{instance} + new_trends: + body: 'The following items need a review before they can be displayed publicly:' + new_trending_links: + no_approved_links: There are currently no approved trending links. + requirements: 'Any of these candidates could surpass the #%{rank} approved trending link, which is currently "%{lowest_link_title}" with a score of %{lowest_link_score}.' + title: Trending links + new_trending_statuses: + no_approved_statuses: There are currently no approved trending posts. + requirements: 'Any of these candidates could surpass the #%{rank} approved trending post, which is currently %{lowest_status_url} with a score of %{lowest_status_score}.' + title: Trending posts + new_trending_tags: + no_approved_tags: There are currently no approved trending hashtags. + requirements: 'Any of these candidates could surpass the #%{rank} approved trending hashtag, which is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.' + title: Trending hashtags + subject: New trends up for review on %{instance} aliases: add_new: Create alias created_msg: Successfully created a new alias. You can now initiate the move from the old account. diff --git a/config/navigation.rb b/config/navigation.rb index 3fc3747d595..620f78c5792 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -34,6 +34,7 @@ SimpleNavigation::Configuration.run do |navigation| n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? } n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s| + s.item :statuses, safe_join([fa_icon('comments-o fw'), t('admin.trends.statuses.title')]), admin_trends_statuses_path, highlights_on: %r{/admin/trends/statuses} s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags} s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links} end diff --git a/config/routes.rb b/config/routes.rb index 176438e452f..a820f32adfe 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -327,6 +327,12 @@ Rails.application.routes.draw do end end + resources :statuses, only: [:index] do + collection do + post :batch + end + end + namespace :links do resources :preview_card_providers, only: [:index], path: :publishers do collection do @@ -448,6 +454,7 @@ Rails.application.routes.draw do namespace :trends do resources :links, only: [:index] resources :tags, only: [:index] + resources :statuses, only: [:index] end namespace :emails do @@ -554,6 +561,8 @@ Rails.application.routes.draw do namespace :trends do resources :tags, only: [:index] + resources :links, only: [:index] + resources :statuses, only: [:index] end post :measures, to: 'measures#create' diff --git a/db/migrate/20220202200743_add_trendable_to_accounts.rb b/db/migrate/20220202200743_add_trendable_to_accounts.rb new file mode 100644 index 00000000000..414df5108d9 --- /dev/null +++ b/db/migrate/20220202200743_add_trendable_to_accounts.rb @@ -0,0 +1,7 @@ +class AddTrendableToAccounts < ActiveRecord::Migration[6.1] + def change + add_column :accounts, :trendable, :boolean + add_column :accounts, :reviewed_at, :datetime + add_column :accounts, :requested_review_at, :datetime + end +end diff --git a/db/migrate/20220202200926_add_trendable_to_statuses.rb b/db/migrate/20220202200926_add_trendable_to_statuses.rb new file mode 100644 index 00000000000..7f38c8ca71f --- /dev/null +++ b/db/migrate/20220202200926_add_trendable_to_statuses.rb @@ -0,0 +1,5 @@ +class AddTrendableToStatuses < ActiveRecord::Migration[6.1] + def change + add_column :statuses, :trendable, :boolean + end +end diff --git a/db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb b/db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb new file mode 100644 index 00000000000..d5d995ecefe --- /dev/null +++ b/db/post_migrate/20220202201015_remove_trust_level_from_accounts.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveTrustLevelFromAccounts < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + safety_assured { remove_column :accounts, :trust_level, :integer } + end +end diff --git a/db/schema.rb b/db/schema.rb index 0e9b6e61913..e54de5b3701 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -177,13 +177,15 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do t.string "also_known_as", array: true t.datetime "silenced_at" t.datetime "suspended_at" - t.integer "trust_level" t.boolean "hide_collections" t.integer "avatar_storage_schema_version" t.integer "header_storage_schema_version" t.string "devices_url" t.integer "suspension_origin" t.datetime "sensitized_at" + t.boolean "trendable" + t.datetime "reviewed_at" + t.datetime "requested_review_at" t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id" @@ -887,6 +889,7 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do t.bigint "poll_id" t.datetime "deleted_at" t.datetime "edited_at" + t.boolean "trendable" t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)" t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)" t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))" @@ -1228,5 +1231,4 @@ ActiveRecord::Schema.define(version: 2022_02_24_010024) do ORDER BY (sum(t0.rank)) DESC; SQL add_index "follow_recommendations", ["account_id"], name: "index_follow_recommendations_on_account_id", unique: true - end diff --git a/spec/controllers/api/v1/trends/tags_controller_spec.rb b/spec/controllers/api/v1/trends/tags_controller_spec.rb index e2e26dcab94..d29551c56c0 100644 --- a/spec/controllers/api/v1/trends/tags_controller_spec.rb +++ b/spec/controllers/api/v1/trends/tags_controller_spec.rb @@ -7,10 +7,9 @@ RSpec.describe Api::V1::Trends::TagsController, type: :controller do describe 'GET #index' do before do - trending_tags = double() - - allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag)) - allow(Trends).to receive(:tags).and_return(trending_tags) + Fabricate.times(10, :tag).each do |tag| + 10.times { |i| Trends.tags.add(tag, i) } + end get :index end diff --git a/spec/mailers/previews/admin_mailer_preview.rb b/spec/mailers/previews/admin_mailer_preview.rb index 9c0372b47fc..01436ba7a0d 100644 --- a/spec/mailers/previews/admin_mailer_preview.rb +++ b/spec/mailers/previews/admin_mailer_preview.rb @@ -6,14 +6,9 @@ class AdminMailerPreview < ActionMailer::Preview AdminMailer.new_pending_account(Account.first, User.pending.first) end - # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags - def new_trending_tags - AdminMailer.new_trending_tags(Account.first, Tag.limit(3)) - end - - # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links - def new_trending_links - AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3)) + # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trends + def new_trends + AdminMailer.new_trends(Account.first, PreviewCard.limit(3), Tag.limit(3), Status.where(reblog_of_id: nil).limit(3)) end # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_appeal diff --git a/spec/models/trends/statuses_spec.rb b/spec/models/trends/statuses_spec.rb new file mode 100644 index 00000000000..9cc67acbe1d --- /dev/null +++ b/spec/models/trends/statuses_spec.rb @@ -0,0 +1,110 @@ +require 'rails_helper' + +RSpec.describe Trends::Statuses do + subject! { described_class.new(threshold: 5, review_threshold: 10, score_halflife: 8.hours) } + + let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) } + + describe 'Trends::Statuses::Query' do + let!(:query) { subject.query } + let!(:today) { at_time } + + let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: today) } + let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) } + + before do + 15.times { reblog(status1, today) } + 12.times { reblog(status2, today) } + + subject.refresh(today) + end + + describe '#filtered_for' do + let(:account) { Fabricate(:account) } + + it 'returns a composable query scope' do + expect(query.filtered_for(account)).to be_a Trends::Query + end + + it 'filters out blocked accounts' do + account.block!(status1.account) + expect(query.filtered_for(account).to_a).to eq [status2] + end + + it 'filters out muted accounts' do + account.mute!(status2.account) + expect(query.filtered_for(account).to_a).to eq [status1] + end + + it 'filters out blocked-by accounts' do + status1.account.block!(account) + expect(query.filtered_for(account).to_a).to eq [status2] + end + end + end + + describe '#add' do + let(:status) { Fabricate(:status) } + + before do + subject.add(status, 1, at_time) + end + + it 'records use' do + expect(subject.send(:recently_used_ids, at_time)).to eq [status.id] + end + end + + describe '#query' do + it 'returns a composable query scope' do + expect(subject.query).to be_a Trends::Query + end + + it 'responds to filtered_for' do + expect(subject.query).to respond_to(:filtered_for) + end + end + + describe '#refresh' do + let!(:today) { at_time } + let!(:yesterday) { today - 1.day } + + let!(:status1) { Fabricate(:status, text: 'Foo', trendable: true, created_at: yesterday) } + let!(:status2) { Fabricate(:status, text: 'Bar', trendable: true, created_at: today) } + let!(:status3) { Fabricate(:status, text: 'Baz', trendable: true, created_at: today) } + + before do + 13.times { reblog(status1, today) } + 13.times { reblog(status2, today) } + 4.times { reblog(status3, today) } + end + + context do + before do + subject.refresh(today) + end + + it 'calculates and re-calculates scores' do + expect(subject.query.limit(10).to_a).to eq [status2, status1] + end + + it 'omits statuses below threshold' do + expect(subject.query.limit(10).to_a).to_not include(status3) + end + end + + it 'decays scores' do + subject.refresh(today) + original_score = subject.score(status2.id) + expect(original_score).to be_a Float + subject.refresh(today + subject.options[:score_halflife]) + decayed_score = subject.score(status2.id) + expect(decayed_score).to be <= original_score / 2 + end + end + + def reblog(status, at_time) + reblog = Fabricate(:status, reblog: status, created_at: at_time) + subject.add(status, reblog.account_id, at_time) + end +end diff --git a/spec/models/trends/tags_spec.rb b/spec/models/trends/tags_spec.rb index 4f98c6aa4c2..f48c735035a 100644 --- a/spec/models/trends/tags_spec.rb +++ b/spec/models/trends/tags_spec.rb @@ -21,7 +21,7 @@ RSpec.describe Trends::Tags do end end - describe '#get' do + describe '#query' do pending end @@ -47,11 +47,11 @@ RSpec.describe Trends::Tags do end it 'calculates and re-calculates scores' do - expect(subject.get(false, 10)).to eq [tag1, tag3] + expect(subject.query.limit(10).to_a).to eq [tag1, tag3] end it 'omits hashtags below threshold' do - expect(subject.get(false, 10)).to_not include(tag2) + expect(subject.query.limit(10).to_a).to_not include(tag2) end end