From 6e50134a42cb303e6e42f89f9ddb5aacf83e7a6d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 25 Nov 2021 13:07:38 +0100 Subject: [PATCH 1/6] Add trending links (#16917) * Add trending links * Add overriding specific links trendability * Add link type to preview cards and only trend articles Change trends review notifications from being sent every 5 minutes to being sent every 2 hours Change threshold from 5 unique accounts to 15 unique accounts * Fix tests --- app/chewy/tags_index.rb | 2 +- app/controllers/admin/dashboard_controller.rb | 2 +- app/controllers/admin/tags_controller.rb | 76 +---------- .../preview_card_providers_controller.rb | 41 ++++++ .../admin/trends/links_controller.rb | 45 ++++++ .../admin/trends/tags_controller.rb | 41 ++++++ .../api/v1/admin/dimensions_controller.rb | 3 +- .../api/v1/admin/measures_controller.rb | 3 +- .../api/v1/admin/trends/tags_controller.rb | 16 +++ .../api/v1/admin/trends_controller.rb | 16 --- .../api/v1/trends/links_controller.rb | 21 +++ .../api/v1/trends/tags_controller.rb | 21 +++ app/controllers/api/v1/trends_controller.rb | 15 -- app/helpers/admin/filter_helper.rb | 2 + app/helpers/languages_helper.rb | 94 +++++++++++++ app/helpers/settings_helper.rb | 89 +----------- .../mastodon/components/admin/Counter.js | 5 +- .../mastodon/components/admin/Dimension.js | 5 +- .../mastodon/components/admin/Trends.js | 2 +- app/javascript/styles/mastodon/accounts.scss | 16 +++ app/javascript/styles/mastodon/dashboard.scss | 10 ++ app/lib/activitypub/activity.rb | 2 - app/lib/activitypub/activity/announce.rb | 5 +- app/lib/activitypub/activity/create.rb | 7 +- app/lib/admin/metrics/dimension.rb | 9 +- .../admin/metrics/dimension/base_dimension.rb | 13 +- .../metrics/dimension/languages_dimension.rb | 4 +- .../dimension/tag_languages_dimension.rb | 36 +++++ .../dimension/tag_servers_dimension.rb | 35 +++++ app/lib/admin/metrics/measure.rb | 10 +- .../metrics/measure/active_users_measure.rb | 4 +- app/lib/admin/metrics/measure/base_measure.rb | 15 +- .../metrics/measure/interactions_measure.rb | 4 +- .../metrics/measure/tag_accounts_measure.rb | 41 ++++++ .../metrics/measure/tag_servers_measure.rb | 47 +++++++ .../admin/metrics/measure/tag_uses_measure.rb | 41 ++++++ app/lib/link_details_extractor.rb | 49 ++++++- app/mailers/admin_mailer.rb | 22 ++- app/models/account_statuses_cleanup_policy.rb | 4 +- app/models/form/preview_card_batch.rb | 65 +++++++++ .../form/preview_card_provider_batch.rb | 33 +++++ app/models/form/tag_batch.rb | 8 +- app/models/preview_card.rb | 42 +++++- app/models/preview_card_filter.rb | 53 ++++++++ app/models/preview_card_provider.rb | 57 ++++++++ app/models/preview_card_provider_filter.rb | 49 +++++++ app/models/tag.rb | 23 +--- app/models/tag_filter.rb | 56 +++++--- app/models/trending_tags.rb | 128 ------------------ app/models/trends.rb | 27 ++++ app/models/trends/base.rb | 80 +++++++++++ app/models/trends/history.rb | 98 ++++++++++++++ app/models/trends/links.rb | 117 ++++++++++++++++ app/models/trends/tags.rb | 111 +++++++++++++++ app/policies/preview_card_policy.rb | 11 ++ app/policies/preview_card_provider_policy.rb | 11 ++ .../rest/trends/link_serializer.rb | 5 + app/services/fetch_link_card_service.rb | 3 +- app/services/post_status_service.rb | 3 +- app/services/process_hashtags_service.rb | 2 +- app/services/reblog_service.rb | 13 +- app/views/admin/dashboard/index.html.haml | 2 +- app/views/admin/tags/_tag.html.haml | 19 --- app/views/admin/tags/index.html.haml | 74 ---------- app/views/admin/tags/show.html.haml | 68 ++++++---- .../trends/links/_preview_card.html.haml | 30 ++++ app/views/admin/trends/links/index.html.haml | 41 ++++++ .../_preview_card_provider.html.haml | 16 +++ .../preview_card_providers/index.html.haml | 43 ++++++ app/views/admin/trends/tags/_tag.html.haml | 24 ++++ app/views/admin/trends/tags/index.html.haml | 38 ++++++ .../admin_mailer/new_trending_links.text.erb | 16 +++ .../admin_mailer/new_trending_tag.text.erb | 5 - .../admin_mailer/new_trending_tags.text.erb | 16 +++ app/views/application/_sidebar.html.haml | 2 +- .../refresh_scheduler.rb} | 4 +- .../trends/review_notifications_scheduler.rb | 11 ++ config/brakeman.ignore | 126 +++++++++++------ config/locales/en.yml | 73 +++++++--- config/locales/simple_form.en.yml | 4 +- config/navigation.rb | 6 +- config/routes.rb | 36 ++++- config/sidekiq.yml | 8 +- ...031031021_create_preview_card_providers.rb | 12 ++ ...112011713_add_language_to_preview_cards.rb | 7 + ...15032527_add_trendable_to_preview_cards.rb | 5 + ...23212714_add_link_type_to_preview_cards.rb | 5 + db/schema.rb | 21 ++- lib/mastodon/snowflake.rb | 5 +- lib/tasks/repo.rake | 2 +- .../controllers/admin/tags_controller_spec.rb | 12 -- .../api/v1/trends/tags_controller_spec.rb | 22 +++ .../api/v1/trends_controller_spec.rb | 18 --- ...elper_spec.rb => languages_helper_spec.rb} | 9 +- spec/mailers/previews/admin_mailer_preview.rb | 10 ++ spec/models/trending_tags_spec.rb | 68 ---------- spec/models/trends/tags_spec.rb | 67 +++++++++ 97 files changed, 2071 insertions(+), 722 deletions(-) create mode 100644 app/controllers/admin/trends/links/preview_card_providers_controller.rb create mode 100644 app/controllers/admin/trends/links_controller.rb create mode 100644 app/controllers/admin/trends/tags_controller.rb create mode 100644 app/controllers/api/v1/admin/trends/tags_controller.rb delete mode 100644 app/controllers/api/v1/admin/trends_controller.rb create mode 100644 app/controllers/api/v1/trends/links_controller.rb create mode 100644 app/controllers/api/v1/trends/tags_controller.rb delete mode 100644 app/controllers/api/v1/trends_controller.rb create mode 100644 app/helpers/languages_helper.rb create mode 100644 app/lib/admin/metrics/dimension/tag_languages_dimension.rb create mode 100644 app/lib/admin/metrics/dimension/tag_servers_dimension.rb create mode 100644 app/lib/admin/metrics/measure/tag_accounts_measure.rb create mode 100644 app/lib/admin/metrics/measure/tag_servers_measure.rb create mode 100644 app/lib/admin/metrics/measure/tag_uses_measure.rb create mode 100644 app/models/form/preview_card_batch.rb create mode 100644 app/models/form/preview_card_provider_batch.rb create mode 100644 app/models/preview_card_filter.rb create mode 100644 app/models/preview_card_provider.rb create mode 100644 app/models/preview_card_provider_filter.rb delete mode 100644 app/models/trending_tags.rb create mode 100644 app/models/trends.rb create mode 100644 app/models/trends/base.rb create mode 100644 app/models/trends/history.rb create mode 100644 app/models/trends/links.rb create mode 100644 app/models/trends/tags.rb create mode 100644 app/policies/preview_card_policy.rb create mode 100644 app/policies/preview_card_provider_policy.rb create mode 100644 app/serializers/rest/trends/link_serializer.rb delete mode 100644 app/views/admin/tags/_tag.html.haml delete mode 100644 app/views/admin/tags/index.html.haml create mode 100644 app/views/admin/trends/links/_preview_card.html.haml create mode 100644 app/views/admin/trends/links/index.html.haml create mode 100644 app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml create mode 100644 app/views/admin/trends/links/preview_card_providers/index.html.haml create mode 100644 app/views/admin/trends/tags/_tag.html.haml create mode 100644 app/views/admin/trends/tags/index.html.haml create mode 100644 app/views/admin_mailer/new_trending_links.text.erb delete mode 100644 app/views/admin_mailer/new_trending_tag.text.erb create mode 100644 app/views/admin_mailer/new_trending_tags.text.erb rename app/workers/scheduler/{trending_tags_scheduler.rb => trends/refresh_scheduler.rb} (57%) create mode 100644 app/workers/scheduler/trends/review_notifications_scheduler.rb create mode 100644 db/migrate/20211031031021_create_preview_card_providers.rb create mode 100644 db/migrate/20211112011713_add_language_to_preview_cards.rb create mode 100644 db/migrate/20211115032527_add_trendable_to_preview_cards.rb create mode 100644 db/migrate/20211123212714_add_link_type_to_preview_cards.rb create mode 100644 spec/controllers/api/v1/trends/tags_controller_spec.rb delete mode 100644 spec/controllers/api/v1/trends_controller_spec.rb rename spec/helpers/{settings_helper_spec.rb => languages_helper_spec.rb} (56%) delete mode 100644 spec/models/trending_tags_spec.rb create mode 100644 spec/models/trends/tags_spec.rb diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb index f811a8d670..f9db2b03af 100644 --- a/app/chewy/tags_index.rb +++ b/app/chewy/tags_index.rb @@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index end field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? } - field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } } + field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } } field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at } end end diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index cbfff27075..f0a9354110 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -4,7 +4,7 @@ module Admin class DashboardController < BaseController def index @system_checks = Admin::SystemCheck.perform - @time_period = (1.month.ago.to_date...Time.now.utc.to_date) + @time_period = (29.days.ago.to_date...Time.now.utc.to_date) @pending_users_count = User.pending.count @pending_reports_count = Report.unresolved.count @pending_tags_count = Tag.pending_review.count diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index eed4feea2f..749e2f144d 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -2,38 +2,12 @@ module Admin class TagsController < BaseController - before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all] - before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all] - before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all] - - def index - authorize :tag, :index? - - @tags = filtered_tags.page(params[:page]) - @form = Form::TagBatch.new - end - - def batch - @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) - @form.save - rescue ActionController::ParameterMissing - flash[:alert] = I18n.t('admin.accounts.no_account_selected') - ensure - redirect_to admin_tags_path(filter_params) - end - - def approve_all - Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save - redirect_to admin_tags_path(filter_params) - end - - def reject_all - Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save - redirect_to admin_tags_path(filter_params) - end + before_action :set_tag def show authorize @tag, :show? + + @time_period = (6.days.ago.to_date...Time.now.utc.to_date) end def update @@ -52,52 +26,8 @@ module Admin @tag = Tag.find(params[:id]) end - def set_usage_by_domain - @usage_by_domain = @tag.statuses - .with_public_visibility - .excluding_silenced_accounts - .where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day))) - .joins(:account) - .group('accounts.domain') - .reorder(statuses_count: :desc) - .pluck(Arel.sql('accounts.domain, count(*) AS statuses_count')) - end - - def set_counters - @accounts_today = @tag.history.first[:accounts] - @accounts_week = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" }) - end - - def filtered_tags - TagFilter.new(filter_params).results - end - - def filter_params - params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) - end - def tag_params params.require(:tag).permit(:name, :trendable, :usable, :listable) end - - def current_week_days - now = Time.now.utc.beginning_of_day.to_date - - (Date.commercial(now.cwyear, now.cweek)..now).map do |date| - date.to_time(:utc).beginning_of_day.to_i - end - end - - def form_tag_batch_params - params.require(:form_tag_batch).permit(:action, tag_ids: []) - end - - def action_from_button - if params[:approve] - 'approve' - elsif params[:reject] - 'reject' - end - end end end diff --git a/app/controllers/admin/trends/links/preview_card_providers_controller.rb b/app/controllers/admin/trends/links/preview_card_providers_controller.rb new file mode 100644 index 0000000000..2c26e03f36 --- /dev/null +++ b/app/controllers/admin/trends/links/preview_card_providers_controller.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController + def index + authorize :preview_card_provider, :index? + + @preview_card_providers = filtered_preview_card_providers.page(params[:page]) + @form = Form::PreviewCardProviderBatch.new + end + + def batch + @form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_links_preview_card_providers_path(filter_params) + end + + private + + def filtered_preview_card_providers + PreviewCardProviderFilter.new(filter_params).results + end + + def filter_params + params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS) + end + + def form_preview_card_provider_batch_params + params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:reject] + 'reject' + end + end +end diff --git a/app/controllers/admin/trends/links_controller.rb b/app/controllers/admin/trends/links_controller.rb new file mode 100644 index 0000000000..619b37deb1 --- /dev/null +++ b/app/controllers/admin/trends/links_controller.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class Admin::Trends::LinksController < Admin::BaseController + def index + authorize :preview_card, :index? + + @preview_cards = filtered_preview_cards.page(params[:page]) + @form = Form::PreviewCardBatch.new + end + + def batch + @form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_links_path(filter_params) + end + + private + + def filtered_preview_cards + PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results + end + + def filter_params + params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS) + end + + def form_preview_card_batch_params + params.require(:form_preview_card_batch).permit(:action, preview_card_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:approve_all] + 'approve_all' + elsif params[:reject] + 'reject' + elsif params[:reject_all] + 'reject_all' + end + end +end diff --git a/app/controllers/admin/trends/tags_controller.rb b/app/controllers/admin/trends/tags_controller.rb new file mode 100644 index 0000000000..91ff33d406 --- /dev/null +++ b/app/controllers/admin/trends/tags_controller.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Trends::TagsController < Admin::BaseController + def index + authorize :tag, :index? + + @tags = filtered_tags.page(params[:page]) + @form = Form::TagBatch.new + end + + def batch + @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_tags_path(filter_params) + end + + private + + def filtered_tags + TagFilter.new(filter_params).results + end + + def filter_params + params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) + end + + def form_tag_batch_params + params.require(:form_tag_batch).permit(:action, tag_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:reject] + 'reject' + end + end +end diff --git a/app/controllers/api/v1/admin/dimensions_controller.rb b/app/controllers/api/v1/admin/dimensions_controller.rb index 170596d273..5e8f0f89f0 100644 --- a/app/controllers/api/v1/admin/dimensions_controller.rb +++ b/app/controllers/api/v1/admin/dimensions_controller.rb @@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController params[:keys], params[:start_at], params[:end_at], - params[:limit] + params[:limit], + params ) end end diff --git a/app/controllers/api/v1/admin/measures_controller.rb b/app/controllers/api/v1/admin/measures_controller.rb index a3ac6fe85f..f281917534 100644 --- a/app/controllers/api/v1/admin/measures_controller.rb +++ b/app/controllers/api/v1/admin/measures_controller.rb @@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController @measures = Admin::Metrics::Measure.retrieve( params[:keys], params[:start_at], - params[:end_at] + params[:end_at], + params ) end end diff --git a/app/controllers/api/v1/admin/trends/tags_controller.rb b/app/controllers/api/v1/admin/trends/tags_controller.rb new file mode 100644 index 0000000000..3653d1dd13 --- /dev/null +++ b/app/controllers/api/v1/admin/trends/tags_controller.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class Api::V1::Admin::Trends::TagsController < Api::BaseController + before_action :require_staff! + before_action :set_tags + + def index + render json: @tags, each_serializer: REST::Admin::TagSerializer + end + + private + + def set_tags + @tags = Trends.tags.get(false, limit_param(10)) + end +end diff --git a/app/controllers/api/v1/admin/trends_controller.rb b/app/controllers/api/v1/admin/trends_controller.rb deleted file mode 100644 index e32ab5d2c7..0000000000 --- a/app/controllers/api/v1/admin/trends_controller.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class Api::V1::Admin::TrendsController < Api::BaseController - before_action :require_staff! - before_action :set_trends - - def index - render json: @trends, each_serializer: REST::Admin::TagSerializer - end - - private - - def set_trends - @trends = TrendingTags.get(10, filtered: false) - end -end diff --git a/app/controllers/api/v1/trends/links_controller.rb b/app/controllers/api/v1/trends/links_controller.rb new file mode 100644 index 0000000000..1c3ab1e1c2 --- /dev/null +++ b/app/controllers/api/v1/trends/links_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::Trends::LinksController < Api::BaseController + before_action :set_links + + def index + render json: @links, each_serializer: REST::Trends::LinkSerializer + end + + private + + def set_links + @links = begin + if Setting.trends + Trends.links.get(true, limit_param(10)) + else + [] + end + end + end +end diff --git a/app/controllers/api/v1/trends/tags_controller.rb b/app/controllers/api/v1/trends/tags_controller.rb new file mode 100644 index 0000000000..947b53de23 --- /dev/null +++ b/app/controllers/api/v1/trends/tags_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::Trends::TagsController < Api::BaseController + before_action :set_tags + + def index + render json: @tags, each_serializer: REST::TagSerializer + end + + private + + def set_tags + @tags = begin + if Setting.trends + Trends.tags.get(true, limit_param(10)) + else + [] + end + end + end +end diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb deleted file mode 100644 index c875e90414..0000000000 --- a/app/controllers/api/v1/trends_controller.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -class Api::V1::TrendsController < Api::BaseController - before_action :set_tags - - def index - render json: @tags, each_serializer: REST::TagSerializer - end - - private - - def set_tags - @tags = TrendingTags.get(limit_param(10)) - end -end diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb index ba0ca96389..5f69f176a6 100644 --- a/app/helpers/admin/filter_helper.rb +++ b/app/helpers/admin/filter_helper.rb @@ -6,6 +6,8 @@ module Admin::FilterHelper CustomEmojiFilter::KEYS, ReportFilter::KEYS, TagFilter::KEYS, + PreviewCardProviderFilter::KEYS, + PreviewCardFilter::KEYS, InstanceFilter::KEYS, InviteFilter::KEYS, RelationshipFilter::KEYS, diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb new file mode 100644 index 0000000000..7307242086 --- /dev/null +++ b/app/helpers/languages_helper.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module LanguagesHelper + HUMAN_LOCALES = { + af: 'Afrikaans', + ar: 'العربية', + ast: 'Asturianu', + bg: 'Български', + bn: 'বাংলা', + br: 'Breton', + ca: 'Català', + co: 'Corsu', + cs: 'Čeština', + cy: 'Cymraeg', + da: 'Dansk', + de: 'Deutsch', + el: 'Ελληνικά', + en: 'English', + eo: 'Esperanto', + 'es-AR': 'Español (Argentina)', + 'es-MX': 'Español (México)', + es: 'Español', + et: 'Eesti', + eu: 'Euskara', + fa: 'فارسی', + fi: 'Suomi', + fr: 'Français', + ga: 'Gaeilge', + gd: 'Gàidhlig', + gl: 'Galego', + he: 'עברית', + hi: 'हिन्दी', + hr: 'Hrvatski', + hu: 'Magyar', + hy: 'Հայերեն', + id: 'Bahasa Indonesia', + io: 'Ido', + is: 'Íslenska', + it: 'Italiano', + ja: '日本語', + ka: 'ქართული', + kab: 'Taqbaylit', + kk: 'Қазақша', + kmr: 'Kurmancî', + kn: 'ಕನ್ನಡ', + ko: '한국어', + ku: 'سۆرانی', + lt: 'Lietuvių', + lv: 'Latviešu', + mk: 'Македонски', + ml: 'മലയാളം', + mr: 'मराठी', + ms: 'Bahasa Melayu', + nl: 'Nederlands', + nn: 'Nynorsk', + no: 'Norsk', + oc: 'Occitan', + pl: 'Polski', + 'pt-BR': 'Português (Brasil)', + 'pt-PT': 'Português (Portugal)', + pt: 'Português', + ro: 'Română', + ru: 'Русский', + sa: 'संस्कृतम्', + sc: 'Sardu', + si: 'සිංහල', + sk: 'Slovenčina', + sl: 'Slovenščina', + sq: 'Shqip', + 'sr-Latn': 'Srpski (latinica)', + sr: 'Српски', + sv: 'Svenska', + ta: 'தமிழ்', + te: 'తెలుగు', + th: 'ไทย', + tr: 'Türkçe', + uk: 'Українська', + ur: 'اُردُو', + vi: 'Tiếng Việt', + zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ', + 'zh-CN': '简体中文', + 'zh-HK': '繁體中文（香港）', + 'zh-TW': '繁體中文（臺灣）', + zh: '中文', + }.freeze + + def human_locale(locale) + if locale == 'und' + I18n.t('generic.none') + else + HUMAN_LOCALES[locale.to_sym] || locale + end + end +end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index ac4c187461..23739d1cd4 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -1,95 +1,8 @@ # frozen_string_literal: true module SettingsHelper - HUMAN_LOCALES = { - af: 'Afrikaans', - ar: 'العربية', - ast: 'Asturianu', - bg: 'Български', - bn: 'বাংলা', - br: 'Breton', - ca: 'Català', - co: 'Corsu', - cs: 'Čeština', - cy: 'Cymraeg', - da: 'Dansk', - de: 'Deutsch', - el: 'Ελληνικά', - en: 'English', - eo: 'Esperanto', - 'es-AR': 'Español (Argentina)', - 'es-MX': 'Español (México)', - es: 'Español', - et: 'Eesti', - eu: 'Euskara', - fa: 'فارسی', - fi: 'Suomi', - fr: 'Français', - ga: 'Gaeilge', - gd: 'Gàidhlig', - gl: 'Galego', - he: 'עברית', - hi: 'हिन्दी', - hr: 'Hrvatski', - hu: 'Magyar', - hy: 'Հայերեն', - id: 'Bahasa Indonesia', - io: 'Ido', - is: 'Íslenska', - it: 'Italiano', - ja: '日本語', - ka: 'ქართული', - kab: 'Taqbaylit', - kk: 'Қазақша', - kmr: 'Kurmancî', - kn: 'ಕನ್ನಡ', - ko: '한국어', - ku: 'سۆرانی', - lt: 'Lietuvių', - lv: 'Latviešu', - mk: 'Македонски', - ml: 'മലയാളം', - mr: 'मराठी', - ms: 'Bahasa Melayu', - nl: 'Nederlands', - nn: 'Nynorsk', - no: 'Norsk', - oc: 'Occitan', - pl: 'Polski', - 'pt-BR': 'Português (Brasil)', - 'pt-PT': 'Português (Portugal)', - pt: 'Português', - ro: 'Română', - ru: 'Русский', - sa: 'संस्कृतम्', - sc: 'Sardu', - si: 'සිංහල', - sk: 'Slovenčina', - sl: 'Slovenščina', - sq: 'Shqip', - 'sr-Latn': 'Srpski (latinica)', - sr: 'Српски', - sv: 'Svenska', - ta: 'தமிழ்', - te: 'తెలుగు', - th: 'ไทย', - tr: 'Türkçe', - uk: 'Українська', - ur: 'اُردُو', - vi: 'Tiếng Việt', - zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ', - 'zh-CN': '简体中文', - 'zh-HK': '繁體中文（香港）', - 'zh-TW': '繁體中文（臺灣）', - zh: '中文', - }.freeze - - def human_locale(locale) - HUMAN_LOCALES[locale] - end - def filterable_languages - LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?)) + LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?)) end def hash_to_object(hash) diff --git a/app/javascript/mastodon/components/admin/Counter.js b/app/javascript/mastodon/components/admin/Counter.js index cda572dced..047e864b25 100644 --- a/app/javascript/mastodon/components/admin/Counter.js +++ b/app/javascript/mastodon/components/admin/Counter.js @@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent { end_at: PropTypes.string.isRequired, label: PropTypes.string.isRequired, href: PropTypes.string, + params: PropTypes.object, }; state = { @@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent { }; componentDidMount () { - const { measure, start_at, end_at } = this.props; + const { measure, start_at, end_at, params } = this.props; - api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => { + api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => { this.setState({ loading: false, data: res.data, diff --git a/app/javascript/mastodon/components/admin/Dimension.js b/app/javascript/mastodon/components/admin/Dimension.js index ac6dbd1c79..977c8208df 100644 --- a/app/javascript/mastodon/components/admin/Dimension.js +++ b/app/javascript/mastodon/components/admin/Dimension.js @@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent { end_at: PropTypes.string.isRequired, limit: PropTypes.number.isRequired, label: PropTypes.string.isRequired, + params: PropTypes.object, }; state = { @@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent { }; componentDidMount () { - const { start_at, end_at, dimension, limit } = this.props; + const { start_at, end_at, dimension, limit, params } = this.props; - api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => { + api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => { this.setState({ loading: false, data: res.data, diff --git a/app/javascript/mastodon/components/admin/Trends.js b/app/javascript/mastodon/components/admin/Trends.js index 46307a28ad..635bdf37d5 100644 --- a/app/javascript/mastodon/components/admin/Trends.js +++ b/app/javascript/mastodon/components/admin/Trends.js @@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent { componentDidMount () { const { limit } = this.props; - api().get('/api/v1/admin/trends', { params: { limit } }).then(res => { + api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => { this.setState({ loading: false, data: res.data, diff --git a/app/javascript/styles/mastodon/accounts.scss b/app/javascript/styles/mastodon/accounts.scss index 2c78e81be2..b8a6c80188 100644 --- a/app/javascript/styles/mastodon/accounts.scss +++ b/app/javascript/styles/mastodon/accounts.scss @@ -325,3 +325,19 @@ margin-top: 10px; } } + +.batch-table__row--muted .pending-account__header { + &, + a, + strong { + color: lighten($ui-base-color, 26%); + } +} + +.batch-table__row--attention .pending-account__header { + &, + a, + strong { + color: $gold-star; + } +} diff --git a/app/javascript/styles/mastodon/dashboard.scss b/app/javascript/styles/mastodon/dashboard.scss index 5e900e8c51..0a881bc108 100644 --- a/app/javascript/styles/mastodon/dashboard.scss +++ b/app/javascript/styles/mastodon/dashboard.scss @@ -100,6 +100,16 @@ transition: all 200ms ease-out; } + &.positive { + background: lighten($ui-base-color, 4%); + color: $valid-value-color; + } + + &.negative { + background: lighten($ui-base-color, 4%); + color: $error-value-color; + } + span { flex: 1 1 auto; } diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index d2ec122a4d..3aeecb4ec0 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -129,8 +129,6 @@ class ActivityPub::Activity end def crawl_links(status) - return if status.spoiler_text? - # Spread out crawling randomly to avoid DDoSing the link LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id) end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 9f778ffb98..6c5d88d185 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity visibility: visibility_from_audience ) - original_status.tags.each do |tag| - tag.use!(@account) - end + Trends.tags.register(@status) + Trends.links.register(@status) distribute(@status) end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 4c13a80a67..8a0dc9d33d 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def attach_tags(status) @tags.each do |tag| status.tags << tag - tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility? + tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago) end + # If we're processing an old status, this may register tags as being used now + # as opposed to when the status was really published, but this is probably + # not a big deal + Trends.tags.register(status) + @mentions.each do |mention| mention.status = status mention.save diff --git a/app/lib/admin/metrics/dimension.rb b/app/lib/admin/metrics/dimension.rb index 279539f686..d8392ddfcc 100644 --- a/app/lib/admin/metrics/dimension.rb +++ b/app/lib/admin/metrics/dimension.rb @@ -7,9 +7,14 @@ class Admin::Metrics::Dimension servers: Admin::Metrics::Dimension::ServersDimension, space_usage: Admin::Metrics::Dimension::SpaceUsageDimension, software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension, + tag_servers: Admin::Metrics::Dimension::TagServersDimension, + tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension, }.freeze - def self.retrieve(dimension_keys, start_at, end_at, limit) - Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact + def self.retrieve(dimension_keys, start_at, end_at, limit, params) + Array(dimension_keys).map do |key| + klass = DIMENSIONS[key.to_sym] + klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil) + end.compact end end diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb index 8ed8d7683a..5872c22cbc 100644 --- a/app/lib/admin/metrics/dimension/base_dimension.rb +++ b/app/lib/admin/metrics/dimension/base_dimension.rb @@ -1,10 +1,15 @@ # frozen_string_literal: true class Admin::Metrics::Dimension::BaseDimension - def initialize(start_at, end_at, limit) + def self.with_params? + false + end + + def initialize(start_at, end_at, limit, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime @limit = limit&.to_i + @params = params end def key @@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension protected def time_period - (@start_at...@end_at) + (@start_at..@end_at) + end + + def params + raise NotImplementedError end end diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb index 2d0ac124e0..a6aaf5d215 100644 --- a/app/lib/admin/metrics/dimension/languages_dimension.rb +++ b/app/lib/admin/metrics/dimension/languages_dimension.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + def key 'languages' end @@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension: rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]]) - rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } } + rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } } end end diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb new file mode 100644 index 0000000000..1cfa07478e --- /dev/null +++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + + def self.with_params? + true + end + + def key + 'tag_languages' + end + + def data + sql = <<-SQL.squish + SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value + FROM statuses + INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + GROUP BY COALESCE(statuses.language, 'und') + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } } + end + + private + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb new file mode 100644 index 0000000000..12c5980d72 --- /dev/null +++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension + def self.with_params? + true + end + + def key + 'tag_servers' + end + + def data + sql = <<-SQL.squish + SELECT accounts.domain, count(*) AS value + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + GROUP BY accounts.domain + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } + end + + private + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure.rb b/app/lib/admin/metrics/measure.rb index 5cebf0331e..a839498a13 100644 --- a/app/lib/admin/metrics/measure.rb +++ b/app/lib/admin/metrics/measure.rb @@ -7,9 +7,15 @@ class Admin::Metrics::Measure interactions: Admin::Metrics::Measure::InteractionsMeasure, opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure, resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure, + tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure, + tag_uses: Admin::Metrics::Measure::TagUsesMeasure, + tag_servers: Admin::Metrics::Measure::TagServersMeasure, }.freeze - def self.retrieve(measure_keys, start_at, end_at) - Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact + def self.retrieve(measure_keys, start_at, end_at, params) + Array(measure_keys).map do |key| + klass = MEASURES[key.to_sym] + klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil) + end.compact end end diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb index ac022eb9d1..5131897806 100644 --- a/app/lib/admin/metrics/measure/active_users_measure.rb +++ b/app/lib/admin/metrics/measure/active_users_measure.rb @@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas end def time_period - (@start_at.to_date...@end_at.to_date) + (@start_at.to_date..@end_at.to_date) end def previous_time_period - ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period)) + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) end end diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb index 4c336a69e6..0107ffd9c1 100644 --- a/app/lib/admin/metrics/measure/base_measure.rb +++ b/app/lib/admin/metrics/measure/base_measure.rb @@ -1,9 +1,14 @@ # frozen_string_literal: true class Admin::Metrics::Measure::BaseMeasure - def initialize(start_at, end_at) + def self.with_params? + false + end + + def initialize(start_at, end_at, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime + @params = params end def key @@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure protected def time_period - (@start_at...@end_at) + (@start_at..@end_at) end def previous_time_period - ((@start_at - length_of_period)...(@end_at - length_of_period)) + ((@start_at - length_of_period)..(@end_at - length_of_period)) end def length_of_period @length_of_period ||= @end_at - @start_at end + + def params + raise NotImplementedError + end end diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb index 9a4ef6d637..b928fdb8fb 100644 --- a/app/lib/admin/metrics/measure/interactions_measure.rb +++ b/app/lib/admin/metrics/measure/interactions_measure.rb @@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba end def time_period - (@start_at.to_date...@end_at.to_date) + (@start_at.to_date..@end_at.to_date) end def previous_time_period - ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period)) + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) end end diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb new file mode 100644 index 0000000000..ef773081be --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_accounts' + end + + def total + tag.history.aggregate(time_period).accounts + end + + def previous_total + tag.history.aggregate(previous_time_period).accounts + end + + def data + time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb new file mode 100644 index 0000000000..8c3e0551a1 --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_servers' + end + + def total + tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') + end + + def previous_total + tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') + end + + def data + sql = <<-SQL.squish + SELECT axis.*, ( + SELECT count(*) AS value + FROM statuses + WHERE statuses.id BETWEEN $1 AND $2 + AND date_trunc('day', statuses.created_at)::date = axis.day + ) + FROM ( + SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day + ) as axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]]) + + rows.map { |row| { date: row['day'], value: row['value'].to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb new file mode 100644 index 0000000000..b7667bc6cf --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_uses' + end + + def total + tag.history.aggregate(time_period).uses + end + + def previous_total + tag.history.aggregate(previous_time_period).uses + end + + def data + time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index 8b38e8d0c6..56ad0717b7 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -4,6 +4,11 @@ class LinkDetailsExtractor include ActionView::Helpers::TagHelper class StructuredData + SUPPORTED_TYPES = %w( + NewsArticle + WebPage + ).freeze + def initialize(data) @data = data end @@ -16,6 +21,14 @@ class LinkDetailsExtractor json['description'] end + def language + json['inLanguage'] + end + + def type + json['@type'] + end + def image obj = first_of_value(json['image']) @@ -44,6 +57,10 @@ class LinkDetailsExtractor publisher['name'] end + def publisher_logo + publisher.dig('logo', 'url') + end + private def author @@ -58,8 +75,12 @@ class LinkDetailsExtractor arr.is_a?(Array) ? arr.first : arr end + def root_array(root) + root.is_a?(Array) ? root : [root] + end + def json - @json ||= first_of_value(Oj.load(@data)) + @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {} end end @@ -75,6 +96,7 @@ class LinkDetailsExtractor description: description || '', image_remote_url: image, type: type, + link_type: link_type, width: width || 0, height: height || 0, html: html || '', @@ -83,6 +105,7 @@ class LinkDetailsExtractor author_name: author_name || '', author_url: author_url || '', embed_url: embed_url || '', + language: language, } end @@ -90,6 +113,14 @@ class LinkDetailsExtractor player_url.present? ? :video : :link end + def link_type + if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article' + :article + else + :unknown + end + end + def html player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil end @@ -138,6 +169,14 @@ class LinkDetailsExtractor valid_url_or_nil(opengraph_tag('twitter:player:stream')) end + def language + valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first) + end + + def icon + valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon')) + end + private def player_url @@ -162,6 +201,14 @@ class LinkDetailsExtractor nil end + def valid_locale_or_nil(str) + return nil if str.blank? + + code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA + locale = ISO_639.find(code) + locale&.alpha2 + end + def link_tag(name) document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first end diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index 11fd09e30e..0fbd9932d8 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer end end - def new_trending_tag(recipient, tag) - @tag = tag - @me = recipient - @instance = Rails.configuration.x.local_domain + def new_trending_tags(recipient, tags) + @tags = tags + @me = recipient + @instance = Rails.configuration.x.local_domain + @lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last locale_for_account(@me) do - mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name) + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance) + end + end + + def new_trending_links(recipient, links) + @links = links + @me = recipient + @instance = Rails.configuration.x.local_domain + @lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last + + locale_for_account(@me) do + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance) end end end diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb index 0a9551ec28..0f78c1a546 100644 --- a/app/models/account_statuses_cleanup_policy.rb +++ b/app/models/account_statuses_cleanup_policy.rb @@ -4,8 +4,8 @@ # # Table name: account_statuses_cleanup_policies # -# id :bigint not null, primary key -# account_id :bigint not null +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null # enabled :boolean default(TRUE), not null # min_status_age :integer default(1209600), not null # keep_direct :boolean default(TRUE), not null diff --git a/app/models/form/preview_card_batch.rb b/app/models/form/preview_card_batch.rb new file mode 100644 index 0000000000..5f6e6522a2 --- /dev/null +++ b/app/models/form/preview_card_batch.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Form::PreviewCardBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_ids, :action, :current_account, :precision + + def save + case action + when 'approve' + approve! + when 'approve_all' + approve_all! + when 'reject' + reject! + when 'reject_all' + reject_all! + end + end + + private + + def preview_cards + @preview_cards ||= PreviewCard.where(id: preview_card_ids) + end + + def preview_card_providers + @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) } + end + + def approve! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: true) + end + + def approve_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: true, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def reject! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: false) + end + + def reject_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: false, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def action_time + @action_time ||= Time.now.utc + end +end diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/form/preview_card_provider_batch.rb new file mode 100644 index 0000000000..e6ab3d8fa3 --- /dev/null +++ b/app/models/form/preview_card_provider_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Form::PreviewCardProviderBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_provider_ids, :action, :current_account + + def save + case action + when 'approve' + approve! + when 'reject' + reject! + end + end + + private + + def preview_card_providers + PreviewCardProvider.where(id: preview_card_provider_ids) + end + + def approve! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) + end + + def reject! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) + end +end diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb index fd517a1a64..b9330745fe 100644 --- a/app/models/form/tag_batch.rb +++ b/app/models/form/tag_batch.rb @@ -23,11 +23,15 @@ class Form::TagBatch def approve! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: true, reviewed_at: Time.now.utc) + tags.update_all(trendable: true, reviewed_at: action_time) end def reject! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: false, reviewed_at: Time.now.utc) + tags.update_all(trendable: false, reviewed_at: action_time) + end + + def action_time + @action_time ||= Time.now.utc end end diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index bca3a3ce8f..f2ab8ecabf 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -24,6 +24,11 @@ # embed_url :string default(""), not null # image_storage_schema_version :integer # blurhash :string +# language :string +# max_score :float +# max_score_at :datetime +# trendable :boolean +# link_type :integer # class PreviewCard < ApplicationRecord @@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord self.inheritance_column = false enum type: [:link, :photo, :video, :rich] + enum link_type: [:unknown, :article] has_and_belongs_to_many :statuses @@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord before_save :extract_dimensions, if: :link? + def appropriate_for_trends? + link? && article? && title.present? && description.present? && image.present? && provider_name.present? + end + + def domain + @domain ||= Addressable::URI.parse(url).normalized_host + end + + def provider + @provider ||= PreviewCardProvider.matching_domain(domain) + end + + def trendable? + if attributes['trendable'].nil? + provider&.trendable? + else + attributes['trendable'] + end + end + + def requires_review_notification? + attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?) + end + + attr_writer :provider + def local? false end @@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord save! end + def history + @history ||= Trends::History.new('links', id) + end + class << self private - # rubocop:disable Naming/MethodParameterName - def image_styles(f) + def image_styles(file) styles = { original: { geometry: '400x400>', @@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord }, } - styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif' + styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif' styles end - # rubocop:enable Naming/MethodParameterName end private diff --git a/app/models/preview_card_filter.rb b/app/models/preview_card_filter.rb new file mode 100644 index 0000000000..8dda9989ce --- /dev/null +++ b/app/models/preview_card_filter.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class PreviewCardFilter + KEYS = %i( + trending + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCard.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'trending' + trending_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def trending_scope(value) + ids = begin + case value.to_s + when 'allowed' + Trends.links.currently_trending_ids(true, -1) + else + Trends.links.currently_trending_ids(false, -1) + end + end + + if ids.empty? + PreviewCard.none + else + PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') + end + end +end diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb new file mode 100644 index 0000000000..15b24e2bdf --- /dev/null +++ b/app/models/preview_card_provider.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: preview_card_providers +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# icon_file_name :string +# icon_content_type :string +# icon_file_size :bigint(8) +# icon_updated_at :datetime +# trendable :boolean +# reviewed_at :datetime +# requested_review_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# + +class PreviewCardProvider < ApplicationRecord + include DomainNormalizable + include Attachmentable + + ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze + LIMIT = 1.megabyte + + validates :domain, presence: true, uniqueness: true, domain: true + + has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false + validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT } + remotable_attachment :icon, LIMIT + + scope :trendable, -> { where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } + scope :reviewed, -> { where.not(reviewed_at: nil) } + scope :pending_review, -> { where(reviewed_at: nil) } + + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def requires_review_notification? + requires_review? && !requested_review? + end + + def self.matching_domain(domain) + segments = domain.split('.') + where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first + end +end diff --git a/app/models/preview_card_provider_filter.rb b/app/models/preview_card_provider_filter.rb new file mode 100644 index 0000000000..1e90d3c9da --- /dev/null +++ b/app/models/preview_card_provider_filter.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class PreviewCardProviderFilter + KEYS = %i( + status + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCardProvider.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope.order(domain: :asc) + end + + private + + def scope_for(key, value) + case key.to_s + when 'status' + status_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def status_scope(value) + case value.to_s + when 'approved' + PreviewCardProvider.trendable + when 'rejected' + PreviewCardProvider.not_trendable + when 'pending_review' + PreviewCardProvider.pending_review + else + raise "Unknown status: #{value}" + end + end +end diff --git a/app/models/tag.rb b/app/models/tag.rb index dcce283911..f35d92b5dd 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -36,6 +36,7 @@ class Tag < ApplicationRecord scope :usable, -> { where(usable: [true, nil]) } scope :listable, -> { where(listable: [true, nil]) } scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) } scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index @@ -75,28 +76,12 @@ class Tag < ApplicationRecord requested_review_at.present? end - def use!(account, status: nil, at_time: Time.now.utc) - TrendingTags.record_use!(self, account, status: status, at_time: at_time) - end - - def trending? - TrendingTags.trending?(self) + def requires_review_notification? + requires_review? && !requested_review? end def history - days = [] - - 7.times do |i| - day = i.days.ago.beginning_of_day.to_i - - days << { - day: day.to_s, - uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0', - accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s, - } - end - - days + @history ||= Trends::History.new('tags', id) end class << self diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb index 85bfcbea5a..ecdb52503e 100644 --- a/app/models/tag_filter.rb +++ b/app/models/tag_filter.rb @@ -2,13 +2,8 @@ class TagFilter KEYS = %i( - directory - reviewed - unreviewed - pending_review - popular - active - name + trending + status ).freeze attr_reader :params @@ -18,7 +13,13 @@ class TagFilter end def results - scope = Tag.unscoped + scope = begin + if params[:status] == 'pending_review' + Tag.unscoped + else + trending_scope + end + end params.each do |key, value| next if key.to_s == 'page' @@ -26,27 +27,40 @@ class TagFilter scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end - scope.order(id: :desc) + scope end private def scope_for(key, value) case key.to_s - when 'reviewed' - Tag.reviewed.order(reviewed_at: :desc) - when 'unreviewed' - Tag.unreviewed - when 'pending_review' - Tag.pending_review.order(requested_review_at: :desc) - when 'popular' - Tag.order('max_score DESC NULLS LAST') - when 'active' - Tag.order('last_status_at DESC NULLS LAST') - when 'name' - Tag.matches_name(value) + when 'status' + status_scope(value) else raise "Unknown filter: #{key}" end end + + def trending_scope + ids = Trends.tags.currently_trending_ids(false, -1) + + if ids.empty? + Tag.none + else + Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering') + end + end + + def status_scope(value) + case value.to_s + when 'approved' + Tag.trendable + when 'rejected' + Tag.not_trendable + when 'pending_review' + Tag.pending_review + else + raise "Unknown status: #{value}" + end + end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb deleted file mode 100644 index 31890b0824..0000000000 --- a/app/models/trending_tags.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -class TrendingTags - KEY = 'trending_tags' - EXPIRE_HISTORY_AFTER = 7.days.seconds - EXPIRE_TRENDS_AFTER = 1.day.seconds - THRESHOLD = 5 - LIMIT = 10 - REVIEW_THRESHOLD = 3 - MAX_SCORE_COOLDOWN = 2.days.freeze - MAX_SCORE_HALFLIFE = 2.hours.freeze - - class << self - include Redisable - - def record_use!(tag, account, status: nil, at_time: Time.now.utc) - return unless tag.usable? && !account.silenced? - - # Even if a tag is not allowed to trend, we still need to - # record the stats since they can be displayed in other places - increment_historical_use!(tag.id, at_time) - increment_unique_use!(tag.id, account.id, at_time) - increment_use!(tag.id, at_time) - - # Only update when the tag was last used once every 12 hours - # and only if a status is given (lets use ignore reblogs) - tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago)) - end - - def update!(at_time = Time.now.utc) - tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) - tags = Tag.trendable.where(id: tag_ids.uniq) - - # First pass to calculate scores and update the set - - tags.each do |tag| - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - max_time = tag.max_score_at - max_score = tag.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) - - score = begin - if expected > observed || observed < THRESHOLD - 0 - else - ((observed - expected)**2) / expected - end - end - - if score > max_score - max_score = score - max_time = at_time - - # Not interested in triggering any callbacks for this - tag.update_columns(max_score: max_score, max_score_at: max_time) - end - - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) - - if decaying_score.zero? - redis.zrem(KEY, tag.id) - else - redis.zadd(KEY, decaying_score, tag.id) - end - end - - users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) - - # Second pass to notify about previously unreviewed trends - - tags.each do |tag| - current_rank = redis.zrevrank(KEY, tag.id) - needs_review_notification = tag.requires_review? && !tag.requested_review? - rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD - - next unless !tag.trendable? && rank_passes_threshold && needs_review_notification - - tag.touch(:requested_review_at) - - users_for_review.each do |user| - AdminMailer.new_trending_tag(user.account, tag).deliver_later! - end - end - - # Trim older items - - redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) - redis.zremrangebyscore(KEY, '(0.3', '-inf') - end - - def get(limit, filtered: true) - tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) - - tags = Tag.where(id: tag_ids) - tags = tags.trendable if filtered - tags = tags.index_by(&:id) - - tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit) - end - - def trending?(tag) - rank = redis.zrevrank(KEY, tag.id) - rank.present? && rank < LIMIT - end - - private - - def increment_historical_use!(tag_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" - redis.incrby(key, 1) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_unique_use!(tag_id, account_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" - redis.pfadd(key, account_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_use!(tag_id, at_time) - key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" - redis.sadd(key, tag_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - end -end diff --git a/app/models/trends.rb b/app/models/trends.rb new file mode 100644 index 0000000000..7dd3a9c87e --- /dev/null +++ b/app/models/trends.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Trends + def self.table_name_prefix + 'trends_' + end + + def self.links + @links ||= Trends::Links.new + end + + def self.tags + @tags ||= Trends::Tags.new + end + + def self.refresh! + [links, tags].each(&:refresh) + end + + def self.request_review! + [links, tags].each(&:request_review) if enabled? + end + + def self.enabled? + Setting.trends + end +end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb new file mode 100644 index 0000000000..b767dcb1ac --- /dev/null +++ b/app/models/trends/base.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +class Trends::Base + include Redisable + + class_attribute :default_options + + attr_reader :options + + # @param [Hash] options + # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score + # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review + # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from + # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays + def initialize(options = {}) + @options = self.class.default_options.merge(options) + end + + def register(_status) + raise NotImplementedError + end + + def add(*) + raise NotImplementedError + end + + def refresh(*) + raise NotImplementedError + end + + def request_review + raise NotImplementedError + end + + def get(*) + raise NotImplementedError + end + + def score(id) + redis.zscore("#{key_prefix}:all", id) || 0 + end + + def rank(id) + redis.zrevrank("#{key_prefix}:allowed", id) + end + + def currently_trending_ids(allowed, limit) + redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i) + end + + protected + + def key_prefix + raise NotImplementedError + end + + def recently_used_ids(at_time = Time.now.utc) + redis.smembers(used_key(at_time)).map(&:to_i) + end + + def record_used_id(id, at_time = Time.now.utc) + redis.sadd(used_key(at_time), id) + redis.expire(used_key(at_time), 1.day.seconds) + end + + def trim_older_items + redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1') + redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1') + end + + def score_at_rank(rank) + redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 + end + + private + + def used_key(at_time) + "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}" + end +end diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb new file mode 100644 index 0000000000..608e337924 --- /dev/null +++ b/app/models/trends/history.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +class Trends::History + include Enumerable + + class Aggregate + include Redisable + + def initialize(prefix, id, date_range) + @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) } + end + + def uses + redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum + end + + def accounts + redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) + end + end + + class Day + include Redisable + + EXPIRE_AFTER = 14.days.seconds + + def initialize(prefix, id, day) + @prefix = prefix + @id = id + @day = day.beginning_of_day + end + + attr_reader :day + + def accounts + redis.pfcount(key_for(:accounts)) + end + + def uses + redis.get(key_for(:uses))&.to_i || 0 + end + + def add(account_id) + redis.pipelined do + redis.incrby(key_for(:uses), 1) + redis.pfadd(key_for(:accounts), account_id) + redis.expire(key_for(:uses), EXPIRE_AFTER) + redis.expire(key_for(:accounts), EXPIRE_AFTER) + end + end + + def as_json + { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s } + end + + def key_for(suffix) + case suffix + when :accounts + "#{key_prefix}:#{suffix}" + when :uses + key_prefix + end + end + + def key_prefix + "activity:#{@prefix}:#{@id}:#{day.to_i}" + end + end + + def initialize(prefix, id) + @prefix = prefix + @id = id + end + + def get(date) + Day.new(@prefix, @id, date) + end + + def add(account_id, at_time = Time.now.utc) + Day.new(@prefix, @id, at_time).add(account_id) + end + + def aggregate(date_range) + Aggregate.new(@prefix, @id, date_range) + end + + def each(&block) + if block_given? + (0...7).map { |i| block.call(get(i.days.ago)) } + else + to_enum(:each) + end + end + + def as_json(*) + map(&:as_json) + end +end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb new file mode 100644 index 0000000000..a0d65138ba --- /dev/null +++ b/app/models/trends/links.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +class Trends::Links < Trends::Base + PREFIX = 'trending_links' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 8.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? && + !original_status.spoiler_text? + + original_status.preview_cards.each do |preview_card| + add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends? + end + end + + def add(preview_card, account_id, at_time = Time.now.utc) + preview_card.history.add(account_id, at_time) + record_used_id(preview_card.id, at_time) + end + + def get(allowed, limit) + preview_card_ids = currently_trending_ids(allowed, limit) + preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) + preview_card_ids.map { |id| preview_cards[id] }.compact + end + + def refresh(at_time = Time.now.utc) + preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(preview_cards, at_time) + trim_older_items + end + + def request_review + preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) + + preview_cards_requiring_review = preview_cards.filter_map do |preview_card| + next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? + + if preview_card.provider.nil? + preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) + else + preview_card.provider.touch(:requested_review_at) + end + + preview_card + end + + return if preview_cards_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(preview_cards, at_time) + preview_cards.each do |preview_card| + expected = preview_card.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = preview_card.history.get(at_time).accounts.to_f + max_time = preview_card.max_score_at + max_score = preview_card.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + preview_card.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", preview_card.id) + redis.zrem("#{PREFIX}:allowed", preview_card.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) + + if preview_card.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) + else + redis.zrem("#{PREFIX}:allowed", preview_card.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb new file mode 100644 index 0000000000..13e0ab56b4 --- /dev/null +++ b/app/models/trends/tags.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +class Trends::Tags < Trends::Base + PREFIX = 'trending_tags' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 4.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? + + original_status.tags.each do |tag| + add(tag, status.account_id, at_time) if tag.usable? + end + end + + def add(tag, account_id, at_time = Time.now.utc) + tag.history.add(account_id, at_time) + record_used_id(tag.id, at_time) + end + + def refresh(at_time = Time.now.utc) + tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(tags, at_time) + trim_older_items + end + + def get(allowed, limit) + tag_ids = currently_trending_ids(allowed, limit) + tags = Tag.where(id: tag_ids).index_by(&:id) + tag_ids.map { |id| tags[id] }.compact + end + + def request_review + tags = Tag.where(id: currently_trending_ids(false, -1)) + + tags_requiring_review = tags.filter_map do |tag| + next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? + + tag.touch(:requested_review_at) + tag + end + + return if tags_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(tags, at_time) + tags.each do |tag| + expected = tag.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = tag.history.get(at_time).accounts.to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", tag.id) + redis.zrem("#{PREFIX}:allowed", tag.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, tag.id) + + if tag.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) + else + redis.zrem("#{PREFIX}:allowed", tag.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/policies/preview_card_policy.rb b/app/policies/preview_card_policy.rb new file mode 100644 index 0000000000..4f485d7fc6 --- /dev/null +++ b/app/policies/preview_card_policy.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class PreviewCardPolicy < ApplicationPolicy + def index? + staff? + end + + def update? + staff? + end +end diff --git a/app/policies/preview_card_provider_policy.rb b/app/policies/preview_card_provider_policy.rb new file mode 100644 index 0000000000..598d54a5e2 --- /dev/null +++ b/app/policies/preview_card_provider_policy.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class PreviewCardProviderPolicy < ApplicationPolicy + def index? + staff? + end + + def update? + staff? + end +end diff --git a/app/serializers/rest/trends/link_serializer.rb b/app/serializers/rest/trends/link_serializer.rb new file mode 100644 index 0000000000..2324834908 --- /dev/null +++ b/app/serializers/rest/trends/link_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class REST::Trends::LinkSerializer < REST::PreviewCardSerializer + attributes :history +end diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 51956ce7e0..94dc6389fa 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -50,7 +50,7 @@ class FetchLinkCardService < BaseService # We follow redirects, and ideally we want to save the preview card for # the destination URL and not any link shortener in-between, so here # we set the URL to the one of the last response in the redirect chain - @url = res.request.uri.to_s.to_s + @url = res.request.uri.to_s @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url if res.code == 200 && res.mime_type == 'text/html' @@ -66,6 +66,7 @@ class FetchLinkCardService < BaseService def attach_card @status.preview_cards << @card Rails.cache.delete(@status) + Trends.links.register(@status) end def parse_urls diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 85aaec4d65..294ae43eb2 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -91,7 +91,8 @@ class PostStatusService < BaseService end def postprocess_status! - LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text? + Trends.tags.register(@status) + LinkCrawlWorker.perform_async(@status.id) DistributionWorker.perform_async(@status.id) ActivityPub::DistributionWorker.perform_async(@status.id) PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index c42b79db80..47277c56c0 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -8,7 +8,7 @@ class ProcessHashtagsService < BaseService Tag.find_or_create_by_names(tags) do |tag| status.tags << tag records << tag - tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility? + tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago) end return unless status.distributable? diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 744bdf5673..ece91847a8 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -30,12 +30,13 @@ class ReblogService < BaseService reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit]) + Trends.tags.register(reblog) + Trends.links.register(reblog) DistributionWorker.perform_async(reblog.id) ActivityPub::DistributionWorker.perform_async(reblog.id) create_notification(reblog) bump_potential_friendship(account, reblog) - record_use(account, reblog) reblog end @@ -60,16 +61,6 @@ class ReblogService < BaseService PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog) end - def record_use(account, reblog) - return unless reblog.public_visibility? - - original_status = reblog.reblog - - original_status.tags.each do |tag| - tag.use!(account) - end - end - def build_json(reblog) Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account)) end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 560eba7b4d..895333a585 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -42,7 +42,7 @@ %span= t('admin.dashboard.pending_users_html', count: @pending_users_count) = fa_icon 'chevron-right fw' - = link_to admin_tags_path(pending_review: '1'), class: 'dashboard__quick-access' do + = link_to admin_trends_tags_path(status: 'pending_review'), class: 'dashboard__quick-access' do %span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count) = fa_icon 'chevron-right fw' diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml deleted file mode 100644 index ac0c728165..0000000000 --- a/app/views/admin/tags/_tag.html.haml +++ /dev/null @@ -1,19 +0,0 @@ -.batch-table__row - - if batch_available - %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox - = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id - - .directory__tag - = link_to admin_tag_path(tag.id) do - %h4 - = fa_icon 'hashtag' - = tag.name - - %small - = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts]) - - - if tag.trending? - = fa_icon 'fire fw' - = t('admin.tags.trending_right_now') - - .trends__item__current= friendly_number_to_human tag.history.first[:uses] diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml deleted file mode 100644 index d78f3c6d16..0000000000 --- a/app/views/admin/tags/index.html.haml +++ /dev/null @@ -1,74 +0,0 @@ -- content_for :page_title do - = t('admin.tags.title') - -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - -.filters - .filter-subset - %strong= t('admin.tags.review') - %ul - %li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil - %li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil - %li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil - %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil - - .filter-subset - %strong= t('generic.order_by') - %ul - %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil - %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil - %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil - - -= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do - .fields-group - - TagFilter::KEYS.each do |key| - = hidden_field_tag key, params[key] if params[key].present? - - - %i(name).each do |key| - .input.string.optional - = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}") - - .actions - %button.button= t('admin.accounts.search') - = link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative' - -%hr.spacer/ - -= form_for(@form, url: batch_admin_tags_path) do |f| - = hidden_field_tag :page, params[:page] || 1 - - - TagFilter::KEYS.each do |key| - = hidden_field_tag key, params[key] if params[key].present? - - .batch-table.optional - .batch-table__toolbar - - if params[:pending_review] == '1' || params[:unreviewed] == '1' - %label.batch-table__toolbar__select.batch-checkbox-all - = check_box_tag :batch_checkbox_all, nil, false - .batch-table__toolbar__actions - = f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - - = f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - - else - .batch-table__toolbar__actions - %span.neutral-hint= t('generic.no_batch_actions_available') - - .batch-table__body - - if @tags.empty? - = nothing_here 'nothing-here--under-tabs' - - else - = render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' } - -= paginate @tags - -- if params[:pending_review] == '1' || params[:unreviewed] == '1' - %hr.spacer/ - - %div.action-buttons - %div - = link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button' - - %div - = link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive' diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml index c4caffda1e..007dc005e0 100644 --- a/app/views/admin/tags/show.html.haml +++ b/app/views/admin/tags/show.html.haml @@ -1,15 +1,50 @@ +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + - content_for :page_title do = "##{@tag.name}" -.dashboard__counters - %div - = link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do - .dashboard__counters__num= number_with_delimiter @accounts_today - .dashboard__counters__label= t 'admin.tags.accounts_today' - %div - %div - .dashboard__counters__num= number_with_delimiter @accounts_week - .dashboard__counters__label= t 'admin.tags.accounts_week' +- content_for :heading_actions do + = l(@time_period.first) + = ' - ' + = l(@time_period.last) + +.dashboard + .dashboard__item + = react_admin_component :counter, measure: 'tag_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_accounts_measure') + .dashboard__item + = react_admin_component :counter, measure: 'tag_uses', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_uses_measure') + .dashboard__item + = react_admin_component :counter, measure: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_servers_measure') + .dashboard__item + = react_admin_component :dimension, dimension: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_servers_dimension') + .dashboard__item + = react_admin_component :dimension, dimension: 'tag_languages', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_languages_dimension') + .dashboard__item + = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.usable? ? 'positive' : 'negative'] do + - if @tag.usable? + %span= t('admin.trends.tags.usable') + = fa_icon 'check fw' + - else + %span= t('admin.trends.tags.not_usable') + = fa_icon 'lock fw' + + = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.trendable? ? 'positive' : 'negative'] do + - if @tag.trendable? + %span= t('admin.trends.tags.trendable') + = fa_icon 'check fw' + - else + %span= t('admin.trends.tags.not_trendable') + = fa_icon 'lock fw' + + + = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.listable? ? 'positive' : 'negative'] do + - if @tag.listable? + %span= t('admin.trends.tags.listable') + = fa_icon 'check fw' + - else + %span= t('admin.trends.tags.not_listable') + = fa_icon 'lock fw' %hr.spacer/ @@ -26,18 +61,3 @@ .actions = f.button :button, t('generic.save_changes'), type: :submit - -%hr.spacer/ - -%h3= t 'admin.tags.breakdown' - -.table-wrapper - %table.table - %tbody - - total = @usage_by_domain.sum(&:last).to_f - - - @usage_by_domain.each do |(domain, count)| - %tr - %th= domain || site_hostname - %td= number_to_percentage((count / total) * 100, precision: 1) - %td= number_with_delimiter count diff --git a/app/views/admin/trends/links/_preview_card.html.haml b/app/views/admin/trends/links/_preview_card.html.haml new file mode 100644 index 0000000000..dfed13b68d --- /dev/null +++ b/app/views/admin/trends/links/_preview_card.html.haml @@ -0,0 +1,30 @@ +.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id + + .batch-table__row__content.pending-account + .pending-account__header + = link_to preview_card.title, preview_card.url + + %br/ + + - if preview_card.provider_name.present? + = preview_card.provider_name + • + + - if preview_card.language.present? + = human_locale(preview_card.language) + • + + = t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts }) + + - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id)) + • + %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) + + - if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago + • + = t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short)) + - elsif preview_card.provider&.requires_review? + • + = t('admin.trends.pending_review') diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml new file mode 100644 index 0000000000..240ae722b5 --- /dev/null +++ b/app/views/admin/trends/links/index.html.haml @@ -0,0 +1,41 @@ +- content_for :page_title do + = t('admin.trends.links.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.filters + .filter-subset + %strong= t('admin.trends.trending') + %ul + %li= filter_link_to t('generic.all'), trending: nil + %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' + .back-link + = link_to admin_trends_links_preview_card_providers_path do + = t('admin.trends.preview_card_providers.title') + = fa_icon 'chevron-right fw' + +%hr.spacer/ + += form_for(@form, url: batch_admin_trends_links_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - PreviewCardFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @preview_cards.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'preview_card', collection: @preview_cards, locals: { f: f } + += paginate @preview_cards diff --git a/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml b/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml new file mode 100644 index 0000000000..e40e6529d5 --- /dev/null +++ b/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml @@ -0,0 +1,16 @@ +.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id + + .batch-table__row__content.pending-account + .pending-account__header + %strong= preview_card_provider.domain + + %br/ + + - if preview_card_provider.requires_review? + = t('admin.trends.pending_review') + - elsif preview_card_provider.trendable? + = t('admin.trends.preview_card_providers.allowed') + - else + = t('admin.trends.preview_card_providers.rejected') diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml new file mode 100644 index 0000000000..eac6e641fe --- /dev/null +++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml @@ -0,0 +1,43 @@ +- content_for :page_title do + = t('admin.trends.preview_card_providers.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.filters + .filter-subset + %strong= t('admin.tags.review') + %ul + %li= filter_link_to t('generic.all'), status: nil + %li= filter_link_to t('admin.trends.approved'), status: 'approved' + %li= filter_link_to t('admin.trends.rejected'), status: 'rejected' + %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review' + .back-link + = link_to admin_trends_links_path do + = fa_icon 'chevron-left fw' + = t('admin.trends.links.title') + + +%hr.spacer/ + += form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - PreviewCardProviderFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table.optional + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + + .batch-table__body + - if @preview_card_providers.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f } + += paginate @preview_card_providers diff --git a/app/views/admin/trends/tags/_tag.html.haml b/app/views/admin/trends/tags/_tag.html.haml new file mode 100644 index 0000000000..c4af77b004 --- /dev/null +++ b/app/views/admin/trends/tags/_tag.html.haml @@ -0,0 +1,24 @@ +.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id + + .batch-table__row__content.pending-account + .pending-account__header + = link_to admin_tag_path(tag.id) do + = fa_icon 'hashtag' + = tag.name + + %br/ + + = t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts }) + + - if tag.trendable? && (rank = Trends.tags.rank(tag.id)) + • + %abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) + + - if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago + • + = t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short)) + - elsif tag.requires_review? + • + = t('admin.trends.pending_review') diff --git a/app/views/admin/trends/tags/index.html.haml b/app/views/admin/trends/tags/index.html.haml new file mode 100644 index 0000000000..8df0a9920b --- /dev/null +++ b/app/views/admin/trends/tags/index.html.haml @@ -0,0 +1,38 @@ +- content_for :page_title do + = t('admin.trends.tags.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.filters + .filter-subset + %strong= t('admin.tags.review') + %ul + %li= filter_link_to t('generic.all'), status: nil + %li= filter_link_to t('admin.trends.approved'), status: 'approved' + %li= filter_link_to t('admin.trends.rejected'), status: 'rejected' + %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review' + +%hr.spacer/ + += form_for(@form, url: batch_admin_trends_tags_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - TagFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table.optional + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + + .batch-table__body + - if @tags.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'tag', collection: @tags, locals: { f: f } + += paginate @tags diff --git a/app/views/admin_mailer/new_trending_links.text.erb b/app/views/admin_mailer/new_trending_links.text.erb new file mode 100644 index 0000000000..51789aca5c --- /dev/null +++ b/app/views/admin_mailer/new_trending_links.text.erb @@ -0,0 +1,16 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trending_links.body') %> + +<% @links.each do |link| %> +- <%= link.title %> • <%= link.url %> + <%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> +<% end %> + +<% if @lowest_trending_link %> +<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %> +<% else %> +<%= t('admin_mailer.new_trending_links.no_approved_links') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb deleted file mode 100644 index e4bfdc5912..0000000000 --- a/app/views/admin_mailer/new_trending_tag.text.erb +++ /dev/null @@ -1,5 +0,0 @@ -<%= raw t('application_mailer.salutation', name: display_name(@me)) %> - -<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %> - -<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %> diff --git a/app/views/admin_mailer/new_trending_tags.text.erb b/app/views/admin_mailer/new_trending_tags.text.erb new file mode 100644 index 0000000000..5051e8a969 --- /dev/null +++ b/app/views/admin_mailer/new_trending_tags.text.erb @@ -0,0 +1,16 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trending_tags.body') %> + +<% @tags.each do |tag| %> +- #<%= tag.name %> + <%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> +<% end %> + +<% if @lowest_trending_tag %> +<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %> +<% else %> +<%= t('admin_mailer.new_trending_tags.no_approved_tags') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %> diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml index 7ec91c06aa..6826c3b587 100644 --- a/app/views/application/_sidebar.html.haml +++ b/app/views/application/_sidebar.html.haml @@ -6,7 +6,7 @@ %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html') - if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - - trends = TrendingTags.get(3) + - trends = Trends.tags.get(true, 3) - unless trends.empty? .endorsements-widget.trends-widget diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trends/refresh_scheduler.rb similarity index 57% rename from app/workers/scheduler/trending_tags_scheduler.rb rename to app/workers/scheduler/trends/refresh_scheduler.rb index 94d76d010d..b559ba46b4 100644 --- a/app/workers/scheduler/trending_tags_scheduler.rb +++ b/app/workers/scheduler/trends/refresh_scheduler.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -class Scheduler::TrendingTagsScheduler +class Scheduler::Trends::RefreshScheduler include Sidekiq::Worker sidekiq_options retry: 0 def perform - TrendingTags.update! if Setting.trends + Trends.refresh! end end diff --git a/app/workers/scheduler/trends/review_notifications_scheduler.rb b/app/workers/scheduler/trends/review_notifications_scheduler.rb new file mode 100644 index 0000000000..f334261bd7 --- /dev/null +++ b/app/workers/scheduler/trends/review_notifications_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::Trends::ReviewNotificationsScheduler + include Sidekiq::Worker + + sidekiq_options retry: 0 + + def perform + Trends.request_review! + end +end diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 35f2c31788..c032e5412a 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -67,7 +67,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/account.rb", - "line": 479, + "line": 484, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "find_by_sql([\" WITH first_degree AS (\\n SELECT target_account_id\\n FROM follows\\n WHERE account_id = ?\\n UNION ALL\\n SELECT ?\\n )\\n SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)\\n WHERE accounts.id IN (SELECT * FROM first_degree)\\n AND #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, account.id, limit, offset])", "render_path": null, @@ -100,6 +100,26 @@ "confidence": "Weak", "note": "" }, + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "75fcd147b7611763ab6915faf8c5b0709e612b460f27c05c72d8b9bd0a6a77f8", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "lib/mastodon/snowflake.rb", + "line": 87, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "connection.execute(\"CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\nRETURNS bigint AS\\n$$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n$$ LANGUAGE plpgsql VOLATILE;\\n\")", + "render_path": null, + "location": { + "type": "method", + "class": "Mastodon::Snowflake", + "method": "define_timestamp_id" + }, + "user_input": "SecureRandom.hex(16)", + "confidence": "Medium", + "note": "" + }, { "warning_type": "Mass Assignment", "warning_code": 105, @@ -140,6 +160,26 @@ "confidence": "High", "note": "" }, + { + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "8c1d8c4b76c1cd3960e90dff999f854a6ff742fcfd8de6c7184ac5a1b1a4d7dd", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/preview_card_filter.rb", + "line": 50, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "PreviewCard.joins(\"join unnest(array[#{(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id\")", + "render_path": null, + "location": { + "type": "method", + "class": "PreviewCardFilter", + "method": "trending_scope" + }, + "user_input": "(Trends.links.currently_trending_ids(true, -1) or Trends.links.currently_trending_ids(false, -1)).map(&:to_i).join(\",\")", + "confidence": "Medium", + "note": "" + }, { "warning_type": "SQL Injection", "warning_code": 0, @@ -147,7 +187,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/account.rb", - "line": 448, + "line": 453, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "find_by_sql([\" SELECT\\n accounts.*,\\n ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, limit, offset])", "render_path": null, @@ -160,26 +200,6 @@ "confidence": "Medium", "note": "" }, - { - "warning_type": "SQL Injection", - "warning_code": 0, - "fingerprint": "9ccb9ba6a6947400e187d515e0bf719d22993d37cfc123c824d7fafa6caa9ac3", - "check_name": "SQL", - "message": "Possible SQL injection", - "file": "lib/mastodon/snowflake.rb", - "line": 87, - "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", - "code": "connection.execute(\" CREATE OR REPLACE FUNCTION timestamp_id(table_name text)\\n RETURNS bigint AS\\n $$\\n DECLARE\\n time_part bigint;\\n sequence_base bigint;\\n tail bigint;\\n BEGIN\\n time_part := (\\n -- Get the time in milliseconds\\n ((date_part('epoch', now()) * 1000))::bigint\\n -- And shift it over two bytes\\n << 16);\\n\\n sequence_base := (\\n 'x' ||\\n -- Take the first two bytes (four hex characters)\\n substr(\\n -- Of the MD5 hash of the data we documented\\n md5(table_name ||\\n '#{SecureRandom.hex(16)}' ||\\n time_part::text\\n ),\\n 1, 4\\n )\\n -- And turn it into a bigint\\n )::bit(16)::bigint;\\n\\n -- Finally, add our sequence number to our base, and chop\\n -- it to the last two bytes\\n tail := (\\n (sequence_base + nextval(table_name || '_id_seq'))\\n & 65535);\\n\\n -- Return the time part and the sequence part. OR appears\\n -- faster here than addition, but they're equivalent:\\n -- time_part has no trailing two bytes, and tail is only\\n -- the last two bytes.\\n RETURN time_part | tail;\\n END\\n $$ LANGUAGE plpgsql VOLATILE;\\n\")", - "render_path": null, - "location": { - "type": "method", - "class": "Mastodon::Snowflake", - "method": "define_timestamp_id" - }, - "user_input": "SecureRandom.hex(16)", - "confidence": "Medium", - "note": "" - }, { "warning_type": "Redirect", "warning_code": 18, @@ -201,23 +221,53 @@ "note": "" }, { - "warning_type": "Redirect", - "warning_code": 18, - "fingerprint": "ba699ddcc6552c422c4ecd50d2cd217f616a2446659e185a50b05a0f2dad8d33", - "check_name": "Redirect", - "message": "Possible unprotected redirect", - "file": "app/controllers/media_controller.rb", - "line": 20, - "link": "https://brakemanscanner.org/docs/warning_types/redirect/", - "code": "redirect_to(MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original))", + "warning_type": "SQL Injection", + "warning_code": 0, + "fingerprint": "c32a484ccd9da46abd3bc93d08b72029d7dbc0576ccf4e878a9627e9a83cad2e", + "check_name": "SQL", + "message": "Possible SQL injection", + "file": "app/models/tag_filter.rb", + "line": 50, + "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", + "code": "Tag.joins(\"join unnest(array[#{Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id\")", "render_path": null, "location": { "type": "method", - "class": "MediaController", - "method": "show" + "class": "TagFilter", + "method": "trending_scope" }, - "user_input": "MediaAttachment.attached.find_by!(:shortcode => ((params[:id] or params[:medium_id]))).file.url(:original)", - "confidence": "High", + "user_input": "Trends.tags.currently_trending_ids(false, -1).map(&:to_i).join(\",\")", + "confidence": "Medium", + "note": "" + }, + { + "warning_type": "Cross-Site Scripting", + "warning_code": 4, + "fingerprint": "cd5cfd7f40037fbfa753e494d7129df16e358bfc43ef0da3febafbf4ee1ed3ac", + "check_name": "LinkToHref", + "message": "Potentially unsafe model attribute in `link_to` href", + "file": "app/views/admin/trends/links/_preview_card.html.haml", + "line": 7, + "link": "https://brakemanscanner.org/docs/warning_types/link_to_href", + "code": "link_to((Unresolved Model).new.title, (Unresolved Model).new.url)", + "render_path": [ + { + "type": "template", + "name": "admin/trends/links/index", + "line": 37, + "file": "app/views/admin/trends/links/index.html.haml", + "rendered": { + "name": "admin/trends/links/_preview_card", + "file": "app/views/admin/trends/links/_preview_card.html.haml" + } + } + ], + "location": { + "type": "template", + "template": "admin/trends/links/_preview_card" + }, + "user_input": "(Unresolved Model).new.url", + "confidence": "Weak", "note": "" }, { @@ -227,7 +277,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/account.rb", - "line": 495, + "line": 500, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "find_by_sql([\" SELECT\\n accounts.*,\\n (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank\\n FROM accounts\\n LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)\\n WHERE #{query} @@ #{textsearch}\\n AND accounts.suspended_at IS NULL\\n AND accounts.moved_to_account_id IS NULL\\n GROUP BY accounts.id\\n ORDER BY rank DESC\\n LIMIT ? OFFSET ?\\n\".squish, account.id, account.id, limit, offset])", "render_path": null, @@ -261,6 +311,6 @@ "note": "" } ], - "updated": "2021-05-11 20:22:27 +0900", - "brakeman_version": "5.0.1" + "updated": "2021-11-14 05:26:09 +0100", + "brakeman_version": "5.1.2" } diff --git a/config/locales/en.yml b/config/locales/en.yml index be15ad4b05..c98b828015 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -674,8 +674,8 @@ en: desc_html: Affects hashtags that have not been previously disallowed title: Allow hashtags to trend without prior review trends: - desc_html: Publicly display previously reviewed hashtags that are currently trending - title: Trending hashtags + desc_html: Publicly display previously reviewed content that is currently trending + title: Trends site_uploads: delete: Delete uploaded file destroyed_msg: Site upload successfully deleted! @@ -702,21 +702,51 @@ en: sidekiq_process_check: message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration tags: - accounts_today: Unique uses today - accounts_week: Unique uses this week - breakdown: Breakdown of today's usage by source - last_active: Recently used - most_popular: Most popular - most_recent: Recently created - name: Hashtag review: Review status - reviewed: Reviewed - title: Hashtags - trending_right_now: Trending right now - unique_uses_today: "%{count} posting today" - unreviewed: Not reviewed updated_msg: Hashtag settings updated successfully title: Administration + trends: + allow: Allow + approved: Approved + disallow: Disallow + links: + allow: Allow link + allow_provider: Allow publisher + disallow: Disallow link + disallow_provider: Disallow publisher + shared_by_over_week: + one: Shared by one person over the last week + other: Shared by %{count} people over the last week + title: Trending links + usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday + pending_review: Pending review + preview_card_providers: + allowed: Links from this publisher can trend + rejected: Links from this publisher won't trend + title: Publishers + rejected: Rejected + tags: + current_score: Current score %{score} + dashboard: + tag_accounts_measure: unique uses + tag_languages_dimension: Top languages + tag_servers_dimension: Top servers + tag_servers_measure: different servers + tag_uses_measure: total uses + listable: Can be suggested + not_listable: Won't be suggested + not_trendable: Won't appear under trends + not_usable: Cannot be used + peaked_on_and_decaying: Peaked on %{date}, now decaying + title: Trending hashtags + trendable: Can appear under trends + trending_rank: 'Trending #%{rank}' + usable: Can be used + usage_comparison: Used %{today} times today, compared to %{yesterday} yesterday + used_by_over_week: + one: Used by one person over the last week + other: Used by %{count} people over the last week + title: Trends warning_presets: add_new: Add new delete: Delete @@ -731,9 +761,16 @@ en: body: "%{reporter} has reported %{target}" body_remote: Someone from %{domain} has reported %{target} subject: New report for %{instance} (#%{id}) - new_trending_tag: - body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.' - subject: New hashtag up for review on %{instance} (#%{name}) + new_trending_links: + body: The following links are trending today, but their publishers have not been previously reviewed. They will not be displayed publicly unless you approve them. Further notifications from the same publishers will not be generated. + no_approved_links: There are currently no approved trending links. + requirements: The lowest approved trending link is currently "%{lowest_link_title}" with a score of %{lowest_link_score}. + subject: New trending links up for review on %{instance} + new_trending_tags: + body: 'The following hashtags are trending today, but they have not been previously reviewed. They will not be displayed publicly unless you approve them:' + no_approved_tags: There are currently no approved trending hashtags. + requirements: 'The lowest approved trending hashtag is currently #%{lowest_tag_name} with a score of %{lowest_tag_score}.' + subject: New trending hashtags up for review on %{instance} aliases: add_new: Create alias created_msg: Successfully created a new alias. You can now initiate the move from the old account. @@ -940,7 +977,7 @@ en: changes_saved_msg: Changes successfully saved! copy: Copy delete: Delete - no_batch_actions_available: No batch actions available on this page + none: None order_by: Order by save_changes: Save changes validation_errors: diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index bf864748ce..d6376782d4 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -204,8 +204,8 @@ en: mention: Someone mentioned you pending_account: New account needs review reblog: Someone boosted your post - report: New report is submitted - trending_tag: An unreviewed hashtag is trending + report: A new report is submitted + trending_tag: A new trend requires approval rule: text: Rule tag: diff --git a/config/navigation.rb b/config/navigation.rb index 37bfd7549b..477d1c9ffb 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -34,12 +34,16 @@ SimpleNavigation::Configuration.run do |navigation| n.item :invites, safe_join([fa_icon('user-plus fw'), t('invites.title')]), invites_path, if: proc { Setting.min_invite_role == 'user' && current_user.functional? } n.item :development, safe_join([fa_icon('code fw'), t('settings.development')]), settings_applications_url, if: -> { current_user.functional? } + n.item :trends, safe_join([fa_icon('fire fw'), t('admin.trends.title')]), admin_trends_tags_path, if: proc { current_user.staff? } do |s| + s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.trends.tags.title')]), admin_trends_tags_path, highlights_on: %r{/admin/tags|/admin/trends/tags} + s.item :links, safe_join([fa_icon('newspaper-o fw'), t('admin.trends.links.title')]), admin_trends_links_path, highlights_on: %r{/admin/trends/links} + end + n.item :moderation, safe_join([fa_icon('gavel fw'), t('moderation.title')]), admin_reports_url, if: proc { current_user.staff? } do |s| s.item :action_logs, safe_join([fa_icon('bars fw'), t('admin.action_logs.title')]), admin_action_logs_url s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports} s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts} s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path - s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags} s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations} s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? } s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? } diff --git a/config/routes.rb b/config/routes.rb index 86f699516e..c7317d1732 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -301,12 +301,27 @@ Rails.application.routes.draw do resources :account_moderation_notes, only: [:create, :destroy] resource :follow_recommendations, only: [:show, :update] + resources :tags, only: [:show, :update] - resources :tags, only: [:index, :show, :update] do - collection do - post :approve_all - post :reject_all - post :batch + namespace :trends do + resources :links, only: [:index] do + collection do + post :batch + end + end + + resources :tags, only: [:index] do + collection do + post :batch + end + end + + namespace :links do + resources :preview_card_providers, only: [:index], path: :publishers do + collection do + post :batch + end + end end end end @@ -399,7 +414,7 @@ Rails.application.routes.draw do resources :favourites, only: [:index] resources :bookmarks, only: [:index] resources :reports, only: [:create] - resources :trends, only: [:index] + resources :trends, only: [:index], controller: 'trends/tags' resources :filters, only: [:index, :create, :show, :update, :destroy] resources :endorsements, only: [:index] resources :markers, only: [:index, :create] @@ -410,6 +425,11 @@ Rails.application.routes.draw do resources :apps, only: [:create] + namespace :trends do + resources :links, only: [:index] + resources :tags, only: [:index] + end + namespace :emails do resources :confirmations, only: [:create] end @@ -512,7 +532,9 @@ Rails.application.routes.draw do end end - resources :trends, only: [:index] + namespace :trends do + resources :tags, only: [:index] + end post :measures, to: 'measures#create' post :dimensions, to: 'dimensions#create' diff --git a/config/sidekiq.yml b/config/sidekiq.yml index eab74338e0..9dde5a053b 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -13,9 +13,13 @@ every: '5m' class: Scheduler::ScheduledStatusesScheduler queue: scheduler - trending_tags_scheduler: + trends_refresh_scheduler: every: '5m' - class: Scheduler::TrendingTagsScheduler + class: Scheduler::Trends::RefreshScheduler + queue: scheduler + trends_review_notifications_scheduler: + every: '2h' + class: Scheduler::Trends::ReviewNotificationsScheduler queue: scheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' diff --git a/db/migrate/20211031031021_create_preview_card_providers.rb b/db/migrate/20211031031021_create_preview_card_providers.rb new file mode 100644 index 0000000000..0bd46198e2 --- /dev/null +++ b/db/migrate/20211031031021_create_preview_card_providers.rb @@ -0,0 +1,12 @@ +class CreatePreviewCardProviders < ActiveRecord::Migration[6.1] + def change + create_table :preview_card_providers do |t| + t.string :domain, null: false, default: '', index: { unique: true } + t.attachment :icon + t.boolean :trendable + t.datetime :reviewed_at + t.datetime :requested_review_at + t.timestamps + end + end +end diff --git a/db/migrate/20211112011713_add_language_to_preview_cards.rb b/db/migrate/20211112011713_add_language_to_preview_cards.rb new file mode 100644 index 0000000000..995934de4c --- /dev/null +++ b/db/migrate/20211112011713_add_language_to_preview_cards.rb @@ -0,0 +1,7 @@ +class AddLanguageToPreviewCards < ActiveRecord::Migration[6.1] + def change + add_column :preview_cards, :language, :string + add_column :preview_cards, :max_score, :float + add_column :preview_cards, :max_score_at, :datetime + end +end diff --git a/db/migrate/20211115032527_add_trendable_to_preview_cards.rb b/db/migrate/20211115032527_add_trendable_to_preview_cards.rb new file mode 100644 index 0000000000..87bf3d7a23 --- /dev/null +++ b/db/migrate/20211115032527_add_trendable_to_preview_cards.rb @@ -0,0 +1,5 @@ +class AddTrendableToPreviewCards < ActiveRecord::Migration[6.1] + def change + add_column :preview_cards, :trendable, :boolean + end +end diff --git a/db/migrate/20211123212714_add_link_type_to_preview_cards.rb b/db/migrate/20211123212714_add_link_type_to_preview_cards.rb new file mode 100644 index 0000000000..9f57e02194 --- /dev/null +++ b/db/migrate/20211123212714_add_link_type_to_preview_cards.rb @@ -0,0 +1,5 @@ +class AddLinkTypeToPreviewCards < ActiveRecord::Migration[6.1] + def change + add_column :preview_cards, :link_type, :int + end +end diff --git a/db/schema.rb b/db/schema.rb index 2376afff74..00969daf10 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_08_08_071221) do +ActiveRecord::Schema.define(version: 2021_11_23_212714) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -689,6 +689,20 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do t.index ["status_id"], name: "index_polls_on_status_id" end + create_table "preview_card_providers", force: :cascade do |t| + t.string "domain", default: "", null: false + t.string "icon_file_name" + t.string "icon_content_type" + t.bigint "icon_file_size" + t.datetime "icon_updated_at" + t.boolean "trendable" + t.datetime "reviewed_at" + t.datetime "requested_review_at" + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["domain"], name: "index_preview_card_providers_on_domain", unique: true + end + create_table "preview_cards", force: :cascade do |t| t.string "url", default: "", null: false t.string "title", default: "", null: false @@ -710,6 +724,11 @@ ActiveRecord::Schema.define(version: 2021_08_08_071221) do t.string "embed_url", default: "", null: false t.integer "image_storage_schema_version" t.string "blurhash" + t.string "language" + t.float "max_score" + t.datetime "max_score_at" + t.boolean "trendable" + t.integer "link_type" t.index ["url"], name: "index_preview_cards_on_url", unique: true end diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb index 8e2d82a974..fe0dc1722e 100644 --- a/lib/mastodon/snowflake.rb +++ b/lib/mastodon/snowflake.rb @@ -84,10 +84,7 @@ module Mastodon::Snowflake -- Take the first two bytes (four hex characters) substr( -- Of the MD5 hash of the data we documented - md5(table_name || - '#{SecureRandom.hex(16)}' || - time_part::text - ), + md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text), 1, 4 ) -- And turn it into a bigint diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake index d004c5751b..bbf7f20ee7 100644 --- a/lib/tasks/repo.rake +++ b/lib/tasks/repo.rake @@ -96,7 +96,7 @@ namespace :repo do end.uniq.compact missing_available_locales = locales_in_files - I18n.available_locales - missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) } + missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) } critical = false diff --git a/spec/controllers/admin/tags_controller_spec.rb b/spec/controllers/admin/tags_controller_spec.rb index 9145d887de..85c801a9c7 100644 --- a/spec/controllers/admin/tags_controller_spec.rb +++ b/spec/controllers/admin/tags_controller_spec.rb @@ -9,18 +9,6 @@ RSpec.describe Admin::TagsController, type: :controller do sign_in Fabricate(:user, admin: true) end - describe 'GET #index' do - let!(:tag) { Fabricate(:tag) } - - before do - get :index - end - - it 'returns status 200' do - expect(response).to have_http_status(200) - end - end - describe 'GET #show' do let!(:tag) { Fabricate(:tag) } diff --git a/spec/controllers/api/v1/trends/tags_controller_spec.rb b/spec/controllers/api/v1/trends/tags_controller_spec.rb new file mode 100644 index 0000000000..e2e26dcab9 --- /dev/null +++ b/spec/controllers/api/v1/trends/tags_controller_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe Api::V1::Trends::TagsController, type: :controller do + render_views + + describe 'GET #index' do + before do + trending_tags = double() + + allow(trending_tags).to receive(:get).and_return(Fabricate.times(10, :tag)) + allow(Trends).to receive(:tags).and_return(trending_tags) + + get :index + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + end +end diff --git a/spec/controllers/api/v1/trends_controller_spec.rb b/spec/controllers/api/v1/trends_controller_spec.rb deleted file mode 100644 index 91e0d18fe7..0000000000 --- a/spec/controllers/api/v1/trends_controller_spec.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe Api::V1::TrendsController, type: :controller do - render_views - - describe 'GET #index' do - before do - allow(TrendingTags).to receive(:get).and_return(Fabricate.times(10, :tag)) - get :index - end - - it 'returns http success' do - expect(response).to have_http_status(200) - end - end -end diff --git a/spec/helpers/settings_helper_spec.rb b/spec/helpers/languages_helper_spec.rb similarity index 56% rename from spec/helpers/settings_helper_spec.rb rename to spec/helpers/languages_helper_spec.rb index 092c375836..6db617824d 100644 --- a/spec/helpers/settings_helper_spec.rb +++ b/spec/helpers/languages_helper_spec.rb @@ -2,20 +2,15 @@ require 'rails_helper' -describe SettingsHelper do +describe LanguagesHelper do describe 'the HUMAN_LOCALES constant' do it 'includes all I18n locales' do - options = I18n.available_locales - - expect(described_class::HUMAN_LOCALES.keys).to include(*options) + expect(described_class::HUMAN_LOCALES.keys).to include(*I18n.available_locales) end end describe 'human_locale' do it 'finds the human readable local description from a key' do - # Ensure the value is as we expect - expect(described_class::HUMAN_LOCALES[:en]).to eq('English') - expect(helper.human_locale(:en)).to eq('English') end end diff --git a/spec/mailers/previews/admin_mailer_preview.rb b/spec/mailers/previews/admin_mailer_preview.rb index 561a56b787..75ffbbf40f 100644 --- a/spec/mailers/previews/admin_mailer_preview.rb +++ b/spec/mailers/previews/admin_mailer_preview.rb @@ -5,4 +5,14 @@ class AdminMailerPreview < ActionMailer::Preview def new_pending_account AdminMailer.new_pending_account(Account.first, User.pending.first) end + + # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_tags + def new_trending_tags + AdminMailer.new_trending_tags(Account.first, Tag.limit(3)) + end + + # Preview this email at http://localhost:3000/rails/mailers/admin_mailer/new_trending_links + def new_trending_links + AdminMailer.new_trending_links(Account.first, PreviewCard.limit(3)) + end end diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb deleted file mode 100644 index dfbc7d6f80..0000000000 --- a/spec/models/trending_tags_spec.rb +++ /dev/null @@ -1,68 +0,0 @@ -require 'rails_helper' - -RSpec.describe TrendingTags do - describe '.record_use!' do - pending - end - - describe '.update!' do - let!(:at_time) { Time.now.utc } - let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) } - let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) } - let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) } - - before do - allow(Redis.current).to receive(:pfcount) do |key| - case key - when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" - 2 - when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts" - 16 - when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" - 0 - when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts" - 4 - when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" - 13 - end - end - - Redis.current.zadd('trending_tags', 0.9, tag3.id) - Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id]) - - tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours) - - described_class.update!(at_time) - end - - it 'calculates and re-calculates scores' do - expect(described_class.get(10, filtered: false)).to eq [tag1, tag3] - end - - it 'omits hashtags below threshold' do - expect(described_class.get(10, filtered: false)).to_not include(tag2) - end - - it 'decays scores' do - expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9 - end - end - - describe '.trending?' do - let(:tag) { Fabricate(:tag) } - - before do - 10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) } - end - - it 'returns true if the hashtag is within limit' do - Redis.current.zadd('trending_tags', 11, tag.id) - expect(described_class.trending?(tag)).to be true - end - - it 'returns false if the hashtag is outside the limit' do - Redis.current.zadd('trending_tags', 0, tag.id) - expect(described_class.trending?(tag)).to be false - end - end -end diff --git a/spec/models/trends/tags_spec.rb b/spec/models/trends/tags_spec.rb new file mode 100644 index 0000000000..4f98c6aa4c --- /dev/null +++ b/spec/models/trends/tags_spec.rb @@ -0,0 +1,67 @@ +require 'rails_helper' + +RSpec.describe Trends::Tags do + subject { described_class.new(threshold: 5, review_threshold: 10) } + + let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) } + + describe '#add' do + let(:tag) { Fabricate(:tag) } + + before do + subject.add(tag, 1, at_time) + end + + it 'records history' do + expect(tag.history.get(at_time).accounts).to eq 1 + end + + it 'records use' do + expect(subject.send(:recently_used_ids, at_time)).to eq [tag.id] + end + end + + describe '#get' do + pending + end + + describe '#refresh' do + let!(:today) { at_time } + let!(:yesterday) { today - 1.day } + + let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) } + let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) } + let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) } + + before do + 2.times { |i| subject.add(tag1, i, yesterday) } + 13.times { |i| subject.add(tag3, i, yesterday) } + 16.times { |i| subject.add(tag1, i, today) } + 4.times { |i| subject.add(tag2, i, today) } + end + + context do + before do + subject.refresh(yesterday + 12.hours) + subject.refresh(at_time) + end + + it 'calculates and re-calculates scores' do + expect(subject.get(false, 10)).to eq [tag1, tag3] + end + + it 'omits hashtags below threshold' do + expect(subject.get(false, 10)).to_not include(tag2) + end + end + + it 'decays scores' do + subject.refresh(yesterday + 12.hours) + original_score = subject.score(tag3.id) + expect(original_score).to eq 144.0 + subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife]) + decayed_score = subject.score(tag3.id) + expect(decayed_score).to be <= original_score / 2 + end + end +end From 013bee6afb9f76c354e5f8d096d9c85d1ac484ce Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:30 +0100 Subject: [PATCH 2/6] Fix filtering DMs from non-followed users (#17042) --- app/services/notify_service.rb | 43 +++++++++++++++++++++++++++- spec/services/notify_service_spec.rb | 17 +++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index a1b5ca1e37..09e28b76b0 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -67,8 +67,49 @@ class NotifyService < BaseService message? && @notification.target_status.direct_visibility? end + # Returns true if the sender has been mentionned by the recipient up the thread def response_to_recipient? - @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility? + return false if @notification.target_status.in_reply_to_id.nil? + + # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads + !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero? + WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender) AS ( + SELECT + s.id, s.in_reply_to_id, (CASE + WHEN s.account_id = :recipient_id THEN + EXISTS ( + SELECT * + FROM mentions m + WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id + ) + ELSE + FALSE + END) + FROM statuses s + WHERE s.id = :id + UNION ALL + SELECT + s.id, + s.in_reply_to_id, + (CASE + WHEN s.account_id = :recipient_id THEN + EXISTS ( + SELECT * + FROM mentions m + WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id + ) + ELSE + FALSE + END) + FROM ancestors st + JOIN statuses s ON s.id = st.in_reply_to_id + WHERE st.replying_to_sender IS FALSE + ) + SELECT COUNT(*) + FROM ancestors st + JOIN statuses s ON s.id = st.id + WHERE st.replying_to_sender IS TRUE AND s.visibility = 3 + SQL end def from_staff? diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index f2cb22c5ef..7433866b76 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -64,8 +64,9 @@ RSpec.describe NotifyService, type: :service do is_expected.to_not change(Notification, :count) end - context 'if the message chain initiated by recipient, but is not direct message' do + context 'if the message chain is initiated by recipient, but is not direct message' do let(:reply_to) { Fabricate(:status, account: recipient) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does not notify' do @@ -73,8 +74,20 @@ RSpec.describe NotifyService, type: :service do end end - context 'if the message chain initiated by recipient and is direct message' do + context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do + let(:reply_to) { Fabricate(:status, account: recipient) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } + let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) } + let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } + + it 'does not notify' do + is_expected.to_not change(Notification, :count) + end + end + + context 'if the message chain is initiated by the recipient with a mention to the sender' do let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does notify' do From b1fd6d44901a13450d22884b02eb6e9ae4fc1248 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:39 +0100 Subject: [PATCH 3/6] Fix handling of recursive toots in WebUI (#17041) --- app/javascript/mastodon/features/status/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js index 055963d147..f342a3641c 100644 --- a/app/javascript/mastodon/features/status/index.js +++ b/app/javascript/mastodon/features/status/index.js @@ -83,7 +83,7 @@ const makeMapStateToProps = () => { ancestorsIds = ancestorsIds.withMutations(mutable => { let id = statusId; - while (id) { + while (id && !mutable.includes(id)) { mutable.unshift(id); id = inReplyTos.get(id); } @@ -101,7 +101,7 @@ const makeMapStateToProps = () => { const ids = [statusId]; while (ids.length > 0) { - let id = ids.shift(); + let id = ids.pop(); const replies = contextReplies.get(id); if (statusId !== id) { @@ -110,7 +110,7 @@ const makeMapStateToProps = () => { if (replies) { replies.reverse().forEach(reply => { - ids.unshift(reply); + if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply); }); } } From 67c4e8e317aabef88e7633c9a5534613d9363983 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:39 +0100 Subject: [PATCH 4/6] [Glitch] Fix handling of recursive toots in WebUI Port b1fd6d44901a13450d22884b02eb6e9ae4fc1248 to glitch-soc Signed-off-by: Claire --- app/javascript/flavours/glitch/features/status/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/javascript/flavours/glitch/features/status/index.js b/app/javascript/flavours/glitch/features/status/index.js index bddf1a2efa..12ea407ad3 100644 --- a/app/javascript/flavours/glitch/features/status/index.js +++ b/app/javascript/flavours/glitch/features/status/index.js @@ -70,7 +70,7 @@ const makeMapStateToProps = () => { ancestorsIds = ancestorsIds.withMutations(mutable => { let id = statusId; - while (id) { + while (id && !mutable.includes(id)) { mutable.unshift(id); id = inReplyTos.get(id); } @@ -88,7 +88,7 @@ const makeMapStateToProps = () => { const ids = [statusId]; while (ids.length > 0) { - let id = ids.shift(); + let id = ids.pop(); const replies = contextReplies.get(id); if (statusId !== id) { @@ -97,7 +97,7 @@ const makeMapStateToProps = () => { if (replies) { replies.reverse().forEach(reply => { - ids.unshift(reply); + if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply); }); } } From 0ccaedc92bff79fa2922c700dd7dc42378b99090 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:53:28 +0100 Subject: [PATCH 5/6] Disable trending links review request emails --- app/models/trends.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/trends.rb b/app/models/trends.rb index 7dd3a9c87e..8f8cb02612 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -18,7 +18,7 @@ module Trends end def self.request_review! - [links, tags].each(&:request_review) if enabled? + [tags].each(&:request_review) if enabled? end def self.enabled? From 5f10e64330635bfd609ba5acdd78fa505c12f5b1 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 26 Nov 2021 00:26:06 +0100 Subject: [PATCH 6/6] Fix trends admin page crashing --- app/models/trends/base.rb | 13 ------------- app/models/trends/links.rb | 25 +++++++++++++++---------- app/models/trends/tags.rb | 25 +++++++++++++++---------- spec/models/trends/tags_spec.rb | 9 +++++++-- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index b767dcb1ac..788f128a04 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -3,19 +3,6 @@ class Trends::Base include Redisable - class_attribute :default_options - - attr_reader :options - - # @param [Hash] options - # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score - # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review - # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from - # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays - def initialize(options = {}) - @options = self.class.default_options.merge(options) - end - def register(_status) raise NotImplementedError end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index a0d65138ba..a7cd9e29db 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -3,12 +3,17 @@ class Trends::Links < Trends::Base PREFIX = 'trending_links' - self.default_options = { - threshold: 15, - review_threshold: 10, - max_score_cooldown: 2.days.freeze, - max_score_halflife: 8.hours.freeze, - } + # Minimum amount of uses by unique accounts to begin calculating the score + THRESHOLD = 15 + + # Minimum rank (lower = better) before requesting a review + REVIEW_THRESHOLD = 10 + + # For this amount of time, the peak score (if bigger than current score) is decayed-from + MAX_SCORE_COOLDOWN = 2.days.freeze + + # How quickly a peak score decays + MAX_SCORE_HALFLIFE = 8.hours.freeze def register(status, at_time = Time.now.utc) original_status = status.reblog? ? status.reblog : status @@ -76,10 +81,10 @@ class Trends::Links < Trends::Base observed = preview_card.history.get(at_time).accounts.to_f max_time = preview_card.max_score_at max_score = preview_card.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) score = begin - if expected > observed || observed < options[:threshold] + if expected > observed || observed < THRESHOLD 0 else ((observed - expected)**2) / expected @@ -94,7 +99,7 @@ class Trends::Links < Trends::Base preview_card.update_columns(max_score: max_score, max_score_at: max_time) end - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) if decaying_score.zero? redis.zrem("#{PREFIX}:all", preview_card.id) @@ -112,6 +117,6 @@ class Trends::Links < Trends::Base end def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) + score(id) > score_at_rank(REVIEW_THRESHOLD - 1) end end diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb index 13e0ab56b4..027e3dbac1 100644 --- a/app/models/trends/tags.rb +++ b/app/models/trends/tags.rb @@ -3,12 +3,17 @@ class Trends::Tags < Trends::Base PREFIX = 'trending_tags' - self.default_options = { - threshold: 15, - review_threshold: 10, - max_score_cooldown: 2.days.freeze, - max_score_halflife: 4.hours.freeze, - } + # Minimum amount of uses by unique accounts to begin calculating the score + THRESHOLD = 15 + + # Minimum rank (lower = better) before requesting a review + REVIEW_THRESHOLD = 10 + + # For this amount of time, the peak score (if bigger than current score) is decayed-from + MAX_SCORE_COOLDOWN = 2.days.freeze + + # How quickly a peak score decays + MAX_SCORE_HALFLIFE = 4.hours.freeze def register(status, at_time = Time.now.utc) original_status = status.reblog? ? status.reblog : status @@ -70,10 +75,10 @@ class Trends::Tags < Trends::Base observed = tag.history.get(at_time).accounts.to_f max_time = tag.max_score_at max_score = tag.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) score = begin - if expected > observed || observed < options[:threshold] + if expected > observed || observed < THRESHOLD 0 else ((observed - expected)**2) / expected @@ -88,7 +93,7 @@ class Trends::Tags < Trends::Base tag.update_columns(max_score: max_score, max_score_at: max_time) end - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) if decaying_score.zero? redis.zrem("#{PREFIX}:all", tag.id) @@ -106,6 +111,6 @@ class Trends::Tags < Trends::Base end def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) + score(id) > score_at_rank(REVIEW_THRESHOLD - 1) end end diff --git a/spec/models/trends/tags_spec.rb b/spec/models/trends/tags_spec.rb index 4f98c6aa4c..23f8a7ba70 100644 --- a/spec/models/trends/tags_spec.rb +++ b/spec/models/trends/tags_spec.rb @@ -1,10 +1,15 @@ require 'rails_helper' RSpec.describe Trends::Tags do - subject { described_class.new(threshold: 5, review_threshold: 10) } + subject { described_class.new } let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) } + before do + stub_const 'Trends::Tags::THRESHOLD', 5 + stub_const 'Trends::Tags::REVIEW_THRESHOLD', 10 + end + describe '#add' do let(:tag) { Fabricate(:tag) } @@ -59,7 +64,7 @@ RSpec.describe Trends::Tags do subject.refresh(yesterday + 12.hours) original_score = subject.score(tag3.id) expect(original_score).to eq 144.0 - subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife]) + subject.refresh(yesterday + 12.hours + described_class::MAX_SCORE_HALFLIFE) decayed_score = subject.score(tag3.id) expect(decayed_score).to be <= original_score / 2 end