From b377022cf9dfac4d98d0d10b511aeb65e540e0a3 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 22 Feb 2022 15:27:08 +0100 Subject: [PATCH] Add caching layer to metrics (#17617) --- .../admin/metrics/dimension/base_dimension.rb | 32 +++++++++++- .../metrics/dimension/languages_dimension.rb | 4 +- .../metrics/dimension/servers_dimension.rb | 4 +- .../dimension/software_versions_dimension.rb | 6 +-- .../metrics/dimension/sources_dimension.rb | 4 +- .../dimension/space_usage_dimension.rb | 6 +-- .../dimension/tag_languages_dimension.rb | 6 +-- .../dimension/tag_servers_dimension.rb | 6 +-- .../metrics/measure/active_users_measure.rb | 10 ++-- app/lib/admin/metrics/measure/base_measure.rb | 52 +++++++++++++++++-- .../metrics/measure/interactions_measure.rb | 10 ++-- .../metrics/measure/new_users_measure.rb | 8 +-- .../metrics/measure/opened_reports_measure.rb | 8 +-- .../measure/resolved_reports_measure.rb | 8 +-- .../metrics/measure/tag_accounts_measure.rb | 10 ++-- .../metrics/measure/tag_servers_measure.rb | 10 ++-- .../admin/metrics/measure/tag_uses_measure.rb | 10 ++-- app/lib/admin/metrics/retention.rb | 26 ++++++++++ 18 files changed, 165 insertions(+), 55 deletions(-) diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb index 5872c22cbcc..bd2e4ececbe 100644 --- a/app/lib/admin/metrics/dimension/base_dimension.rb +++ b/app/lib/admin/metrics/dimension/base_dimension.rb @@ -1,23 +1,34 @@ # frozen_string_literal: true class Admin::Metrics::Dimension::BaseDimension + CACHE_TTL = 5.minutes.freeze + def self.with_params? false end + attr_reader :loaded + + alias loaded? loaded + def initialize(start_at, end_at, limit, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime @limit = limit&.to_i @params = params + @loaded = false end def key raise NotImplementedError end + def cache_key + ["metrics/dimension/#{key}", @start_at, @end_at, @limit, canonicalized_params].join(';') + end + def data - raise NotImplementedError + load end def self.model_name @@ -30,11 +41,28 @@ class Admin::Metrics::Dimension::BaseDimension protected + def load + unless loaded? + @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query } + @loaded = true + end + + @values + end + + def perform_query + raise NotImplementedError + end + def time_period (@start_at..@end_at) end def params - raise NotImplementedError + {} + end + + def canonicalized_params + params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';') end end diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb index 1cc5f41207d..f1cf82cf276 100644 --- a/app/lib/admin/metrics/dimension/languages_dimension.rb +++ b/app/lib/admin/metrics/dimension/languages_dimension.rb @@ -7,7 +7,9 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension: 'languages' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT locale, count(*) AS value FROM users diff --git a/app/lib/admin/metrics/dimension/servers_dimension.rb b/app/lib/admin/metrics/dimension/servers_dimension.rb index 3e80b66250f..91bcce65510 100644 --- a/app/lib/admin/metrics/dimension/servers_dimension.rb +++ b/app/lib/admin/metrics/dimension/servers_dimension.rb @@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::ServersDimension < Admin::Metrics::Dimension::B 'servers' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT accounts.domain, count(*) AS value FROM statuses diff --git a/app/lib/admin/metrics/dimension/software_versions_dimension.rb b/app/lib/admin/metrics/dimension/software_versions_dimension.rb index 34917404d14..816615f992e 100644 --- a/app/lib/admin/metrics/dimension/software_versions_dimension.rb +++ b/app/lib/admin/metrics/dimension/software_versions_dimension.rb @@ -7,12 +7,12 @@ class Admin::Metrics::Dimension::SoftwareVersionsDimension < Admin::Metrics::Dim 'software_versions' end - def data + protected + + def perform_query [mastodon_version, ruby_version, postgresql_version, redis_version] end - private - def mastodon_version value = Mastodon::Version.to_s diff --git a/app/lib/admin/metrics/dimension/sources_dimension.rb b/app/lib/admin/metrics/dimension/sources_dimension.rb index a9f061809c8..122807cdcdb 100644 --- a/app/lib/admin/metrics/dimension/sources_dimension.rb +++ b/app/lib/admin/metrics/dimension/sources_dimension.rb @@ -5,7 +5,9 @@ class Admin::Metrics::Dimension::SourcesDimension < Admin::Metrics::Dimension::B 'sources' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT oauth_applications.name, count(*) AS value FROM users diff --git a/app/lib/admin/metrics/dimension/space_usage_dimension.rb b/app/lib/admin/metrics/dimension/space_usage_dimension.rb index aa00a2e18b8..5867c5bab61 100644 --- a/app/lib/admin/metrics/dimension/space_usage_dimension.rb +++ b/app/lib/admin/metrics/dimension/space_usage_dimension.rb @@ -8,12 +8,12 @@ class Admin::Metrics::Dimension::SpaceUsageDimension < Admin::Metrics::Dimension 'space_usage' end - def data + protected + + def perform_query [postgresql_size, redis_size, media_size] end - private - def postgresql_size value = ActiveRecord::Base.connection.execute('SELECT pg_database_size(current_database())').first['pg_database_size'] diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb index afbc8cde8ac..e1349c22943 100644 --- a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb +++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb @@ -11,7 +11,9 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi 'tag_languages' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value FROM statuses @@ -28,8 +30,6 @@ class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimensi rows.map { |row| { key: row['language'], human_key: standard_locale_name(row['language']), value: row['value'].to_s } } end - private - def params @params.permit(:id) end diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb index 12c5980d728..7ddf3378cd4 100644 --- a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb +++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb @@ -9,7 +9,9 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension 'tag_servers' end - def data + protected + + def perform_query sql = <<-SQL.squish SELECT accounts.domain, count(*) AS value FROM statuses @@ -27,8 +29,6 @@ class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } end - private - def params @params.permit(:id) end diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb index 51318978069..e6f09d4bcf6 100644 --- a/app/lib/admin/metrics/measure/active_users_measure.rb +++ b/app/lib/admin/metrics/measure/active_users_measure.rb @@ -5,20 +5,20 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas 'active_users' end - def total + protected + + def perform_total_query activity_tracker.sum(time_period.first, time_period.last) end - def previous_total + def perform_previous_total_query activity_tracker.sum(previous_time_period.first, previous_time_period.last) end - def data + def perform_data_query activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } end - protected - def activity_tracker @activity_tracker ||= ActivityTracker.new('activity:logins', :unique) end diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb index 0107ffd9c15..ed1df9c7d9f 100644 --- a/app/lib/admin/metrics/measure/base_measure.rb +++ b/app/lib/admin/metrics/measure/base_measure.rb @@ -1,14 +1,25 @@ # frozen_string_literal: true class Admin::Metrics::Measure::BaseMeasure + CACHE_TTL = 5.minutes.freeze + def self.with_params? false end + attr_reader :loaded + + alias loaded? loaded + def initialize(start_at, end_at, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime @params = params + @loaded = false + end + + def cache_key + ["metrics/measure/#{key}", @start_at, @end_at, canonicalized_params].join(';') end def key @@ -16,15 +27,15 @@ class Admin::Metrics::Measure::BaseMeasure end def total - raise NotImplementedError + load[:total] end def previous_total - raise NotImplementedError + load[:previous_total] end def data - raise NotImplementedError + load[:data] end def self.model_name @@ -37,6 +48,35 @@ class Admin::Metrics::Measure::BaseMeasure protected + def load + unless loaded? + @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_queries }.with_indifferent_access + @loaded = true + end + + @values + end + + def perform_queries + { + total: perform_total_query, + previous_total: perform_previous_total_query, + data: perform_data_query, + } + end + + def perform_total_query + raise NotImplementedError + end + + def perform_previous_total_query + raise NotImplementedError + end + + def perform_data_query + raise NotImplementedError + end + def time_period (@start_at..@end_at) end @@ -50,6 +90,10 @@ class Admin::Metrics::Measure::BaseMeasure end def params - raise NotImplementedError + {} + end + + def canonicalized_params + params.to_h.to_a.sort_by { |k, _v| k.to_s }.map { |k, v| "#{k}=#{v}" }.join(';') end end diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb index b928fdb8fbc..7a2b7e0fac4 100644 --- a/app/lib/admin/metrics/measure/interactions_measure.rb +++ b/app/lib/admin/metrics/measure/interactions_measure.rb @@ -5,20 +5,20 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba 'interactions' end - def total + protected + + def perform_total_query activity_tracker.sum(time_period.first, time_period.last) end - def previous_total + def perform_previous_total_query activity_tracker.sum(previous_time_period.first, previous_time_period.last) end - def data + def perform_data_query activity_tracker.get(time_period.first, time_period.last).map { |date, value| { date: date.to_time(:utc).iso8601, value: value.to_s } } end - protected - def activity_tracker @activity_tracker ||= ActivityTracker.new('activity:interactions', :basic) end diff --git a/app/lib/admin/metrics/measure/new_users_measure.rb b/app/lib/admin/metrics/measure/new_users_measure.rb index b31679ad3a5..71191f1a221 100644 --- a/app/lib/admin/metrics/measure/new_users_measure.rb +++ b/app/lib/admin/metrics/measure/new_users_measure.rb @@ -5,15 +5,17 @@ class Admin::Metrics::Measure::NewUsersMeasure < Admin::Metrics::Measure::BaseMe 'new_users' end - def total + protected + + def perform_total_query User.where(created_at: time_period).count end - def previous_total + def perform_previous_total_query User.where(created_at: previous_time_period).count end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( WITH new_users AS ( diff --git a/app/lib/admin/metrics/measure/opened_reports_measure.rb b/app/lib/admin/metrics/measure/opened_reports_measure.rb index 9acc2c33db9..4b80a0c8c3b 100644 --- a/app/lib/admin/metrics/measure/opened_reports_measure.rb +++ b/app/lib/admin/metrics/measure/opened_reports_measure.rb @@ -5,15 +5,17 @@ class Admin::Metrics::Measure::OpenedReportsMeasure < Admin::Metrics::Measure::B 'opened_reports' end - def total + protected + + def perform_total_query Report.where(created_at: time_period).count end - def previous_total + def perform_previous_total_query Report.where(created_at: previous_time_period).count end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( WITH new_reports AS ( diff --git a/app/lib/admin/metrics/measure/resolved_reports_measure.rb b/app/lib/admin/metrics/measure/resolved_reports_measure.rb index 00cb24f7e1d..4ab746c8fa5 100644 --- a/app/lib/admin/metrics/measure/resolved_reports_measure.rb +++ b/app/lib/admin/metrics/measure/resolved_reports_measure.rb @@ -5,15 +5,17 @@ class Admin::Metrics::Measure::ResolvedReportsMeasure < Admin::Metrics::Measure: 'resolved_reports' end - def total + protected + + def perform_total_query Report.resolved.where(action_taken_at: time_period).count end - def previous_total + def perform_previous_total_query Report.resolved.where(action_taken_at: previous_time_period).count end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( WITH resolved_reports AS ( diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb index ef773081bed..8f4512efe7c 100644 --- a/app/lib/admin/metrics/measure/tag_accounts_measure.rb +++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb @@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::Bas 'tag_accounts' end - def total + protected + + def perform_total_query tag.history.aggregate(time_period).accounts end - def previous_total + def perform_previous_total_query tag.history.aggregate(previous_time_period).accounts end - def data + def perform_data_query time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } end - protected - def tag @tag ||= Tag.find(params[:id]) end diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb index cc064f63f64..11f229602ee 100644 --- a/app/lib/admin/metrics/measure/tag_servers_measure.rb +++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb @@ -9,15 +9,17 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base 'tag_servers' end - def total + protected + + def perform_total_query tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') end - def previous_total + def perform_previous_total_query tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') end - def data + def perform_data_query sql = <<-SQL.squish SELECT axis.*, ( SELECT count(distinct accounts.domain) AS value @@ -38,8 +40,6 @@ class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::Base rows.map { |row| { date: row['day'], value: row['value'].to_s } } end - protected - def tag @tag ||= Tag.find(params[:id]) end diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb index b7667bc6cf9..bce86b89f10 100644 --- a/app/lib/admin/metrics/measure/tag_uses_measure.rb +++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb @@ -9,20 +9,20 @@ class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMea 'tag_uses' end - def total + protected + + def perform_total_query tag.history.aggregate(time_period).uses end - def previous_total + def perform_previous_total_query tag.history.aggregate(previous_time_period).uses end - def data + def perform_data_query time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } end - protected - def tag @tag ||= Tag.find(params[:id]) end diff --git a/app/lib/admin/metrics/retention.rb b/app/lib/admin/metrics/retention.rb index 0179a6e282d..f6135ac1efc 100644 --- a/app/lib/admin/metrics/retention.rb +++ b/app/lib/admin/metrics/retention.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Admin::Metrics::Retention + CACHE_TTL = 5.minutes.freeze + class Cohort < ActiveModelSerializers::Model attributes :period, :frequency, :data end @@ -9,13 +11,37 @@ class Admin::Metrics::Retention attributes :date, :rate, :value end + attr_reader :loaded + + alias loaded? loaded + def initialize(start_at, end_at, frequency) @start_at = start_at&.to_date @end_at = end_at&.to_date @frequency = %w(day month).include?(frequency) ? frequency : 'day' + @loaded = false + end + + def cache_key + ['metrics/retention', @start_at, @end_at, @frequency].join(';') end def cohorts + load + end + + protected + + def load + unless loaded? + @values = Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) { perform_query } + @loaded = true + end + + @values + end + + def perform_query sql = <<-SQL.squish SELECT axis.*, ( WITH new_users AS (