From 38fc1b498d971f7b33532c583b12e5dd3469af3c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Dec 2017 19:52:04 +0100 Subject: [PATCH] Add more instance stats APIs (#6125) * Add GET /api/v1/instance/peers API to reveal known domains * Add GET /api/v1/instance/activity API * Make new APIs disableable, exclude private statuses from activity stats * Fix code style issue * Fix week timestamps --- app/controllers/admin/settings_controller.rb | 4 +++ .../api/v1/instances/activity_controller.rb | 36 +++++++++++++++++++ .../api/v1/instances/peers_controller.rb | 17 +++++++++ app/controllers/application_controller.rb | 9 +++++ .../auth/confirmations_controller.rb | 6 ---- .../concerns/user_tracking_concern.rb | 1 + app/lib/activity_tracker.rb | 31 ++++++++++++++++ app/models/form/admin_settings.rb | 4 +++ app/models/status.rb | 6 ++++ app/models/user.rb | 15 ++++++++ app/views/admin/settings/edit.html.haml | 8 +++++ config/locales/en.yml | 6 ++++ config/routes.rb | 6 +++- config/settings.yml | 3 +- 14 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 app/controllers/api/v1/instances/activity_controller.rb create mode 100644 app/controllers/api/v1/instances/peers_controller.rb create mode 100644 app/lib/activity_tracker.rb diff --git a/app/controllers/admin/settings_controller.rb b/app/controllers/admin/settings_controller.rb index eed5fb6b574..487282dc35b 100644 --- a/app/controllers/admin/settings_controller.rb +++ b/app/controllers/admin/settings_controller.rb @@ -17,6 +17,8 @@ module Admin bootstrap_timeline_accounts thumbnail min_invite_role + activity_api_enabled + peers_api_enabled ).freeze BOOLEAN_SETTINGS = %w( @@ -24,6 +26,8 @@ module Admin open_deletion timeline_preview show_staff_badge + activity_api_enabled + peers_api_enabled ).freeze UPLOAD_SETTINGS = %w( diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb new file mode 100644 index 00000000000..36f52c38d7f --- /dev/null +++ b/app/controllers/api/v1/instances/activity_controller.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class Api::V1::Instances::ActivityController < Api::BaseController + before_action :require_enabled_api! + + respond_to :json + + def show + render_cached_json('api:v1:instances:activity:show', expires_in: 1.day) { activity } + end + + private + + def activity + weeks = [] + + 12.times do |i| + day = i.weeks.ago.to_date + week_id = day.cweek + week = Date.commercial(day.cwyear, week_id) + + weeks << { + week: week.to_time.to_i.to_s, + statuses: Redis.current.get("activity:statuses:local:#{week_id}") || 0, + logins: Redis.current.pfcount("activity:logins:#{week_id}"), + registrations: Redis.current.get("activity:accounts:local:#{week_id}") || 0, + } + end + + weeks + end + + def require_enabled_api! + head 404 unless Setting.activity_api_enabled + end +end diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb new file mode 100644 index 00000000000..2070c487df1 --- /dev/null +++ b/app/controllers/api/v1/instances/peers_controller.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class Api::V1::Instances::PeersController < Api::BaseController + before_action :require_enabled_api! + + respond_to :json + + def index + render_cached_json('api:v1:instances:peers:index', expires_in: 1.day) { Account.remote.domains } + end + + private + + def require_enabled_api! + head 404 unless Setting.peers_api_enabled + end +end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index a213302cb3c..51a978f440a 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -121,4 +121,13 @@ class ApplicationController < ActionController::Base end end end + + def render_cached_json(cache_key, **options) + data = Rails.cache.fetch(cache_key, { raw: true }.merge(options)) do + yield.to_json + end + + expires_in options[:expires_in], public: true + render json: data + end end diff --git a/app/controllers/auth/confirmations_controller.rb b/app/controllers/auth/confirmations_controller.rb index d5e8e58ede3..2fdb281f40e 100644 --- a/app/controllers/auth/confirmations_controller.rb +++ b/app/controllers/auth/confirmations_controller.rb @@ -2,10 +2,4 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController layout 'auth' - - def show - super do |user| - BootstrapTimelineWorker.perform_async(user.account_id) if user.errors.empty? - end - end end diff --git a/app/controllers/concerns/user_tracking_concern.rb b/app/controllers/concerns/user_tracking_concern.rb index 8663c3086b2..1e313294111 100644 --- a/app/controllers/concerns/user_tracking_concern.rb +++ b/app/controllers/concerns/user_tracking_concern.rb @@ -17,6 +17,7 @@ module UserTrackingConcern # Mark as signed-in today current_user.update_tracked_fields!(request) + ActivityTracker.record('activity:logins', current_user.id) # Regenerate feed if needed regenerate_feed! if user_needs_feed_update? diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb new file mode 100644 index 00000000000..50e927b0ce4 --- /dev/null +++ b/app/lib/activity_tracker.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class ActivityTracker + EXPIRE_AFTER = 90.days.seconds + + class << self + def increment(prefix) + key = [prefix, current_week].join(':') + + redis.incrby(key, 1) + redis.expire(key, EXPIRE_AFTER) + end + + def record(prefix, value) + key = [prefix, current_week].join(':') + + redis.pfadd(key, value) + redis.expire(key, value) + end + + private + + def redis + Redis.current + end + + def current_week + Time.zone.today.cweek + end + end +end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index c1d2cf42077..dd629279c05 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -30,6 +30,10 @@ class Form::AdminSettings :bootstrap_timeline_accounts=, :min_invite_role, :min_invite_role=, + :activity_api_enabled, + :activity_api_enabled=, + :peers_api_enabled, + :peers_api_enabled=, to: Setting ) end diff --git a/app/models/status.rb b/app/models/status.rb index 8579ff9e45a..00dcec624d0 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -135,6 +135,7 @@ class Status < ApplicationRecord end after_create_commit :store_uri, if: :local? + after_create_commit :update_statistics, if: :local? around_create Mastodon::Snowflake::Callbacks @@ -308,4 +309,9 @@ class Status < ApplicationRecord def set_local self.local = account.local? end + + def update_statistics + return unless public_visibility? || unlisted_visibility? + ActivityTracker.increment('activity:statuses:local') + end end diff --git a/app/models/user.rb b/app/models/user.rb index 578622fdf85..3ce6517a65b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -122,9 +122,19 @@ class User < ApplicationRecord update!(disabled: false) end + def confirm + return if confirmed? + + super + update_statistics! + end + def confirm! + return if confirmed? + skip_confirmation! save! + update_statistics! end def promote! @@ -202,4 +212,9 @@ class User < ApplicationRecord def sanitize_languages filtered_languages.reject!(&:blank?) end + + def update_statistics! + BootstrapTimelineWorker.perform_async(account_id) + ActivityTracker.increment('activity:accounts:local') + end end diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index c7c25f52836..4f9115ed2d0 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -46,5 +46,13 @@ .fields-group = f.input :bootstrap_timeline_accounts, wrapper: :with_block_label, label: t('admin.settings.bootstrap_timeline_accounts.title'), hint: t('admin.settings.bootstrap_timeline_accounts.desc_html') + %hr/ + + .fields-group + = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html') + + .fields-group + = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html') + .actions = f.button :button, t('generic.save_changes'), type: :submit diff --git a/config/locales/en.yml b/config/locales/en.yml index 325391cfd32..e4425b424f0 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -265,12 +265,18 @@ en: unresolved: Unresolved view: View settings: + activity_api_enabled: + desc_html: Counts of locally posted statuses, active users, and new registrations in weekly buckets + title: Publish aggregate statistics about user activity bootstrap_timeline_accounts: desc_html: Separate multiple usernames by comma. Only local and unlocked accounts will work. Default when empty is all local admins. title: Default follows for new users contact_information: email: Business e-mail username: Contact username + peers_api_enabled: + desc_html: Domain names this instance has encountered in the fediverse + title: Publish list of discovered instances registrations: closed_message: desc_html: Displayed on frontpage when registrations are closed. You can use HTML tags diff --git a/config/routes.rb b/config/routes.rb index 467849c03de..80a2c6d13de 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -241,7 +241,11 @@ Rails.application.routes.draw do resources :apps, only: [:create] - resource :instance, only: [:show] + resource :instance, only: [:show] do + resources :peers, only: [:index], controller: 'instances/peers' + resource :activity, only: [:show], controller: 'instances/activity' + end + resource :domain_blocks, only: [:show, :create, :destroy] resources :follow_requests, only: [:index] do diff --git a/config/settings.yml b/config/settings.yml index f03a32e50c6..4a2519464bb 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -47,7 +47,8 @@ defaults: &defaults - webmaster - administrator bootstrap_timeline_accounts: '' - + activity_api_enabled: true + peers_api_enabled: true development: <<: *defaults