From 171c20ce72bc5252a7515ce9d23671877c2ddac5 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 13 Jul 2018 02:16:06 +0200
Subject: [PATCH] Add federation relay support (#7998)

* Add federation relay support

* Add admin UI for managing relays

* Include actor on relay-related activities

* Fix i18n
---
 app/controllers/admin/relays_controller.rb    | 58 +++++++++++++++
 app/javascript/styles/mastodon/admin.scss     |  5 ++
 app/models/relay.rb                           | 74 +++++++++++++++++++
 app/policies/relay_policy.rb                  |  7 ++
 .../activitypub/delete_actor_serializer.rb    |  6 +-
 .../activitypub/delete_serializer.rb          |  6 +-
 .../activitypub/undo_announce_serializer.rb   |  6 +-
 .../activitypub/update_serializer.rb          |  6 +-
 app/services/remove_status_service.rb         | 12 +++
 app/services/suspend_account_service.rb       | 12 ++-
 app/views/admin/relays/_relay.html.haml       | 21 ++++++
 app/views/admin/relays/index.html.haml        | 20 +++++
 app/views/admin/relays/new.html.haml          | 13 ++++
 .../activitypub/distribution_worker.rb        | 12 +++
 .../activitypub/update_distribution_worker.rb | 10 ++-
 config/locales/en.yml                         |  8 ++
 config/locales/simple_form.en.yml             |  2 +
 config/navigation.rb                          |  1 +
 config/routes.rb                              |  7 ++
 db/migrate/20180711152640_create_relays.rb    | 12 +++
 db/schema.rb                                  | 11 ++-
 spec/fabricators/relay_fabricator.rb          |  4 +
 spec/models/relay_spec.rb                     |  4 +
 23 files changed, 309 insertions(+), 8 deletions(-)
 create mode 100644 app/controllers/admin/relays_controller.rb
 create mode 100644 app/models/relay.rb
 create mode 100644 app/policies/relay_policy.rb
 create mode 100644 app/views/admin/relays/_relay.html.haml
 create mode 100644 app/views/admin/relays/index.html.haml
 create mode 100644 app/views/admin/relays/new.html.haml
 create mode 100644 db/migrate/20180711152640_create_relays.rb
 create mode 100644 spec/fabricators/relay_fabricator.rb
 create mode 100644 spec/models/relay_spec.rb

diff --git a/app/controllers/admin/relays_controller.rb b/app/controllers/admin/relays_controller.rb
new file mode 100644
index 0000000000..1b02d3c361
--- /dev/null
+++ b/app/controllers/admin/relays_controller.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Admin
+  class RelaysController < BaseController
+    before_action :set_relay, except: [:index, :new, :create]
+
+    def index
+      authorize :relay, :update?
+      @relays = Relay.all
+    end
+
+    def new
+      authorize :relay, :update?
+      @relay = Relay.new(inbox_url: Relay::PRESET_RELAY)
+    end
+
+    def create
+      authorize :relay, :update?
+
+      @relay = Relay.new(resource_params)
+
+      if @relay.save
+        @relay.enable!
+        redirect_to admin_relays_path
+      else
+        render action: :new
+      end
+    end
+
+    def destroy
+      authorize :relay, :update?
+      @relay.destroy
+      redirect_to admin_relays_path
+    end
+
+    def enable
+      authorize :relay, :update?
+      @relay.enable!
+      redirect_to admin_relays_path
+    end
+
+    def disable
+      authorize :relay, :update?
+      @relay.disable!
+      redirect_to admin_relays_path
+    end
+
+    private
+
+    def set_relay
+      @relay = Relay.find(params[:id])
+    end
+
+    def resource_params
+      params.require(:relay).permit(:inbox_url)
+    end
+  end
+end
diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss
index 560b11ddf4..42f5072967 100644
--- a/app/javascript/styles/mastodon/admin.scss
+++ b/app/javascript/styles/mastodon/admin.scss
@@ -165,6 +165,11 @@
       color: $valid-value-color;
       font-weight: 500;
     }
+
+    .negative-hint {
+      color: $error-value-color;
+      font-weight: 500;
+    }
   }
 
   .simple_form {
diff --git a/app/models/relay.rb b/app/models/relay.rb
new file mode 100644
index 0000000000..76143bb27a
--- /dev/null
+++ b/app/models/relay.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: relays
+#
+#  id                 :bigint(8)        not null, primary key
+#  inbox_url          :string           default(""), not null
+#  enabled            :boolean          default(FALSE), not null
+#  follow_activity_id :string
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#
+
+class Relay < ApplicationRecord
+  PRESET_RELAY = 'https://relay.joinmastodon.org/inbox'
+
+  validates :inbox_url, presence: true, uniqueness: true, url: true, if: :will_save_change_to_inbox_url?
+
+  scope :enabled, -> { where(enabled: true) }
+
+  before_destroy :ensure_disabled
+
+  def enable!
+    activity_id = ActivityPub::TagManager.instance.generate_uri_for(nil)
+    payload     = Oj.dump(follow_activity(activity_id))
+
+    ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url)
+    update(enabled: true, follow_activity_id: activity_id)
+  end
+
+  def disable!
+    activity_id = ActivityPub::TagManager.instance.generate_uri_for(nil)
+    payload     = Oj.dump(unfollow_activity(activity_id))
+
+    ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url)
+    update(enabled: false, follow_activity_id: nil)
+  end
+
+  private
+
+  def follow_activity(activity_id)
+    {
+      '@context': ActivityPub::TagManager::CONTEXT,
+      id: activity_id,
+      type: 'Follow',
+      actor: ActivityPub::TagManager.instance.uri_for(some_local_account),
+      object: ActivityPub::TagManager::COLLECTIONS[:public],
+    }
+  end
+
+  def unfollow_activity(activity_id)
+    {
+      '@context': ActivityPub::TagManager::CONTEXT,
+      id: activity_id,
+      type: 'Undo',
+      actor: ActivityPub::TagManager.instance.uri_for(some_local_account),
+      object: {
+        id: follow_activity_id,
+        type: 'Follow',
+        actor: ActivityPub::TagManager.instance.uri_for(some_local_account),
+        object: ActivityPub::TagManager::COLLECTIONS[:public],
+      },
+    }
+  end
+
+  def some_local_account
+    @some_local_account ||= Account.local.find_by(suspended: false)
+  end
+
+  def ensure_disabled
+    return unless enabled?
+    disable!
+  end
+end
diff --git a/app/policies/relay_policy.rb b/app/policies/relay_policy.rb
new file mode 100644
index 0000000000..bd75e21977
--- /dev/null
+++ b/app/policies/relay_policy.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RelayPolicy < ApplicationPolicy
+  def update?
+    admin?
+  end
+end
diff --git a/app/serializers/activitypub/delete_actor_serializer.rb b/app/serializers/activitypub/delete_actor_serializer.rb
index dfea9db4ab..ddf59be970 100644
--- a/app/serializers/activitypub/delete_actor_serializer.rb
+++ b/app/serializers/activitypub/delete_actor_serializer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::DeleteActorSerializer < ActiveModel::Serializer
-  attributes :id, :type, :actor
+  attributes :id, :type, :actor, :to
   attribute :virtual_object, key: :object
 
   def id
@@ -19,4 +19,8 @@ class ActivityPub::DeleteActorSerializer < ActiveModel::Serializer
   def virtual_object
     actor
   end
+
+  def to
+    [ActivityPub::TagManager::COLLECTIONS[:public]]
+  end
 end
diff --git a/app/serializers/activitypub/delete_serializer.rb b/app/serializers/activitypub/delete_serializer.rb
index 2bb65135f7..5012a8383f 100644
--- a/app/serializers/activitypub/delete_serializer.rb
+++ b/app/serializers/activitypub/delete_serializer.rb
@@ -17,7 +17,7 @@ class ActivityPub::DeleteSerializer < ActiveModel::Serializer
     end
   end
 
-  attributes :id, :type, :actor
+  attributes :id, :type, :actor, :to
 
   has_one :object, serializer: TombstoneSerializer
 
@@ -32,4 +32,8 @@ class ActivityPub::DeleteSerializer < ActiveModel::Serializer
   def actor
     ActivityPub::TagManager.instance.uri_for(object.account)
   end
+
+  def to
+    [ActivityPub::TagManager::COLLECTIONS[:public]]
+  end
 end
diff --git a/app/serializers/activitypub/undo_announce_serializer.rb b/app/serializers/activitypub/undo_announce_serializer.rb
index 839847e22c..4fc042727a 100644
--- a/app/serializers/activitypub/undo_announce_serializer.rb
+++ b/app/serializers/activitypub/undo_announce_serializer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::UndoAnnounceSerializer < ActiveModel::Serializer
-  attributes :id, :type, :actor
+  attributes :id, :type, :actor, :to
 
   has_one :object, serializer: ActivityPub::ActivitySerializer
 
@@ -16,4 +16,8 @@ class ActivityPub::UndoAnnounceSerializer < ActiveModel::Serializer
   def actor
     ActivityPub::TagManager.instance.uri_for(object.account)
   end
+
+  def to
+    [ActivityPub::TagManager::COLLECTIONS[:public]]
+  end
 end
diff --git a/app/serializers/activitypub/update_serializer.rb b/app/serializers/activitypub/update_serializer.rb
index ebc667d963..48d7a19294 100644
--- a/app/serializers/activitypub/update_serializer.rb
+++ b/app/serializers/activitypub/update_serializer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::UpdateSerializer < ActiveModel::Serializer
-  attributes :id, :type, :actor
+  attributes :id, :type, :actor, :to
 
   has_one :object, serializer: ActivityPub::ActorSerializer
 
@@ -16,4 +16,8 @@ class ActivityPub::UpdateSerializer < ActiveModel::Serializer
   def actor
     ActivityPub::TagManager.instance.uri_for(object)
   end
+
+  def to
+    [ActivityPub::TagManager::COLLECTIONS[:public]]
+  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 2380991691..fb889140b2 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -90,6 +90,18 @@ class RemoveStatusService < BaseService
     ActivityPub::DeliveryWorker.push_bulk(@account.followers.inboxes) do |inbox_url|
       [signed_activity_json, @account.id, inbox_url]
     end
+
+    relay! if relayable?
+  end
+
+  def relayable?
+    @status.public_visibility?
+  end
+
+  def relay!
+    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+      [signed_activity_json, @account.id, inbox_url]
+    end
   end
 
   def salmon_xml
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 708d15e37d..0a98f5fb9d 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -22,7 +22,13 @@ class SuspendAccountService < BaseService
   end
 
   def purge_content!
-    ActivityPub::RawDistributionWorker.perform_async(delete_actor_json, @account.id) if @account.local?
+    if @account.local?
+      ActivityPub::RawDistributionWorker.perform_async(delete_actor_json, @account.id)
+
+      ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+        [delete_actor_json, @account.id, inbox_url]
+      end
+    end
 
     @account.statuses.reorder(nil).find_in_batches do |statuses|
       BatchedRemoveStatusService.new.call(statuses)
@@ -59,12 +65,14 @@ class SuspendAccountService < BaseService
   end
 
   def delete_actor_json
+    return @delete_actor_json if defined?(@delete_actor_json)
+
     payload = ActiveModelSerializers::SerializableResource.new(
       @account,
       serializer: ActivityPub::DeleteActorSerializer,
       adapter: ActivityPub::Adapter
     ).as_json
 
-    Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account))
+    @delete_actor_json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account))
   end
 end
diff --git a/app/views/admin/relays/_relay.html.haml b/app/views/admin/relays/_relay.html.haml
new file mode 100644
index 0000000000..d974c80a61
--- /dev/null
+++ b/app/views/admin/relays/_relay.html.haml
@@ -0,0 +1,21 @@
+%tr
+  %td
+    %samp= relay.inbox_url
+  %td
+    - if relay.enabled?
+      %span.positive-hint
+        = fa_icon('check')
+        = ' '
+        = t 'admin.relays.enabled'
+    - else
+      %span.negative-hint
+        = fa_icon('times')
+        = ' '
+        = t 'admin.relays.disabled'
+  %td
+    - if relay.enabled?
+      = table_link_to 'power-off', t('admin.relays.disable'), disable_admin_relay_path(relay), method: :post, data: { confirm: t('admin.accounts.are_you_sure') }
+    - else
+      = table_link_to 'power-off', t('admin.relays.enable'), enable_admin_relay_path(relay), method: :post, data: { confirm: t('admin.accounts.are_you_sure') }
+
+    = table_link_to 'times', t('admin.relays.delete'), admin_relay_path(relay), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') }
diff --git a/app/views/admin/relays/index.html.haml b/app/views/admin/relays/index.html.haml
new file mode 100644
index 0000000000..1636a53f85
--- /dev/null
+++ b/app/views/admin/relays/index.html.haml
@@ -0,0 +1,20 @@
+- content_for :page_title do
+  = t('admin.relays.title')
+
+.simple_form
+  %p.hint= t('admin.relays.description_html')
+  = link_to @relays.empty? ? t('admin.relays.setup') : t('admin.relays.add_new'), new_admin_relay_path, class: 'block-button'
+
+- unless @relays.empty?
+  %hr.spacer
+
+  .table-wrapper
+    %table.table
+      %thead
+        %tr
+          %th= t('admin.relays.inbox_url')
+          %th= t('admin.relays.status')
+          %th
+      %tbody
+        = render @relays
+
diff --git a/app/views/admin/relays/new.html.haml b/app/views/admin/relays/new.html.haml
new file mode 100644
index 0000000000..126794acfe
--- /dev/null
+++ b/app/views/admin/relays/new.html.haml
@@ -0,0 +1,13 @@
+- content_for :page_title do
+  = t('admin.relays.add_new')
+
+= simple_form_for @relay, url: admin_relays_path do |f|
+  = render 'shared/error_messages', object: @relay
+
+  .field-group
+    = f.input :inbox_url, as: :string, wrapper: :with_block_label
+
+  .actions
+    = f.button :button, t('admin.relays.save_and_enable'), type: :submit
+
+  %p.hint.subtle-hint= t('admin.relays.enable_hint')
diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb
index 14bb933c00..c2bfd4f2f1 100644
--- a/app/workers/activitypub/distribution_worker.rb
+++ b/app/workers/activitypub/distribution_worker.rb
@@ -14,6 +14,8 @@ class ActivityPub::DistributionWorker
     ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
       [signed_payload, @account.id, inbox_url]
     end
+
+    relay! if relayable?
   rescue ActiveRecord::RecordNotFound
     true
   end
@@ -24,6 +26,10 @@ class ActivityPub::DistributionWorker
     @status.direct_visibility?
   end
 
+  def relayable?
+    @status.public_visibility?
+  end
+
   def inboxes
     @inboxes ||= @account.followers.inboxes
   end
@@ -39,4 +45,10 @@ class ActivityPub::DistributionWorker
       adapter: ActivityPub::Adapter
     ).as_json
   end
+
+  def relay!
+    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+  end
 end
diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb
index f3377dcec5..87efafb3e8 100644
--- a/app/workers/activitypub/update_distribution_worker.rb
+++ b/app/workers/activitypub/update_distribution_worker.rb
@@ -9,7 +9,11 @@ class ActivityPub::UpdateDistributionWorker
     @account = Account.find(account_id)
 
     ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, @account.id, inbox_url]
+      [signed_payload, @account.id, inbox_url]
+    end
+
+    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
     end
   rescue ActiveRecord::RecordNotFound
     true
@@ -21,6 +25,10 @@ class ActivityPub::UpdateDistributionWorker
     @inboxes ||= @account.followers.inboxes
   end
 
+  def signed_payload
+    @signed_payload ||= Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account))
+  end
+
   def payload
     @payload ||= ActiveModelSerializers::SerializableResource.new(
       @account,
diff --git a/config/locales/en.yml b/config/locales/en.yml
index a03b12a397..ec08f0d78a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -261,6 +261,14 @@ en:
         expired: Expired
         title: Filter
       title: Invites
+    relays:
+      add_new: Add new relay
+      description_html: A <strong>federation relay</strong> is an intermediary server that exchanges large volumes of public toots between servers that subscribe and publish to it. <strong>It can help small and medium servers discover content from the fediverse</strong>, which would otherwise require local users manually following other people on remote servers.
+      enable_hint: Once enabled, your server will subscribe to all public toots from this relay, and will begin sending this server's public toots to it.
+      inbox_url: Relay URL
+      setup: Setup a relay connection
+      status: Status
+      title: Relays
     report_notes:
       created_msg: Report note successfully created!
       destroyed_msg: Report note successfully deleted!
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 7d9a5d617e..9ff548f40e 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -13,6 +13,7 @@ en:
           other: <span class="name-counter">%{count}</span> characters left
         fields: You can have up to 4 items displayed as a table on your profile
         header: PNG, GIF or JPG. At most 2MB. Will be downscaled to 700x335px
+        inbox_url: Copy the URL from the frontpage of the relay you want to use
         irreversible: Filtered toots will disappear irreversibly, even if filter is later removed
         locale: The language of the user interface, e-mails and push notifications
         locked: Requires you to manually approve followers
@@ -52,6 +53,7 @@ en:
         expires_in: Expire after
         fields: Profile metadata
         header: Header
+        inbox_url: URL of the relay inbox
         irreversible: Drop instead of hide
         locale: Interface language
         locked: Lock account
diff --git a/config/navigation.rb b/config/navigation.rb
index 3f2e913c62..a13ad6f437 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -36,6 +36,7 @@ SimpleNavigation::Configuration.run do |navigation|
     primary.item :admin, safe_join([fa_icon('cogs fw'), t('admin.title')]), proc { current_user.admin? ? edit_admin_settings_url : admin_custom_emojis_url }, if: proc { current_user.staff? } do |admin|
       admin.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), edit_admin_settings_url, if: -> { current_user.admin? }
       admin.item :custom_emojis, safe_join([fa_icon('smile-o fw'), t('admin.custom_emojis.title')]), admin_custom_emojis_url, highlights_on: %r{/admin/custom_emojis}
+      admin.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/relays}
       admin.item :subscriptions, safe_join([fa_icon('paper-plane-o fw'), t('admin.subscriptions.title')]), admin_subscriptions_url, if: -> { current_user.admin? }
       admin.item :sidekiq, safe_join([fa_icon('diamond fw'), 'Sidekiq']), sidekiq_url, link_html: { target: 'sidekiq' }, if: -> { current_user.admin? }
       admin.item :pghero, safe_join([fa_icon('database fw'), 'PgHero']), pghero_url, link_html: { target: 'pghero' }, if: -> { current_user.admin? }
diff --git a/config/routes.rb b/config/routes.rb
index fd26b4aa74..3d0da1a857 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -131,6 +131,13 @@ Rails.application.routes.draw do
     resource :settings, only: [:edit, :update]
     resources :invites, only: [:index, :create, :destroy]
 
+    resources :relays, only: [:index, :new, :create, :destroy] do
+      member do
+        post :enable
+        post :disable
+      end
+    end
+
     resources :instances, only: [:index] do
       collection do
         post :resubscribe
diff --git a/db/migrate/20180711152640_create_relays.rb b/db/migrate/20180711152640_create_relays.rb
new file mode 100644
index 0000000000..8762f473a2
--- /dev/null
+++ b/db/migrate/20180711152640_create_relays.rb
@@ -0,0 +1,12 @@
+class CreateRelays < ActiveRecord::Migration[5.2]
+  def change
+    create_table :relays do |t|
+      t.string :inbox_url, default: '', null: false
+      t.boolean :enabled, default: false, null: false, index: true
+
+      t.string :follow_activity_id
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 02032c5482..e0da669c46 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2018_07_07_154237) do
+ActiveRecord::Schema.define(version: 2018_07_11_152640) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -371,6 +371,15 @@ ActiveRecord::Schema.define(version: 2018_07_07_154237) do
     t.index ["status_id", "preview_card_id"], name: "index_preview_cards_statuses_on_status_id_and_preview_card_id"
   end
 
+  create_table "relays", force: :cascade do |t|
+    t.string "inbox_url", default: "", null: false
+    t.boolean "enabled", default: false, null: false
+    t.string "follow_activity_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["enabled"], name: "index_relays_on_enabled"
+  end
+
   create_table "report_notes", force: :cascade do |t|
     t.text "content", null: false
     t.bigint "report_id", null: false
diff --git a/spec/fabricators/relay_fabricator.rb b/spec/fabricators/relay_fabricator.rb
new file mode 100644
index 0000000000..2c9df4ad3a
--- /dev/null
+++ b/spec/fabricators/relay_fabricator.rb
@@ -0,0 +1,4 @@
+Fabricator(:relay) do
+  inbox_url "https://example.com/inbox"
+  enabled   true
+end
diff --git a/spec/models/relay_spec.rb b/spec/models/relay_spec.rb
new file mode 100644
index 0000000000..12dc0f20f6
--- /dev/null
+++ b/spec/models/relay_spec.rb
@@ -0,0 +1,4 @@
+require 'rails_helper'
+
+RSpec.describe Relay, type: :model do
+end