Fix compacted JSON-LD possibly causing compatibility issues on forwarding (#17428)

app/helpers/jsonld_helper.rb

@@ -77,6 +77,78 @@ module JsonLdHelper
     compacted
   end
 
+  # Patches a JSON-LD document to avoid compatibility issues on redistribution
+  #
+  # Since compacting a JSON-LD document against Mastodon's built-in vocabulary
+  # means other extension namespaces will be expanded, malformed JSON-LD
+  # attributes lost, and some values “unexpectedly” compacted this method
+  # patches the following likely sources of incompatibility:
+  # - 'https://www.w3.org/ns/activitystreams#Public' being compacted to
+  #   'as:Public' (for instance, pre-3.4.0 Mastodon does not understand
+  #   'as:Public')
+  # - single-item arrays being compacted to the item itself (`[foo]` being
+  #   compacted to `foo`)
+  #
+  # It is not always possible for `patch_for_forwarding!` to produce a document
+  # deemed safe for forwarding. Use `safe_for_forwarding?` to check the status
+  # of the output document.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [void]
+  def patch_for_forwarding!(original, compacted)
+    original.without('@context', 'signature').each do |key, value|
+      next if value.nil? || !compacted.key?(key)
+
+      compacted_value = compacted[key]
+      if value.is_a?(Hash) && compacted_value.is_a?(Hash)
+        patch_for_forwarding!(value, compacted_value)
+      elsif value.is_a?(Array)
+        compacted_value = [compacted_value] unless compacted_value.is_a?(Array)
+        return if value.size != compacted_value.size
+
+        compacted[key] = value.zip(compacted_value).map do |v, vc|
+          if v.is_a?(Hash) && vc.is_a?(Hash)
+            patch_for_forwarding!(v, vc)
+            vc
+          elsif v == 'https://www.w3.org/ns/activitystreams#Public' && vc == 'as:Public'
+            v
+          else
+            vc
+          end
+        end
+      elsif value == 'https://www.w3.org/ns/activitystreams#Public' && compacted_value == 'as:Public'
+        compacted[key] = value
+      end
+    end
+  end
+
+  # Tests whether a JSON-LD compaction is deemed safe for redistribution,
+  # that is, if it doesn't change its meaning to consumers that do not actually
+  # handle JSON-LD, but rely on values being serialized in a certain way.
+  #
+  # See `patch_for_forwarding!` for details.
+  #
+  # @param original [Hash] The original JSON-LD document used as reference
+  # @param compacted [Hash] The compacted JSON-LD document to be patched
+  # @return [Boolean] Whether the patched document is deemed safe
+  def safe_for_forwarding?(original, compacted)
+    original.without('@context', 'signature').all? do |key, value|
+      compacted_value = compacted[key]
+      return false unless value.class == compacted_value.class
+
+      if value.is_a?(Hash)
+        safe_for_forwarding?(value, compacted_value)
+      elsif value.is_a?(Array)
+        value.zip(compacted_value).all? do |v, vc|
+          v.is_a?(Hash) ? (vc.is_a?(Hash) && safe_for_forwarding?(v, vc)) : v == vc
+        end
+      else
+        value == compacted_value
+      end
+    end
+  end
+
   def fetch_resource(uri, id, on_behalf_of = nil)
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)

app/services/activitypub/process_collection_service.rb

@@ -5,13 +5,27 @@ class ActivityPub::ProcessCollectionService < BaseService
 
   def call(body, account, **options)
     @account = account
-    @json    = Oj.load(body, mode: :strict)
+    @json    = original_json = Oj.load(body, mode: :strict)
     @options = options
 
+    begin
       @json = compact(@json) if @json['signature'].is_a?(Hash)
+    rescue JSON::LD::JsonLdError => e
+      Rails.logger.debug "Error when compacting JSON-LD document for #{value_or_id(@json['actor'])}: #{e.message}"
+      @json = original_json.without('signature')
+    end
 
     return if !supported_context? || (different_actor? && verify_account!.nil?) || suspended_actor? || @account.local?
 
+    if @json['signature'].present?
+      # We have verified the signature, but in the compaction step above, might
+      # have introduced incompatibilities with other servers that do not
+      # normalize the JSON-LD documents (for instance, previous Mastodon
+      # versions), so skip redistribution if we can't get a safe document.
+      patch_for_forwarding!(original_json, @json)
+      @json.delete('signature') unless safe_for_forwarding?(original_json, @json)
+    end
+
     case @json['type']
     when 'Collection', 'CollectionPage'
       process_items @json['items']

spec/helpers/jsonld_helper_spec.rb

@@ -89,4 +89,86 @@ describe JsonLdHelper do
       expect(fetch_resource_without_id_validation('https://host.test/')).to eq({})
     end
   end
+
+  context 'compaction and forwarding' do
+    let(:json) do
+      {
+        '@context' => [
+          'https://www.w3.org/ns/activitystreams',
+          'https://w3id.org/security/v1',
+          {
+            'obsolete' => 'http://ostatus.org#',
+            'convo' => 'obsolete:conversation',
+            'new' => 'https://obscure-unreleased-test.joinmastodon.org/#',
+          },
+        ],
+        'type' => 'Create',
+        'to' => ['https://www.w3.org/ns/activitystreams#Public'],
+        'object' => {
+          'id' => 'https://example.com/status',
+          'type' => 'Note',
+          'inReplyTo' => nil,
+          'convo' => 'https://example.com/conversation',
+          'tag' => [
+            {
+              'type' => 'Mention',
+              'href' => ['foo'],
+            }
+          ],
+        },
+        'signature' => {
+          'type' => 'RsaSignature2017',
+          'created' => '2022-02-02T12:00:00Z',
+          'creator' => 'https://example.com/actor#main-key',
+          'signatureValue' => 'some-sig',
+        },
+      }
+    end
+
+    describe '#compact' do
+      it 'properly compacts JSON-LD with alternative context definitions' do
+        expect(compact(json).dig('object', 'conversation')).to eq 'https://example.com/conversation'
+      end
+
+      it 'compacts single-item arrays' do
+        expect(compact(json).dig('object', 'tag', 'href')).to eq 'foo'
+      end
+
+      it 'compacts the activistreams Public collection' do
+        expect(compact(json)['to']).to eq 'as:Public'
+      end
+
+      it 'properly copies signature' do
+        expect(compact(json)['signature']).to eq json['signature']
+      end
+    end
+
+    describe 'patch_for_forwarding!' do
+      it 'properly patches incompatibilities' do
+        json['object'].delete('convo')
+        compacted = compact(json)
+        patch_for_forwarding!(json, compacted)
+        expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public']
+        expect(compacted.dig('object', 'tag', 0, 'href')).to eq ['foo']
+        expect(safe_for_forwarding?(json, compacted)).to eq true
+      end
+    end
+
+    describe 'safe_for_forwarding?' do
+      it 'deems a safe compacting as such' do
+        json['object'].delete('convo')
+        compacted = compact(json)
+        deemed_compatible = patch_for_forwarding!(json, compacted)
+        expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public']
+        expect(safe_for_forwarding?(json, compacted)).to eq true
+      end
+
+      it 'deems an unsafe compacting as such' do
+        compacted = compact(json)
+        deemed_compatible = patch_for_forwarding!(json, compacted)
+        expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public']
+        expect(safe_for_forwarding?(json, compacted)).to eq false
+      end
+    end
+  end
 end