From a24605961a1572f7e1800cc177b5b7defe45969c Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 3 May 2018 23:02:46 +0200 Subject: [PATCH] Fixes/do not override timestamps (#7336) * Revert "Fixes/do not override timestamps (#7331)" This reverts commit 581a5c9d29ef2a12f46b67a1097a9ad6df1c6953. * Document Snowflake ID corner-case a bit more Snowflake IDs are used for two purposes: making object identifiers harder to guess and ensuring they are in chronological order. For this reason, they are based on the `created_at` attribute of the object. Unfortunately, inserting items with older snowflakes IDs will break the assumption of consumers of the paging APIs that new items will always have a greater identifier than the last seen one. * Add `override_timestamps` virtual attribute to not correlate snowflake ID with created_at --- app/lib/activitypub/activity.rb | 8 +++++--- app/lib/activitypub/activity/announce.rb | 1 + app/lib/activitypub/activity/create.rb | 1 + app/lib/ostatus/activity/creation.rb | 10 +++++++++- app/models/status.rb | 4 ++++ app/workers/activitypub/processing_worker.rb | 2 +- app/workers/processing_worker.rb | 2 +- lib/mastodon/snowflake.rb | 2 +- 8 files changed, 23 insertions(+), 7 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 5b97a6208d2..84d4b175206 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -78,9 +78,11 @@ class ActivityPub::Activity notify_about_reblog(status) if reblog_of_local_account?(status) notify_about_mentions(status) - # Only continue if the status is supposed to have - # arrived in real-time - return unless status.within_realtime_window? + # Only continue if the status is supposed to have arrived in real-time. + # Note that if @options[:override_timestamps] isn't set, the status + # may have a lower snowflake id than other existing statuses, potentially + # "hiding" it from paginated API calls + return unless @options[:override_timestamps] || status.within_realtime_window? distribute_to_followers(status) end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 8840a450cb7..7e146ea8c43 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -16,6 +16,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity reblog: original_status, uri: @json['id'], created_at: @json['published'], + override_timestamps: @options[:override_timestamps], visibility: original_status.visibility ) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index edee2691ffc..8d17a4ebe2e 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -48,6 +48,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity language: detected_language, spoiler_text: @object['summary'] || '', created_at: @object['published'], + override_timestamps: @options[:override_timestamps], reply: @object['inReplyTo'].present?, sensitive: @object['sensitive'] || false, visibility: visibility_from_audience, diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb index a24a0093c35..1e7f4702994 100644 --- a/app/lib/ostatus/activity/creation.rb +++ b/app/lib/ostatus/activity/creation.rb @@ -40,6 +40,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base text: content, spoiler_text: content_warning, created_at: published, + override_timestamps: @options[:override_timestamps], reply: thread?, language: content_language, visibility: visibility_scope, @@ -61,7 +62,14 @@ class OStatus::Activity::Creation < OStatus::Activity::Base Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? - DistributionWorker.perform_async(status.id) if status.within_realtime_window? + + # Only continue if the status is supposed to have arrived in real-time. + # Note that if @options[:override_timestamps] isn't set, the status + # may have a lower snowflake id than other existing statuses, potentially + # "hiding" it from paginated API calls + return status unless @options[:override_timestamps] || status.within_realtime_window? + + DistributionWorker.perform_async(status.id) status end diff --git a/app/models/status.rb b/app/models/status.rb index fbb1f89aa0e..853e75b43fa 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -31,6 +31,10 @@ class Status < ApplicationRecord include Cacheable include StatusThreadingConcern + # If `override_timestamps` is set at creation time, Snowflake ID creation + # will be based on current time instead of `created_at` + attr_accessor :override_timestamps + update_index('statuses#status', :proper) if Chewy.enabled? enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb index bb9adf64bde..0e2e0edddba 100644 --- a/app/workers/activitypub/processing_worker.rb +++ b/app/workers/activitypub/processing_worker.rb @@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker sidekiq_options backtrace: true def perform(account_id, body) - ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id)) + ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true) end end diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb index 5df404bcc9b..978c3aba26f 100644 --- a/app/workers/processing_worker.rb +++ b/app/workers/processing_worker.rb @@ -6,6 +6,6 @@ class ProcessingWorker sidekiq_options backtrace: true def perform(account_id, body) - ProcessFeedService.new.call(body, Account.find(account_id)) + ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true) end end diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb index 219e323d487..9e5bc7383c1 100644 --- a/lib/mastodon/snowflake.rb +++ b/lib/mastodon/snowflake.rb @@ -7,7 +7,7 @@ module Mastodon::Snowflake def self.around_create(record) now = Time.now.utc - if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at + if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at || record.override_timestamps yield else record.id = Mastodon::Snowflake.id_at(record.created_at)