From 5c42f47617d311219d06e082e4daa41e671903c8 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 1 Oct 2019 01:19:11 +0200 Subject: [PATCH] Fix records not being indexed sometimes (#12024) It's possible that after commit callbacks were not firing when exceptions occurred in the process. Also, the default Sidekiq strategy does not push indexing jobs immediately, which is not necessary and could be part of the issue too. --- app/models/account.rb | 2 +- app/models/account_stat.rb | 2 +- app/models/application_record.rb | 6 ++++++ app/models/favourite.rb | 2 +- app/models/status.rb | 2 +- app/models/tag.rb | 2 +- config/application.rb | 1 + config/initializers/chewy.rb | 5 +++-- lib/chewy/strategy/custom_sidekiq.rb | 30 ++++++++++++++++++++++++++++ spec/rails_helper.rb | 2 +- 10 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 lib/chewy/strategy/custom_sidekiq.rb diff --git a/app/models/account.rb b/app/models/account.rb index 55fe53fae1a..01d45e36c21 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -129,7 +129,7 @@ class Account < ApplicationRecord delegate :chosen_languages, to: :user, prefix: false, allow_nil: true - update_index('accounts#account', :self) if Chewy.enabled? + update_index('accounts#account', :self) def local? domain.nil? diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index 6d1097cec68..1351f7d8a4d 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -16,7 +16,7 @@ class AccountStat < ApplicationRecord belongs_to :account, inverse_of: :account_stat - update_index('accounts#account', :account) if Chewy.enabled? + update_index('accounts#account', :account) def increment_count!(key) update(attributes_for_increment(key)) diff --git a/app/models/application_record.rb b/app/models/application_record.rb index c1b873da6ac..5d7d3a09610 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -5,6 +5,12 @@ class ApplicationRecord < ActiveRecord::Base include Remotable + class << self + def update_index(_type_name, *_args, &_block) + super if Chewy.enabled? + end + end + def boolean_with_default(key, default_value) value = attributes[key] diff --git a/app/models/favourite.rb b/app/models/favourite.rb index 17f8c9fa631..bf0ec4449c4 100644 --- a/app/models/favourite.rb +++ b/app/models/favourite.rb @@ -13,7 +13,7 @@ class Favourite < ApplicationRecord include Paginable - update_index('statuses#status', :status) if Chewy.enabled? + update_index('statuses#status', :status) belongs_to :account, inverse_of: :favourites belongs_to :status, inverse_of: :favourites diff --git a/app/models/status.rb b/app/models/status.rb index 5e74745775d..078a6456635 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -39,7 +39,7 @@ class Status < ApplicationRecord # will be based on current time instead of `created_at` attr_accessor :override_timestamps - update_index('statuses#status', :proper) if Chewy.enabled? + update_index('statuses#status', :proper) enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility diff --git a/app/models/tag.rb b/app/models/tag.rb index 9aca3983f24..82786daa81d 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -49,7 +49,7 @@ class Tag < ApplicationRecord after_save :save_account_tag_stat - update_index('tags#tag', :self) if Chewy.enabled? + update_index('tags#tag', :self) def account_tag_stat super || build_account_tag_stat diff --git a/config/application.rb b/config/application.rb index 3ced81b8f34..60f73f8bb6c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,6 +15,7 @@ require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' +require_relative '../lib/chewy/strategy/custom_sidekiq' Dotenv::Railtie.load diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index d5347f2bfd0..9ff0dccc1b4 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -12,8 +12,9 @@ Chewy.settings = { sidekiq: { queue: 'pull' }, } -Chewy.root_strategy = enabled ? :sidekiq : :bypass -Chewy.request_strategy = enabled ? :sidekiq : :bypass +Chewy.root_strategy = :custom_sidekiq +Chewy.request_strategy = :custom_sidekiq +Chewy.use_after_commit_callbacks = false module Chewy class << self diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb new file mode 100644 index 00000000000..3e54326ba8b --- /dev/null +++ b/lib/chewy/strategy/custom_sidekiq.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class CustomSidekiq < Base + class Worker + include ::Sidekiq::Worker + + sidekiq_options queue: 'pull' + + def perform(type, ids, options = {}) + options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async + type.constantize.import!(ids, options) + end + end + + def update(type, objects, _options = {}) + return unless Chewy.enabled? + + ids = type.root.id ? Array.wrap(objects) : type.adapter.identify(objects) + + return if ids.empty? + + Worker.perform_async(type.name, ids) + end + + def leave; end + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 3a5e7491e51..6fbceca5396 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -12,7 +12,7 @@ require 'capybara/rspec' Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f } ActiveRecord::Migration.maintain_test_schema! -WebMock.disable_net_connect! +WebMock.disable_net_connect!(allow: Chewy.settings[:host]) Redis.current = Redis::Namespace.new("mastodon_test#{ENV['TEST_ENV_NUMBER']}", redis: Redis.current) Sidekiq::Testing.inline! Sidekiq::Logging.logger = nil