From da3d8aff799e9b92ebe76e2dc8f6f953869a1e42 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 7 Dec 2023 08:40:44 -0600 Subject: [PATCH] Error handling for attachment batch delete process (#28184) Co-authored-by: Claire --- app/lib/attachment_batch.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/lib/attachment_batch.rb b/app/lib/attachment_batch.rb index 13a9da828f..b28f5c3d7f 100644 --- a/app/lib/attachment_batch.rb +++ b/app/lib/attachment_batch.rb @@ -4,7 +4,8 @@ class AttachmentBatch # Maximum amount of objects you can delete in an S3 API call. It's # important to remember that this does not correspond to the number # of records in the batch, since records can have multiple attachments - LIMIT = 1_000 + LIMIT = ENV.fetch('S3_BATCH_DELETE_LIMIT', 1000).to_i + MAX_RETRY = ENV.fetch('S3_BATCH_DELETE_RETRY', 3).to_i # Attributes generated and maintained by Paperclip (not all of them # are always used on every class, however) @@ -95,6 +96,7 @@ class AttachmentBatch # objects can be processed at once, so we have to potentially # separate them into multiple calls. + retries = 0 keys.each_slice(LIMIT) do |keys_slice| logger.debug { "Deleting #{keys_slice.size} objects" } @@ -102,6 +104,17 @@ class AttachmentBatch objects: keys_slice.map { |key| { key: key } }, quiet: true, }) + rescue => e + retries += 1 + + if retries < MAX_RETRY + logger.debug "Retry #{retries}/#{MAX_RETRY} after #{e.message}" + sleep 2**retries + retry + else + logger.error "Batch deletion from S3 failed after #{e.message}" + raise e + end end end