fix: replace imap backoff db columns with redis-based exponential backoff
This commit is contained in:
+27
-10
@@ -39,8 +39,8 @@ class Channel::Email < ApplicationRecord
|
||||
include Reauthorizable
|
||||
|
||||
AUTHORIZATION_ERROR_THRESHOLD = 10
|
||||
MAX_IMAP_RETRIES = 5
|
||||
IMAP_BACKOFF_BASE_SECONDS = 60
|
||||
IMAP_BACKOFF_MAX_INTERVAL_MINUTES = 5
|
||||
IMAP_BACKOFF_MAX_INTERVAL_COUNT = 5
|
||||
|
||||
# TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out).
|
||||
if Chatwoot.encryption_configured?
|
||||
@@ -74,25 +74,34 @@ class Channel::Email < ApplicationRecord
|
||||
imap_enabled && imap_address == 'imap.gmail.com'
|
||||
end
|
||||
|
||||
def imap_retry_count
|
||||
::Redis::Alfred.get(imap_retry_count_key).to_i
|
||||
end
|
||||
|
||||
def in_backoff?
|
||||
imap_retry_after.present? && imap_retry_after > Time.current
|
||||
val = ::Redis::Alfred.get(imap_retry_after_key)
|
||||
val.present? && Time.zone.at(val.to_f) > Time.current
|
||||
end
|
||||
|
||||
def apply_imap_backoff!
|
||||
new_count = imap_retry_count + 1
|
||||
if new_count >= MAX_IMAP_RETRIES
|
||||
update!(imap_retry_count: 0, imap_retry_after: nil)
|
||||
max_retries = (IMAP_BACKOFF_MAX_INTERVAL_MINUTES - 1) + IMAP_BACKOFF_MAX_INTERVAL_COUNT
|
||||
|
||||
if new_count > max_retries
|
||||
Rails.logger.warn "Error for email channel - #{inbox.id} exhausted backoff (#{new_count} failures), prompting reauthorization"
|
||||
clear_imap_backoff!
|
||||
prompt_reauthorization!
|
||||
else
|
||||
backoff_seconds = IMAP_BACKOFF_BASE_SECONDS * (2**new_count)
|
||||
update!(imap_retry_count: new_count, imap_retry_after: backoff_seconds.seconds.from_now)
|
||||
wait_minutes = [new_count, IMAP_BACKOFF_MAX_INTERVAL_MINUTES].min
|
||||
::Redis::Alfred.set(imap_retry_count_key, new_count.to_s)
|
||||
::Redis::Alfred.set(imap_retry_after_key, wait_minutes.minutes.from_now.to_f.to_s)
|
||||
Rails.logger.warn "Error for email channel - #{inbox.id} backoff retry #{new_count}/#{max_retries}, next attempt in #{wait_minutes}m"
|
||||
end
|
||||
end
|
||||
|
||||
def clear_imap_backoff!
|
||||
return unless imap_retry_count.positive? || imap_retry_after.present?
|
||||
|
||||
update!(imap_retry_count: 0, imap_retry_after: nil)
|
||||
::Redis::Alfred.delete(imap_retry_count_key)
|
||||
::Redis::Alfred.delete(imap_retry_after_key)
|
||||
end
|
||||
|
||||
private
|
||||
@@ -100,4 +109,12 @@ class Channel::Email < ApplicationRecord
|
||||
def ensure_forward_to_email
|
||||
self.forward_to_email ||= "#{SecureRandom.hex}@#{account.inbound_email_domain}"
|
||||
end
|
||||
|
||||
def imap_retry_count_key
|
||||
format(::Redis::Alfred::IMAP_BACKOFF_RETRY_COUNT, channel_id: id)
|
||||
end
|
||||
|
||||
def imap_retry_after_key
|
||||
format(::Redis::Alfred::IMAP_BACKOFF_RETRY_AFTER, channel_id: id)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
class AddImapBackoffToChannelEmail < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
add_column :channel_email, :imap_retry_count, :integer, default: 0, null: false
|
||||
add_column :channel_email, :imap_retry_after, :datetime
|
||||
end
|
||||
end
|
||||
+1
-3
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.1].define(version: 2026_02_23_100000) do
|
||||
ActiveRecord::Schema[7.1].define(version: 2026_01_30_061021) do
|
||||
# These extensions should be enabled to support this database
|
||||
enable_extension "pg_stat_statements"
|
||||
enable_extension "pg_trgm"
|
||||
@@ -442,8 +442,6 @@ ActiveRecord::Schema[7.1].define(version: 2026_02_23_100000) do
|
||||
t.jsonb "provider_config", default: {}
|
||||
t.string "provider"
|
||||
t.boolean "verified_for_sending", default: false, null: false
|
||||
t.integer "imap_retry_count", default: 0, null: false
|
||||
t.datetime "imap_retry_after"
|
||||
t.index ["email"], name: "index_channel_email_on_email", unique: true
|
||||
t.index ["forward_to_email"], name: "index_channel_email_on_forward_to_email", unique: true
|
||||
end
|
||||
|
||||
@@ -52,4 +52,8 @@ module Redis::RedisKeys
|
||||
|
||||
## Account Email Rate Limiting
|
||||
ACCOUNT_OUTBOUND_EMAIL_COUNT_KEY = 'OUTBOUND_EMAIL_COUNT::%<account_id>d::%<date>s'.freeze
|
||||
|
||||
## IMAP Backoff / Fetch Tracking (channel_email)
|
||||
IMAP_BACKOFF_RETRY_COUNT = 'IMAP_BACKOFF:channel_email:%<channel_id>d:retry_count'.freeze
|
||||
IMAP_BACKOFF_RETRY_AFTER = 'IMAP_BACKOFF:channel_email:%<channel_id>d:retry_after'.freeze
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user