71 lines
2.4 KiB
Ruby
71 lines
2.4 KiB
Ruby
# Backoffable provides transient-error retry backoff for models that depend on external services.
|
|
#
|
|
# When a transient error occurs (network hiccup, SSL failure, etc.) call apply_backoff!.
|
|
# The wait time ramps from 1 minute up to BACKOFF_MAX_INTERVAL_MINUTES, then holds at that
|
|
# ceiling for BACKOFF_MAX_INTERVAL_COUNT more attempts before calling prompt_reauthorization!.
|
|
#
|
|
# Call clear_backoff! after a successful operation to reset the counter.
|
|
|
|
module Backoffable
|
|
extend ActiveSupport::Concern
|
|
|
|
def backoff_log_identifier
|
|
inbox_id = respond_to?(:inbox) && inbox&.id
|
|
inbox_id ? "#{self.class.name} - #{inbox_id}" : "#{self.class.name}##{id}"
|
|
end
|
|
|
|
def backoff_retry_count
|
|
::Redis::Alfred.get(backoff_retry_count_key).to_i
|
|
end
|
|
|
|
def in_backoff?
|
|
val = ::Redis::Alfred.get(backoff_retry_after_key)
|
|
val.present? && Time.zone.at(val.to_f) > Time.current
|
|
end
|
|
|
|
def apply_backoff!
|
|
new_count = backoff_retry_count + 1
|
|
max_interval, max_retries = backoff_limits
|
|
|
|
if new_count > max_retries
|
|
exhaust_backoff(new_count)
|
|
else
|
|
schedule_backoff_retry(new_count, max_interval, max_retries)
|
|
end
|
|
end
|
|
|
|
def clear_backoff!
|
|
::Redis::Alfred.delete(backoff_retry_count_key)
|
|
::Redis::Alfred.delete(backoff_retry_after_key)
|
|
end
|
|
|
|
private
|
|
|
|
def backoff_limits
|
|
max_interval = GlobalConfigService.load('BACKOFF_MAX_INTERVAL_MINUTES', 5).to_i
|
|
max_count = GlobalConfigService.load('BACKOFF_MAX_INTERVAL_COUNT', 10).to_i
|
|
[max_interval, (max_interval - 1) + max_count]
|
|
end
|
|
|
|
def exhaust_backoff(new_count)
|
|
Rails.logger.warn "#{backoff_log_identifier} backoff exhausted (#{new_count} failures), prompting reauthorization"
|
|
clear_backoff!
|
|
prompt_reauthorization!
|
|
end
|
|
|
|
def schedule_backoff_retry(new_count, max_interval, max_retries)
|
|
wait_minutes = [new_count, max_interval].min
|
|
::Redis::Alfred.set(backoff_retry_count_key, new_count.to_s, ex: 24.hours)
|
|
::Redis::Alfred.set(backoff_retry_after_key, wait_minutes.minutes.from_now.to_f.to_s, ex: 24.hours)
|
|
Rails.logger.warn "#{backoff_log_identifier} backoff retry #{new_count}/#{max_retries}, next attempt in #{wait_minutes}m"
|
|
end
|
|
|
|
def backoff_retry_count_key
|
|
format(::Redis::Alfred::BACKOFF_RETRY_COUNT, obj_type: self.class.table_name.singularize, obj_id: id)
|
|
end
|
|
|
|
def backoff_retry_after_key
|
|
format(::Redis::Alfred::BACKOFF_RETRY_AFTER, obj_type: self.class.table_name.singularize, obj_id: id)
|
|
end
|
|
end
|