From ba804e0f3094dadce7f2e23dc8c4278d917fa622 Mon Sep 17 00:00:00 2001 From: Sivin Varghese <64252451+iamsivin@users.noreply.github.com> Date: Wed, 25 Feb 2026 16:52:45 +0530 Subject: [PATCH 01/13] fix: Upgrade `pico-search` to `0.6.0` (#13645) --- .../routes/dashboard/settings/canned/Index.vue | 5 ++++- .../routes/dashboard/settings/labels/Index.vue | 5 ++++- .../shared/components/ui/label/LabelDropdown.vue | 4 +--- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/javascript/dashboard/routes/dashboard/settings/canned/Index.vue b/app/javascript/dashboard/routes/dashboard/settings/canned/Index.vue index 1a5c067de..5c1810b63 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/canned/Index.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/canned/Index.vue @@ -45,7 +45,10 @@ const records = computed(() => const filteredRecords = computed(() => { const query = searchQuery.value.trim(); if (!query) return records.value; - return picoSearch(records.value, query, ['short_code', 'content']); + return picoSearch(records.value, query, [ + { name: 'short_code', weight: 4 }, + 'content', + ]); }); const uiFlags = computed(() => getters.getUIFlags.value); diff --git a/app/javascript/dashboard/routes/dashboard/settings/labels/Index.vue b/app/javascript/dashboard/routes/dashboard/settings/labels/Index.vue index c15f7df28..9b81f30cf 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/labels/Index.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/labels/Index.vue @@ -32,7 +32,10 @@ const records = computed(() => getters['labels/getLabels'].value); const filteredRecords = computed(() => { const query = searchQuery.value.trim(); if (!query) return records.value; - return picoSearch(records.value, query, ['title', 'description']); + return picoSearch(records.value, query, [ + { name: 'title', weight: 4 }, + 'description', + ]); }); const uiFlags = computed(() => getters['labels/getUIFlags'].value); diff --git a/app/javascript/shared/components/ui/label/LabelDropdown.vue b/app/javascript/shared/components/ui/label/LabelDropdown.vue index a1216acd8..71da844e0 100644 --- a/app/javascript/shared/components/ui/label/LabelDropdown.vue +++ b/app/javascript/shared/components/ui/label/LabelDropdown.vue @@ -46,9 +46,7 @@ export default { filteredActiveLabels() { if (!this.search) return this.accountLabels; - return picoSearch(this.accountLabels, this.search, ['title'], { - threshold: 0.9, - }); + return picoSearch(this.accountLabels, this.search, ['title']); }, noResult() { diff --git a/package.json b/package.json index 7877a7170..b1aa60f00 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "@radix-ui/colors": "^3.0.0", "@rails/actioncable": "6.1.3", "@rails/ujs": "^7.1.400", - "@scmmishra/pico-search": "0.5.4", + "@scmmishra/pico-search": "0.6.0", "@sentry/vue": "^8.55.0", "@sindresorhus/slugify": "2.2.1", "@tailwindcss/typography": "^0.5.15", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f038f123..539a99b21 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,8 +59,8 @@ importers: specifier: ^7.1.400 version: 7.1.400 '@scmmishra/pico-search': - specifier: 0.5.4 - version: 0.5.4 + specifier: 0.6.0 + version: 0.6.0 '@sentry/vue': specifier: ^8.55.0 version: 8.55.0(pinia@3.0.4(typescript@5.6.2)(vue@3.5.12(typescript@5.6.2)))(vue@3.5.12(typescript@5.6.2)) @@ -1240,8 +1240,8 @@ packages: '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} - '@scmmishra/pico-search@0.5.4': - resolution: {integrity: sha512-JdV8KumQ+pE5tqgQ71xUT9biE/qV//tx3NCqTLkW9Z4tsjKGN0B6kVowmtaZBAtErqir9XiMxsKXRTMF/MpUww==} + '@scmmishra/pico-search@0.6.0': + resolution: {integrity: sha512-1zC2cAwPWuv38VEh0It90fdUWkvX75OwBUjgTj+d5LTltARnf3ydbpcN2Ucl0aATBMmaNqPMcVvT25IOCAqCEA==} '@sentry-internal/browser-utils@8.55.0': resolution: {integrity: sha512-ROgqtQfpH/82AQIpESPqPQe0UyWywKJsmVIqi3c5Fh+zkds5LUxnssTj3yNd1x+kxaPDVB023jAP+3ibNgeNDw==} @@ -5795,7 +5795,7 @@ snapshots: '@rtsao/scc@1.1.0': {} - '@scmmishra/pico-search@0.5.4': {} + '@scmmishra/pico-search@0.6.0': {} '@sentry-internal/browser-utils@8.55.0': dependencies: From b98c614669e59c7b63305f6ba0d838b38cd9951a Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 25 Feb 2026 18:33:37 +0530 Subject: [PATCH 02/13] feat: add campaign context to Captain v2 prompts (#13644) Co-authored-by: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com> --- enterprise/app/models/concerns/agentable.rb | 4 ++- .../captain/assistant/agent_runner_service.rb | 3 ++ .../lib/captain/prompts/assistant.liquid | 6 +++- .../lib/captain/prompts/scenario.liquid | 6 +++- .../captain/prompts/snippets/campaign.liquid | 8 +++++ .../models/concerns/agentable_spec.rb | 26 ++++++++++++-- .../assistant/agent_runner_service_spec.rb | 35 +++++++++++++++++++ 7 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 enterprise/lib/captain/prompts/snippets/campaign.liquid diff --git a/enterprise/app/models/concerns/agentable.rb b/enterprise/app/models/concerns/agentable.rb index e5b0b8eef..ed8e0a89f 100644 --- a/enterprise/app/models/concerns/agentable.rb +++ b/enterprise/app/models/concerns/agentable.rb @@ -19,9 +19,11 @@ module Concerns::Agentable state = context.context[:state] || {} conversation_data = state[:conversation] || {} contact_data = state[:contact] || {} + campaign_data = state[:campaign] || {} enhanced_context = enhanced_context.merge( conversation: conversation_data, - contact: contact_data + contact: contact_data, + campaign: campaign_data ) end diff --git a/enterprise/app/services/captain/assistant/agent_runner_service.rb b/enterprise/app/services/captain/assistant/agent_runner_service.rb index 72c44024f..8238ca4d8 100644 --- a/enterprise/app/services/captain/assistant/agent_runner_service.rb +++ b/enterprise/app/services/captain/assistant/agent_runner_service.rb @@ -16,6 +16,8 @@ class Captain::Assistant::AgentRunnerService custom_attributes additional_attributes ].freeze + CAMPAIGN_STATE_ATTRIBUTES = %i[id title message campaign_type description].freeze + def initialize(assistant:, conversation: nil, callbacks: {}) @assistant = assistant @conversation = conversation @@ -129,6 +131,7 @@ class Captain::Assistant::AgentRunnerService state[:conversation] = @conversation.attributes.symbolize_keys.slice(*CONVERSATION_STATE_ATTRIBUTES) state[:channel_type] = @conversation.inbox&.channel_type state[:contact] = @conversation.contact.attributes.symbolize_keys.slice(*CONTACT_STATE_ATTRIBUTES) if @conversation.contact + state[:campaign] = @conversation.campaign.attributes.symbolize_keys.slice(*CAMPAIGN_STATE_ATTRIBUTES) if @conversation.campaign end state diff --git a/enterprise/lib/captain/prompts/assistant.liquid b/enterprise/lib/captain/prompts/assistant.liquid index 0dc7d8577..7b20a089e 100644 --- a/enterprise/lib/captain/prompts/assistant.liquid +++ b/enterprise/lib/captain/prompts/assistant.liquid @@ -8,7 +8,7 @@ You are {{name}}, a helpful and knowledgeable assistant. Your role is to primari Don't digress away from your instructions, and use all the available tools at your disposal for solving customer issues. If you are to state something factual about {{product_name}} ensure you source that information from the FAQs only. Use the `captain--tools--faq_lookup` tool for this. -{% if conversation || contact -%} +{% if conversation || contact || campaign.id -%} # Current Context Here's the metadata we have about the current conversation and the contact associated with it: @@ -20,6 +20,10 @@ Here's the metadata we have about the current conversation and the contact assoc {% if contact -%} {% render 'contact' %} {% endif -%} + +{% if campaign.id -%} +{% render 'campaign' %} +{% endif -%} {% endif -%} {% if response_guidelines.size > 0 -%} diff --git a/enterprise/lib/captain/prompts/scenario.liquid b/enterprise/lib/captain/prompts/scenario.liquid index 1148a7c3a..879a39f52 100644 --- a/enterprise/lib/captain/prompts/scenario.liquid +++ b/enterprise/lib/captain/prompts/scenario.liquid @@ -8,7 +8,7 @@ You are a specialized agent called "{{ title }}", your task is to handle the fol If you believe the user's request is not within the scope of your role, you can assign this conversation back to the orchestrator agent using the `handoff_to_{{ assistant_name }}` tool -{% if conversation || contact %} +{% if conversation || contact || campaign.id %} # Current Context Here's the metadata we have about the current conversation and the contact associated with it: @@ -20,6 +20,10 @@ Here's the metadata we have about the current conversation and the contact assoc {% if contact -%} {% render 'contact' %} {% endif -%} + +{% if campaign.id -%} +{% render 'campaign' %} +{% endif -%} {% endif -%} diff --git a/enterprise/lib/captain/prompts/snippets/campaign.liquid b/enterprise/lib/captain/prompts/snippets/campaign.liquid new file mode 100644 index 000000000..db2ac0e8e --- /dev/null +++ b/enterprise/lib/captain/prompts/snippets/campaign.liquid @@ -0,0 +1,8 @@ +# Campaign Context +This conversation was initiated in response to a campaign message. +- Campaign: {{ campaign.title }} +- Type: {{ campaign.campaign_type }} +{% if campaign.description -%} +- Description: {{ campaign.description }} +{% endif -%} +- Original Message Sent: {{ campaign.message }} diff --git a/spec/enterprise/models/concerns/agentable_spec.rb b/spec/enterprise/models/concerns/agentable_spec.rb index 6b170e8d7..fbf6a58dc 100644 --- a/spec/enterprise/models/concerns/agentable_spec.rb +++ b/spec/enterprise/models/concerns/agentable_spec.rb @@ -97,7 +97,8 @@ RSpec.describe Concerns::Agentable do expected_context = { base_key: 'base_value', conversation: { id: 123 }, - contact: { name: 'John' } + contact: { name: 'John' }, + campaign: {} } expect(Captain::PromptRenderer).to receive(:render).with( @@ -108,6 +109,26 @@ RSpec.describe Concerns::Agentable do dummy_instance.agent_instructions(context_double) end + it 'merges campaign data from context state' do + context_double = instance_double(Agents::RunContext, + context: { + state: { + conversation: { id: 123 }, + contact: { name: 'John' }, + campaign: { id: 10, title: 'Summer Sale', message: 'Check it out' } + } + }) + + expect(Captain::PromptRenderer).to receive(:render).with( + 'dummy_class', + hash_including( + campaign: { id: 10, title: 'Summer Sale', message: 'Check it out' } + ) + ) + + dummy_instance.agent_instructions(context_double) + end + it 'handles context without state' do context_double = instance_double(Agents::RunContext, context: {}) @@ -116,7 +137,8 @@ RSpec.describe Concerns::Agentable do hash_including( base_key: 'base_value', conversation: {}, - contact: {} + contact: {}, + campaign: {} ) ) diff --git a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb index 13e4804ce..0d22b8266 100644 --- a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb +++ b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb @@ -394,6 +394,34 @@ RSpec.describe Captain::Assistant::AgentRunnerService do ) end + it 'does not include campaign when conversation has no campaign' do + state = service.send(:build_state) + + expect(state).not_to have_key(:campaign) + end + + context 'when conversation has a campaign' do + let(:campaign) { create(:campaign, account: account, title: 'Summer Sale', message: 'Check out our deals!', description: 'Seasonal promo') } + let(:conversation) { create(:conversation, account: account, inbox: inbox, contact: contact, campaign: campaign) } + + it 'includes campaign attributes in state' do + state = service.send(:build_state) + + expect(state[:campaign]).to include( + id: campaign.id, + title: 'Summer Sale', + message: 'Check out our deals!', + description: 'Seasonal promo' + ) + end + + it 'only includes attributes defined in CAMPAIGN_STATE_ATTRIBUTES' do + state = service.send(:build_state) + + expect(state[:campaign].keys).to match_array(described_class::CAMPAIGN_STATE_ATTRIBUTES) + end + end + context 'when conversation is nil' do subject(:service) { described_class.new(assistant: assistant, conversation: nil) } @@ -407,6 +435,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do ) expect(state).not_to have_key(:conversation) expect(state).not_to have_key(:contact) + expect(state).not_to have_key(:campaign) end end end @@ -477,5 +506,11 @@ RSpec.describe Captain::Assistant::AgentRunnerService do :id, :name, :email, :phone_number, :identifier, :contact_type ) end + + it 'defines campaign state attributes' do + expect(described_class::CAMPAIGN_STATE_ATTRIBUTES).to include( + :id, :title, :message, :campaign_type, :description + ) + end end end From 5aef9d2dd0f8a9a4b61c2e16531aaca98ed6fc2a Mon Sep 17 00:00:00 2001 From: Sivin Varghese <64252451+iamsivin@users.noreply.github.com> Date: Wed, 25 Feb 2026 20:18:34 +0530 Subject: [PATCH 03/13] fix: Conversation list overlay issue with `Virtua` virtualizer (#13648) --- .../dashboard/components/ChatList.vue | 2 -- .../dashboard/components/ConversationItem.vue | 1 - .../dashboard/components/ui/TimeAgo.vue | 14 +++++++++++- .../widgets/conversation/ConversationCard.vue | 22 ++++++++++++++----- .../widgets/conversation/PriorityMark.vue | 1 - 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/app/javascript/dashboard/components/ChatList.vue b/app/javascript/dashboard/components/ChatList.vue index a060bcc4f..eeb2ad6e7 100644 --- a/app/javascript/dashboard/components/ChatList.vue +++ b/app/javascript/dashboard/components/ChatList.vue @@ -955,10 +955,8 @@ watch(conversationFilters, (newVal, oldVal) => { ref="virtualListRef" v-slot="{ item, index }" :data="conversationList" - :overscan="10" > diff --git a/app/javascript/dashboard/components/widgets/conversation/ConversationCard.vue b/app/javascript/dashboard/components/widgets/conversation/ConversationCard.vue index 9f7805b4e..f12b0c9c1 100644 --- a/app/javascript/dashboard/components/widgets/conversation/ConversationCard.vue +++ b/app/javascript/dashboard/components/widgets/conversation/ConversationCard.vue @@ -1,5 +1,5 @@ diff --git a/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue b/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue index 3f4b31299..3bcef1ca2 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue @@ -3,6 +3,8 @@ import { useVuelidate } from '@vuelidate/core'; import { required, url, minLength } from '@vuelidate/validators'; import wootConstants from 'dashboard/constants/globals'; import { getI18nKey } from 'dashboard/routes/dashboard/settings/helper/settingsHelper'; +import { copyTextToClipboard } from 'shared/helpers/clipboard'; +import { useAlert } from 'dashboard/composables'; import NextButton from 'dashboard/components-next/button/Button.vue'; const { EXAMPLE_WEBHOOK_URL } = wootConstants; @@ -57,10 +59,14 @@ export default { url: this.value.url || '', name: this.value.name || '', subscriptions: this.value.subscriptions || [], + secretVisible: false, supportedWebhookEvents: SUPPORTED_WEBHOOK_EVENTS, }; }, computed: { + hasSecret() { + return !!this.value.secret; + }, webhookURLInputPlaceholder() { return this.$t( 'INTEGRATION_SETTINGS.WEBHOOK.FORM.END_POINT.PLACEHOLDER', @@ -81,6 +87,10 @@ export default { subscriptions: this.subscriptions, }); }, + async copySecret() { + await copyTextToClipboard(this.value.secret); + useAlert(this.$t('INTEGRATION_SETTINGS.WEBHOOK.SECRET.COPY_SUCCESS')); + }, getI18nKey, }, }; @@ -111,6 +121,35 @@ export default { :placeholder="webhookNameInputPlaceholder" /> + diff --git a/app/javascript/dashboard/store/modules/webhooks.js b/app/javascript/dashboard/store/modules/webhooks.js index eb096468e..774c173d3 100644 --- a/app/javascript/dashboard/store/modules/webhooks.js +++ b/app/javascript/dashboard/store/modules/webhooks.js @@ -42,6 +42,7 @@ export const actions = { } = response.data; commit(types.default.ADD_WEBHOOK, webhook); commit(types.default.SET_WEBHOOK_UI_FLAG, { creatingItem: false }); + return webhook; } catch (error) { commit(types.default.SET_WEBHOOK_UI_FLAG, { creatingItem: false }); throw error; diff --git a/app/jobs/webhook_job.rb b/app/jobs/webhook_job.rb index 57d3739b7..54eac45e3 100644 --- a/app/jobs/webhook_job.rb +++ b/app/jobs/webhook_job.rb @@ -1,7 +1,7 @@ class WebhookJob < ApplicationJob queue_as :medium # There are 3 types of webhooks, account, inbox and agent_bot - def perform(url, payload, webhook_type = :account_webhook) - Webhooks::Trigger.execute(url, payload, webhook_type) + def perform(url, payload, webhook_type = :account_webhook, secret: nil, delivery_id: nil) + Webhooks::Trigger.execute(url, payload, webhook_type, secret: secret, delivery_id: delivery_id) end end diff --git a/app/listeners/webhook_listener.rb b/app/listeners/webhook_listener.rb index 82a9fc711..762eaa6ee 100644 --- a/app/listeners/webhook_listener.rb +++ b/app/listeners/webhook_listener.rb @@ -111,7 +111,9 @@ class WebhookListener < BaseListener account.webhooks.account_type.each do |webhook| next unless webhook.subscriptions.include?(payload[:event]) - WebhookJob.perform_later(webhook.url, payload) + WebhookJob.perform_later(webhook.url, payload, :account_webhook, + secret: webhook.secret, + delivery_id: SecureRandom.uuid) end end @@ -119,7 +121,8 @@ class WebhookListener < BaseListener return unless inbox.channel_type == 'Channel::Api' return if inbox.channel.webhook_url.blank? - WebhookJob.perform_later(inbox.channel.webhook_url, payload, :api_inbox_webhook) + WebhookJob.perform_later(inbox.channel.webhook_url, payload, :api_inbox_webhook, + delivery_id: SecureRandom.uuid) end def deliver_webhook_payloads(payload, inbox) diff --git a/app/models/webhook.rb b/app/models/webhook.rb index 1d61c1614..6b36c4bbd 100644 --- a/app/models/webhook.rb +++ b/app/models/webhook.rb @@ -21,6 +21,9 @@ class Webhook < ApplicationRecord belongs_to :account belongs_to :inbox, optional: true + has_secure_token :secret + encrypts :secret if Chatwoot.encryption_configured? + validates :account_id, presence: true validates :url, uniqueness: { scope: [:account_id] }, format: URI::DEFAULT_PARSER.make_regexp(%w[http https]) validate :validate_webhook_subscriptions diff --git a/app/views/api/v1/accounts/webhooks/_webhook.json.jbuilder b/app/views/api/v1/accounts/webhooks/_webhook.json.jbuilder index 5406cf183..7b1943c5d 100644 --- a/app/views/api/v1/accounts/webhooks/_webhook.json.jbuilder +++ b/app/views/api/v1/accounts/webhooks/_webhook.json.jbuilder @@ -3,6 +3,7 @@ json.name webhook.name json.url webhook.url json.account_id webhook.account_id json.subscriptions webhook.subscriptions +json.secret webhook.secret if webhook.inbox json.inbox do json.id webhook.inbox.id diff --git a/db/migrate/20260218075101_add_secret_to_webhooks.rb b/db/migrate/20260218075101_add_secret_to_webhooks.rb new file mode 100644 index 000000000..ff6c40c4c --- /dev/null +++ b/db/migrate/20260218075101_add_secret_to_webhooks.rb @@ -0,0 +1,5 @@ +class AddSecretToWebhooks < ActiveRecord::Migration[7.1] + def change + add_column :webhooks, :secret, :string + end +end diff --git a/db/migrate/20260226084618_backfill_webhook_secrets.rb b/db/migrate/20260226084618_backfill_webhook_secrets.rb new file mode 100644 index 000000000..aec6cfde7 --- /dev/null +++ b/db/migrate/20260226084618_backfill_webhook_secrets.rb @@ -0,0 +1,11 @@ +class BackfillWebhookSecrets < ActiveRecord::Migration[7.1] + def up + Webhook.find_each do |webhook| + webhook.update!(secret: SecureRandom.urlsafe_base64(24)) + end + end + + def down + # no-op: removing the column in the previous migration handles cleanup + end +end diff --git a/db/schema.rb b/db/schema.rb index fd4d18cb1..8a450e734 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_01_30_061021) do +ActiveRecord::Schema[7.1].define(version: 2026_02_26_084618) do # These extensions should be enabled to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" @@ -1250,6 +1250,7 @@ ActiveRecord::Schema[7.1].define(version: 2026_01_30_061021) do t.integer "webhook_type", default: 0 t.jsonb "subscriptions", default: ["conversation_status_changed", "conversation_updated", "conversation_created", "contact_created", "contact_updated", "message_created", "message_updated", "webwidget_triggered"] t.string "name" + t.string "secret" t.index ["account_id", "url"], name: "index_webhooks_on_account_id_and_url", unique: true end diff --git a/enterprise/app/models/enterprise/audit/webhook.rb b/enterprise/app/models/enterprise/audit/webhook.rb index 34e0bcc1b..303141ce2 100644 --- a/enterprise/app/models/enterprise/audit/webhook.rb +++ b/enterprise/app/models/enterprise/audit/webhook.rb @@ -2,6 +2,6 @@ module Enterprise::Audit::Webhook extend ActiveSupport::Concern included do - audited associated_with: :account + audited associated_with: :account, except: [:secret] end end diff --git a/lib/webhooks/trigger.rb b/lib/webhooks/trigger.rb index ef3410b78..456e186ca 100644 --- a/lib/webhooks/trigger.rb +++ b/lib/webhooks/trigger.rb @@ -1,14 +1,16 @@ class Webhooks::Trigger SUPPORTED_ERROR_HANDLE_EVENTS = %w[message_created message_updated].freeze - def initialize(url, payload, webhook_type) + def initialize(url, payload, webhook_type, secret: nil, delivery_id: nil) @url = url @payload = payload @webhook_type = webhook_type + @secret = secret + @delivery_id = delivery_id end - def self.execute(url, payload, webhook_type) - new(url, payload, webhook_type).execute + def self.execute(url, payload, webhook_type, secret: nil, delivery_id: nil) + new(url, payload, webhook_type, secret: secret, delivery_id: delivery_id).execute end def execute @@ -21,15 +23,27 @@ class Webhooks::Trigger private def perform_request + body = @payload.to_json RestClient::Request.execute( method: :post, url: @url, - payload: @payload.to_json, - headers: { content_type: :json, accept: :json }, + payload: body, + headers: request_headers(body), timeout: webhook_timeout ) end + def request_headers(body) + headers = { content_type: :json, accept: :json } + headers['X-Chatwoot-Delivery'] = @delivery_id if @delivery_id.present? + if @secret.present? + ts = Time.now.to_i.to_s + headers['X-Chatwoot-Timestamp'] = ts + headers['X-Chatwoot-Signature'] = "sha256=#{OpenSSL::HMAC.hexdigest('SHA256', @secret, "#{ts}.#{body}")}" + end + headers + end + def handle_error(error) return unless SUPPORTED_ERROR_HANDLE_EVENTS.include?(@payload[:event]) return unless message @@ -72,7 +86,11 @@ class Webhooks::Trigger def message return if message_id.blank? - @message ||= Message.find_by(id: message_id) + if defined?(@message) + @message + else + @message = Message.find_by(id: message_id) + end end def message_id diff --git a/spec/jobs/agent_bots/webhook_job_spec.rb b/spec/jobs/agent_bots/webhook_job_spec.rb index a8117d84e..346d85e83 100644 --- a/spec/jobs/agent_bots/webhook_job_spec.rb +++ b/spec/jobs/agent_bots/webhook_job_spec.rb @@ -16,7 +16,7 @@ RSpec.describe AgentBots::WebhookJob do end it 'executes perform' do - expect(Webhooks::Trigger).to receive(:execute).with(url, payload, webhook_type) + expect(Webhooks::Trigger).to receive(:execute).with(url, payload, webhook_type, secret: nil, delivery_id: nil) perform_enqueued_jobs { job } end end diff --git a/spec/jobs/webhook_job_spec.rb b/spec/jobs/webhook_job_spec.rb index 81802a3c0..c74c1d8a8 100644 --- a/spec/jobs/webhook_job_spec.rb +++ b/spec/jobs/webhook_job_spec.rb @@ -16,7 +16,7 @@ RSpec.describe WebhookJob do end it 'executes perform with default webhook type' do - expect(Webhooks::Trigger).to receive(:execute).with(url, payload, webhook_type) + expect(Webhooks::Trigger).to receive(:execute).with(url, payload, webhook_type, secret: nil, delivery_id: nil) perform_enqueued_jobs { job } end @@ -24,7 +24,7 @@ RSpec.describe WebhookJob do let(:webhook_type) { :api_inbox_webhook } it 'executes perform with inbox webhook type' do - expect(Webhooks::Trigger).to receive(:execute).with(url, payload, webhook_type) + expect(Webhooks::Trigger).to receive(:execute).with(url, payload, webhook_type, secret: nil, delivery_id: nil) perform_enqueued_jobs { job } end end diff --git a/spec/lib/webhooks/trigger_spec.rb b/spec/lib/webhooks/trigger_spec.rb index 79cf92150..1e047b557 100644 --- a/spec/lib/webhooks/trigger_spec.rb +++ b/spec/lib/webhooks/trigger_spec.rb @@ -168,6 +168,71 @@ describe Webhooks::Trigger do end end + describe 'request headers' do + let(:payload) { { event: 'message_created' } } + let(:body) { payload.to_json } + + context 'without secret or delivery_id' do + it 'sends only content-type and accept headers' do + expect(RestClient::Request).to receive(:execute).with( + hash_including(headers: { content_type: :json, accept: :json }) + ) + trigger.execute(url, payload, webhook_type) + end + end + + context 'with delivery_id' do + it 'adds X-Chatwoot-Delivery header' do + expect(RestClient::Request).to receive(:execute) do |args| + expect(args[:headers]['X-Chatwoot-Delivery']).to eq('test-uuid') + expect(args[:headers]).not_to have_key('X-Chatwoot-Signature') + expect(args[:headers]).not_to have_key('X-Chatwoot-Timestamp') + end + trigger.execute(url, payload, webhook_type, delivery_id: 'test-uuid') + end + end + + context 'with secret' do + let(:secret) { 'test-secret' } + + it 'adds X-Chatwoot-Timestamp header' do + expect(RestClient::Request).to receive(:execute) do |args| + expect(args[:headers]['X-Chatwoot-Timestamp']).to match(/\A\d+\z/) + end + trigger.execute(url, payload, webhook_type, secret: secret) + end + + it 'adds X-Chatwoot-Signature header with correct HMAC' do + expect(RestClient::Request).to receive(:execute) do |args| + ts = args[:headers]['X-Chatwoot-Timestamp'] + expected_sig = "sha256=#{OpenSSL::HMAC.hexdigest('SHA256', secret, "#{ts}.#{body}")}" + expect(args[:headers]['X-Chatwoot-Signature']).to eq(expected_sig) + end + trigger.execute(url, payload, webhook_type, secret: secret) + end + + it 'signs timestamp.body not just body' do + expect(RestClient::Request).to receive(:execute) do |args| + args[:headers]['X-Chatwoot-Timestamp'] + wrong_sig = "sha256=#{OpenSSL::HMAC.hexdigest('SHA256', secret, body)}" + expect(args[:headers]['X-Chatwoot-Signature']).not_to eq(wrong_sig) + end + trigger.execute(url, payload, webhook_type, secret: secret) + end + end + + context 'with both secret and delivery_id' do + it 'includes all three security headers' do + expect(RestClient::Request).to receive(:execute) do |args| + expect(args[:headers]['X-Chatwoot-Delivery']).to eq('abc-123') + expect(args[:headers]['X-Chatwoot-Timestamp']).to be_present + expect(args[:headers]['X-Chatwoot-Signature']).to start_with('sha256=') + end + trigger.execute(url, payload, webhook_type, secret: 'mysecret', delivery_id: 'abc-123') + end + end + end + it 'does not update message status if webhook fails for other events' do payload = { event: 'conversation_created', conversation: { id: conversation.id }, id: message.id } diff --git a/spec/listeners/webhook_listener_spec.rb b/spec/listeners/webhook_listener_spec.rb index 5062b11bc..51dae239b 100644 --- a/spec/listeners/webhook_listener_spec.rb +++ b/spec/listeners/webhook_listener_spec.rb @@ -28,7 +28,10 @@ describe WebhookListener do context 'when webhook is configured and event is subscribed' do it 'triggers the webhook event' do webhook = create(:webhook, inbox: inbox, account: account) - expect(WebhookJob).to receive(:perform_later).with(webhook.url, message.webhook_data.merge(event: 'message_created')).once + expect(WebhookJob).to receive(:perform_later).with( + webhook.url, message.webhook_data.merge(event: 'message_created'), :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) + ).once listener.message_created(message_created_event) end end @@ -54,8 +57,10 @@ describe WebhookListener do conversation: api_conversation ) api_event = Events::Base.new(event_name, Time.zone.now, message: api_message) - expect(WebhookJob).to receive(:perform_later).with(channel_api.webhook_url, api_message.webhook_data.merge(event: 'message_created'), - :api_inbox_webhook).once + expect(WebhookJob).to receive(:perform_later).with( + channel_api.webhook_url, api_message.webhook_data.merge(event: 'message_created'), + :api_inbox_webhook, delivery_id: instance_of(String) + ).once listener.message_created(api_event) end @@ -90,7 +95,10 @@ describe WebhookListener do context 'when webhook is configured' do it 'triggers webhook' do webhook = create(:webhook, inbox: inbox, account: account) - expect(WebhookJob).to receive(:perform_later).with(webhook.url, conversation.webhook_data.merge(event: 'conversation_created')).once + expect(WebhookJob).to receive(:perform_later).with( + webhook.url, conversation.webhook_data.merge(event: 'conversation_created'), :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) + ).once listener.conversation_created(conversation_created_event) end end @@ -101,9 +109,11 @@ describe WebhookListener do api_inbox = channel_api.inbox api_conversation = create(:conversation, account: account, inbox: api_inbox, assignee: user) api_event = Events::Base.new(event_name, Time.zone.now, conversation: api_conversation) - expect(WebhookJob).to receive(:perform_later).with(channel_api.webhook_url, - api_conversation.webhook_data.merge(event: 'conversation_created'), - :api_inbox_webhook).once + expect(WebhookJob).to receive(:perform_later).with( + channel_api.webhook_url, + api_conversation.webhook_data.merge(event: 'conversation_created'), + :api_inbox_webhook, delivery_id: instance_of(String) + ).once listener.conversation_created(api_event) end @@ -156,7 +166,9 @@ describe WebhookListener do } } ] - ) + ), + :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) ).once listener.conversation_updated(conversation_updated_event) @@ -177,7 +189,10 @@ describe WebhookListener do context 'when webhook is configured' do it 'triggers webhook' do webhook = create(:webhook, account: account) - expect(WebhookJob).to receive(:perform_later).with(webhook.url, contact.webhook_data.merge(event: 'contact_created')).once + expect(WebhookJob).to receive(:perform_later).with( + webhook.url, contact.webhook_data.merge(event: 'contact_created'), :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) + ).once listener.contact_created(contact_event) end end @@ -213,7 +228,9 @@ describe WebhookListener do contact.webhook_data.merge( event: 'contact_updated', changed_attributes: [{ 'name' => { :current_value => 'Jane Doe', :previous_value => 'Jane' } }] - ) + ), + :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) ).once listener.contact_updated(contact_updated_event) end @@ -235,7 +252,10 @@ describe WebhookListener do it 'triggers webhook' do inbox_data = Inbox::EventDataPresenter.new(inbox).push_data webhook = create(:webhook, account: account, subscriptions: ['inbox_created']) - expect(WebhookJob).to receive(:perform_later).with(webhook.url, inbox_data.merge(event: 'inbox_created')).once + expect(WebhookJob).to receive(:perform_later).with( + webhook.url, inbox_data.merge(event: 'inbox_created'), :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) + ).once listener.inbox_created(inbox_created_event) end end @@ -272,7 +292,9 @@ describe WebhookListener do expect(WebhookJob).to receive(:perform_later).with( webhook.url, - inbox_data.merge(event: 'inbox_updated', changed_attributes: changed_attributes_data) + inbox_data.merge(event: 'inbox_updated', changed_attributes: changed_attributes_data), + :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) ).once listener.inbox_updated(inbox_updated_event) @@ -302,7 +324,10 @@ describe WebhookListener do is_private: false } - expect(WebhookJob).to receive(:perform_later).with(webhook.url, payload).once + expect(WebhookJob).to receive(:perform_later).with( + webhook.url, payload, :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) + ).once listener.conversation_typing_on(typing_event) end end @@ -321,7 +346,10 @@ describe WebhookListener do is_private: false } - expect(WebhookJob).to receive(:perform_later).with(channel_api.webhook_url, payload, :api_inbox_webhook).once + expect(WebhookJob).to receive(:perform_later).with( + channel_api.webhook_url, payload, :api_inbox_webhook, + delivery_id: instance_of(String) + ).once listener.conversation_typing_on(api_event) end end @@ -349,7 +377,10 @@ describe WebhookListener do is_private: false } - expect(WebhookJob).to receive(:perform_later).with(webhook.url, payload).once + expect(WebhookJob).to receive(:perform_later).with( + webhook.url, payload, :account_webhook, + secret: webhook.secret, delivery_id: instance_of(String) + ).once listener.conversation_typing_off(typing_event) end end diff --git a/spec/models/webhook_spec.rb b/spec/models/webhook_spec.rb index 81e6d9551..b8570de59 100644 --- a/spec/models/webhook_spec.rb +++ b/spec/models/webhook_spec.rb @@ -8,4 +8,20 @@ RSpec.describe Webhook do describe 'associations' do it { is_expected.to belong_to(:account) } end + + describe 'secret token' do + let!(:account) { create(:account) } + + it 'auto-generates a secret on create' do + webhook = create(:webhook, account: account) + expect(webhook.secret).to be_present + end + + it 'does not regenerate the secret on update' do + webhook = create(:webhook, account: account) + original_secret = webhook.secret + webhook.update!(url: "#{webhook.url}?updated=1") + expect(webhook.reload.secret).to eq(original_secret) + end + end end From 9ca03c1af31e4b059d1a97324db1a68f3f7a81eb Mon Sep 17 00:00:00 2001 From: Tanmay Deep Sharma <32020192+tds-1@users.noreply.github.com> Date: Thu, 26 Feb 2026 18:01:13 +0530 Subject: [PATCH 12/13] chore: make all the deprecated feature flag reclaimable (#13646) ## Docs https://www.notion.so/chatwoot/Redeeming-a-depreciated-feature-flag-313a5f274c9280f381cdd811eab42019?source=copy_link ## Description Marks 8 unused feature flags as deprecated: true in features.yml, freeing their bit slots for future reuse. Removes dead code references from JS constants, help URLs, and enterprise billing config. ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? - Simulated the "claim a slot" workflow ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- app/javascript/dashboard/featureFlags.js | 3 --- app/javascript/dashboard/helper/featureHelper.js | 1 - config/features.yml | 4 +--- .../enterprise/billing/handle_stripe_event_service.rb | 1 - enterprise/config/premium_features.yml | 1 - 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/app/javascript/dashboard/featureFlags.js b/app/javascript/dashboard/featureFlags.js index 353bed96e..858c0ecbc 100644 --- a/app/javascript/dashboard/featureFlags.js +++ b/app/javascript/dashboard/featureFlags.js @@ -21,10 +21,8 @@ export const FEATURE_FLAGS = { AUDIT_LOGS: 'audit_logs', INBOX_VIEW: 'inbox_view', SLA: 'sla', - RESPONSE_BOT: 'response_bot', CHANNEL_EMAIL: 'channel_email', CHANNEL_FACEBOOK: 'channel_facebook', - CHANNEL_TWITTER: 'channel_twitter', CHANNEL_WEBSITE: 'channel_website', CUSTOM_REPLY_DOMAIN: 'custom_reply_domain', CUSTOM_REPLY_EMAIL: 'custom_reply_email', @@ -36,7 +34,6 @@ export const FEATURE_FLAGS = { CAPTAIN: 'captain_integration', CUSTOM_ROLES: 'custom_roles', CHATWOOT_V4: 'chatwoot_v4', - REPORT_V4: 'report_v4', CHANNEL_INSTAGRAM: 'channel_instagram', CHANNEL_TIKTOK: 'channel_tiktok', CONTACT_CHATWOOT_SUPPORT_TEAM: 'contact_chatwoot_support_team', diff --git a/app/javascript/dashboard/helper/featureHelper.js b/app/javascript/dashboard/helper/featureHelper.js index ae805ccf1..c90ec15db 100644 --- a/app/javascript/dashboard/helper/featureHelper.js +++ b/app/javascript/dashboard/helper/featureHelper.js @@ -13,7 +13,6 @@ const FEATURE_HELP_URLS = { integrations: 'https://chwt.app/hc/integrations', labels: 'https://chwt.app/hc/labels', macros: 'https://chwt.app/hc/macros', - message_reply_to: 'https://chwt.app/hc/reply-to', reports: 'https://chwt.app/hc/reports', sla: 'https://chwt.app/hc/sla', team_management: 'https://chwt.app/hc/teams', diff --git a/config/features.yml b/config/features.yml index d8378d61c..65b3c6194 100644 --- a/config/features.yml +++ b/config/features.yml @@ -108,12 +108,10 @@ - name: response_bot display_name: Response Bot enabled: false - premium: true deprecated: true - name: message_reply_to display_name: Message Reply To enabled: false - help_url: https://chwt.app/hc/reply-to deprecated: true - name: insert_article_in_reply display_name: Insert Article in Reply @@ -149,7 +147,7 @@ enabled: true - name: report_v4 display_name: Report V4 - enabled: true + enabled: false deprecated: true - name: contact_chatwoot_support_team display_name: Contact Chatwoot Support Team diff --git a/enterprise/app/services/enterprise/billing/handle_stripe_event_service.rb b/enterprise/app/services/enterprise/billing/handle_stripe_event_service.rb index 52a28844f..d3c5b15db 100644 --- a/enterprise/app/services/enterprise/billing/handle_stripe_event_service.rb +++ b/enterprise/app/services/enterprise/billing/handle_stripe_event_service.rb @@ -11,7 +11,6 @@ class Enterprise::Billing::HandleStripeEventService help_center campaigns team_management - channel_twitter channel_facebook channel_email channel_instagram diff --git a/enterprise/config/premium_features.yml b/enterprise/config/premium_features.yml index 64275503d..0cb89df01 100644 --- a/enterprise/config/premium_features.yml +++ b/enterprise/config/premium_features.yml @@ -1,7 +1,6 @@ # List of the premium features in EE edition - disable_branding - audit_logs -- response_bot - sla - custom_roles - captain_integration From 7acd239c7050887fc9b0c5dd562e545a96080abc Mon Sep 17 00:00:00 2001 From: Tanmay Deep Sharma <32020192+tds-1@users.noreply.github.com> Date: Thu, 26 Feb 2026 18:01:23 +0530 Subject: [PATCH 13/13] fix: call authorization_error! on IMAP auth failures (#13560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Notion document https://www.notion.so/chatwoot/Email-IMAP-Issue-30aa5f274c928062aa6bddc2e5877a63?showMoveTo=true&saveParent=true ## Description PLAIN IMAP channels (non-OAuth) were silently retrying failed authentication every minute, forever. When credentials are wrong/expired, Net::IMAP::NoResponseError was caught and logged but channel.authorization_error! was never called — so the Redis error counter never incremented, reauthorization_required? was never set, and admins were never notified. OAuth channels already had this handled correctly via the Reauthorizable concern. Additionally, Net::IMAP::ResponseParseError (raised by non-RFC-compliant IMAP servers) was falling through to the StandardError catch-all, flooding Estimated impact before fix: ~70–75 broken IMAP inboxes generating ~700k–750k wasted Sidekiq jobs/week. ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- .../inboxes/fetch_imap_email_inboxes_job.rb | 1 + app/jobs/inboxes/fetch_imap_emails_job.rb | 29 ++++---- app/models/channel/email.rb | 4 ++ app/models/concerns/backoffable.rb | 70 +++++++++++++++++++ app/models/concerns/reauthorizable.rb | 3 + app/services/imap/authentication_error.rb | 1 + app/services/imap/base_fetch_email_service.rb | 6 +- config/installation_config.yml | 13 ++++ lib/exception_list.rb | 6 ++ lib/redis/redis_keys.rb | 4 ++ .../inboxes/fetch_imap_emails_job_spec.rb | 50 +++++++++++++ spec/models/channel/email_spec.rb | 2 + spec/models/concerns/backoffable_shared.rb | 43 ++++++++++++ 13 files changed, 218 insertions(+), 14 deletions(-) create mode 100644 app/models/concerns/backoffable.rb create mode 100644 app/services/imap/authentication_error.rb create mode 100644 spec/models/concerns/backoffable_shared.rb diff --git a/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb b/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb index ea2705955..ea7f7664f 100644 --- a/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb +++ b/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb @@ -15,6 +15,7 @@ class Inboxes::FetchImapEmailInboxesJob < ApplicationJob return false if inbox.account.suspended? return false unless inbox.channel.imap_enabled return false if inbox.channel.reauthorization_required? + return false if inbox.channel.in_backoff? return true unless ChatwootApp.chatwoot_cloud? return false if default_plan?(inbox.account) diff --git a/app/jobs/inboxes/fetch_imap_emails_job.rb b/app/jobs/inboxes/fetch_imap_emails_job.rb index ec5717f3b..74368c830 100644 --- a/app/jobs/inboxes/fetch_imap_emails_job.rb +++ b/app/jobs/inboxes/fetch_imap_emails_job.rb @@ -6,26 +6,29 @@ class Inboxes::FetchImapEmailsJob < MutexApplicationJob def perform(channel, interval = 1) return unless should_fetch_email?(channel) - key = format(::Redis::Alfred::EMAIL_MESSAGE_MUTEX, inbox_id: channel.inbox.id) - - with_lock(key, 5.minutes) do - process_email_for_channel(channel, interval) - end - rescue *ExceptionList::IMAP_EXCEPTIONS => e - Rails.logger.error "Authorization error for email channel - #{channel.inbox.id} : #{e.message}" - rescue EOFError, OpenSSL::SSL::SSLError, Net::IMAP::NoResponseError, Net::IMAP::BadResponseError, Net::IMAP::InvalidResponseError, - Net::IMAP::ResponseParseError, Net::IMAP::ResponseReadError, Net::IMAP::ResponseTooLargeError => e - Rails.logger.error "Error for email channel - #{channel.inbox.id} : #{e.message}" - rescue LockAcquisitionError - Rails.logger.error "Lock failed for #{channel.inbox.id}" + fetch_emails_with_backoff(channel, interval) rescue StandardError => e ChatwootExceptionTracker.new(e, account: channel.account).capture_exception end private + def fetch_emails_with_backoff(channel, interval) + key = format(::Redis::Alfred::EMAIL_MESSAGE_MUTEX, inbox_id: channel.inbox.id) + with_lock(key, 5.minutes) { process_email_for_channel(channel, interval) } + channel.clear_backoff! + rescue Imap::AuthenticationError => e + Rails.logger.error "#{channel.backoff_log_identifier} authentication error : #{e.message}" + channel.authorization_error! + rescue *ExceptionList::IMAP_TRANSIENT_EXCEPTIONS => e + Rails.logger.error "#{channel.backoff_log_identifier} transient error : #{e.message}" + channel.apply_backoff! + rescue LockAcquisitionError + Rails.logger.error "Lock failed for #{channel.inbox.id}" + end + def should_fetch_email?(channel) - channel.imap_enabled? && !channel.reauthorization_required? + channel.imap_enabled? && !channel.reauthorization_required? && !channel.in_backoff? end def process_email_for_channel(channel, interval) diff --git a/app/models/channel/email.rb b/app/models/channel/email.rb index b1124dd75..1091b17d8 100644 --- a/app/models/channel/email.rb +++ b/app/models/channel/email.rb @@ -72,6 +72,10 @@ class Channel::Email < ApplicationRecord imap_enabled && imap_address == 'imap.gmail.com' end + def backoff_log_identifier + "Error for email channel - #{inbox.id}" + end + private def ensure_forward_to_email diff --git a/app/models/concerns/backoffable.rb b/app/models/concerns/backoffable.rb new file mode 100644 index 000000000..60b6ac98c --- /dev/null +++ b/app/models/concerns/backoffable.rb @@ -0,0 +1,70 @@ +# Backoffable provides transient-error retry backoff for models that depend on external services. +# +# When a transient error occurs (network hiccup, SSL failure, etc.) call apply_backoff!. +# The wait time ramps from 1 minute up to BACKOFF_MAX_INTERVAL_MINUTES, then holds at that +# ceiling for BACKOFF_MAX_INTERVAL_COUNT more attempts before calling prompt_reauthorization!. +# +# Call clear_backoff! after a successful operation to reset the counter. + +module Backoffable + extend ActiveSupport::Concern + + def backoff_log_identifier + inbox_id = respond_to?(:inbox) && inbox&.id + inbox_id ? "#{self.class.name} - #{inbox_id}" : "#{self.class.name}##{id}" + end + + def backoff_retry_count + ::Redis::Alfred.get(backoff_retry_count_key).to_i + end + + def in_backoff? + val = ::Redis::Alfred.get(backoff_retry_after_key) + val.present? && Time.zone.at(val.to_f) > Time.current + end + + def apply_backoff! + new_count = backoff_retry_count + 1 + max_interval, max_retries = backoff_limits + + if new_count > max_retries + exhaust_backoff(new_count) + else + schedule_backoff_retry(new_count, max_interval, max_retries) + end + end + + def clear_backoff! + ::Redis::Alfred.delete(backoff_retry_count_key) + ::Redis::Alfred.delete(backoff_retry_after_key) + end + + private + + def backoff_limits + max_interval = GlobalConfigService.load('BACKOFF_MAX_INTERVAL_MINUTES', 5).to_i + max_count = GlobalConfigService.load('BACKOFF_MAX_INTERVAL_COUNT', 10).to_i + [max_interval, (max_interval - 1) + max_count] + end + + def exhaust_backoff(new_count) + Rails.logger.warn "#{backoff_log_identifier} backoff exhausted (#{new_count} failures), prompting reauthorization" + clear_backoff! + prompt_reauthorization! + end + + def schedule_backoff_retry(new_count, max_interval, max_retries) + wait_minutes = [new_count, max_interval].min + ::Redis::Alfred.set(backoff_retry_count_key, new_count.to_s, ex: 24.hours) + ::Redis::Alfred.set(backoff_retry_after_key, wait_minutes.minutes.from_now.to_f.to_s, ex: 24.hours) + Rails.logger.warn "#{backoff_log_identifier} backoff retry #{new_count}/#{max_retries}, next attempt in #{wait_minutes}m" + end + + def backoff_retry_count_key + format(::Redis::Alfred::BACKOFF_RETRY_COUNT, obj_type: self.class.table_name.singularize, obj_id: id) + end + + def backoff_retry_after_key + format(::Redis::Alfred::BACKOFF_RETRY_AFTER, obj_type: self.class.table_name.singularize, obj_id: id) + end +end diff --git a/app/models/concerns/reauthorizable.rb b/app/models/concerns/reauthorizable.rb index 7a09f6436..b25f5525e 100644 --- a/app/models/concerns/reauthorizable.rb +++ b/app/models/concerns/reauthorizable.rb @@ -13,6 +13,8 @@ module Reauthorizable extend ActiveSupport::Concern + include Backoffable + AUTHORIZATION_ERROR_THRESHOLD = 2 # model attribute @@ -65,6 +67,7 @@ module Reauthorizable def reauthorized! ::Redis::Alfred.delete(authorization_error_count_key) ::Redis::Alfred.delete(reauthorization_required_key) + clear_backoff! invalidate_inbox_cache unless instance_of?(::AutomationRule) end diff --git a/app/services/imap/authentication_error.rb b/app/services/imap/authentication_error.rb new file mode 100644 index 000000000..5698f5b3b --- /dev/null +++ b/app/services/imap/authentication_error.rb @@ -0,0 +1 @@ +class Imap::AuthenticationError < StandardError; end diff --git a/app/services/imap/base_fetch_email_service.rb b/app/services/imap/base_fetch_email_service.rb index 09332092c..49e06491b 100644 --- a/app/services/imap/base_fetch_email_service.rb +++ b/app/services/imap/base_fetch_email_service.rb @@ -107,7 +107,11 @@ class Imap::BaseFetchEmailService def build_imap_client imap = Net::IMAP.new(channel.imap_address, port: channel.imap_port, ssl: true) - imap.authenticate(authentication_type, channel.imap_login, imap_password) + begin + imap.authenticate(authentication_type, channel.imap_login, imap_password) + rescue Net::IMAP::NoResponseError => e + raise Imap::AuthenticationError, e.message + end imap.select('INBOX') imap end diff --git a/config/installation_config.yml b/config/installation_config.yml index 34cb736bf..0d22de6fc 100644 --- a/config/installation_config.yml +++ b/config/installation_config.yml @@ -96,6 +96,19 @@ locked: false # ------- End of Account Related Config ------- # +# ------- Transient Error Backoff Config ------- # +- name: BACKOFF_MAX_INTERVAL_MINUTES + display_title: 'Backoff Max Interval (minutes)' + description: 'Maximum wait time in minutes between retry attempts before the backoff plateaus' + value: 5 + locked: false +- name: BACKOFF_MAX_INTERVAL_COUNT + display_title: 'Backoff Max Retry Count' + description: 'Number of additional retries at the maximum interval before prompting reauthorization' + value: 10 + locked: false +# ------- End of Transient Error Backoff Config ------- # + # ------- Email Related Config ------- # - name: MAILER_INBOUND_EMAIL_DOMAIN display_title: 'Inbound Email Domain' diff --git a/lib/exception_list.rb b/lib/exception_list.rb index 2fee0a170..dd468f07b 100644 --- a/lib/exception_list.rb +++ b/lib/exception_list.rb @@ -16,4 +16,10 @@ module ExceptionList Errno::ECONNRESET, Errno::ENETUNREACH, Net::IMAP::ByeResponseError, SocketError ].freeze + + IMAP_TRANSIENT_EXCEPTIONS = (IMAP_EXCEPTIONS + [ + EOFError, OpenSSL::SSL::SSLError, Net::IMAP::NoResponseError, Net::IMAP::BadResponseError, + Net::IMAP::InvalidResponseError, Net::IMAP::ResponseParseError, + Net::IMAP::ResponseReadError, Net::IMAP::ResponseTooLargeError + ]).freeze end diff --git a/lib/redis/redis_keys.rb b/lib/redis/redis_keys.rb index 8c9361ab5..fa9eeb31e 100644 --- a/lib/redis/redis_keys.rb +++ b/lib/redis/redis_keys.rb @@ -52,4 +52,8 @@ module Redis::RedisKeys ## Account Email Rate Limiting ACCOUNT_OUTBOUND_EMAIL_COUNT_KEY = 'OUTBOUND_EMAIL_COUNT::%d::%s'.freeze + + ## Transient Error Backoff + BACKOFF_RETRY_COUNT = 'BACKOFF:%s:%d:retry_count'.freeze + BACKOFF_RETRY_AFTER = 'BACKOFF:%s:%d:retry_after'.freeze end diff --git a/spec/jobs/inboxes/fetch_imap_emails_job_spec.rb b/spec/jobs/inboxes/fetch_imap_emails_job_spec.rb index da4b95b15..1abd7450b 100644 --- a/spec/jobs/inboxes/fetch_imap_emails_job_spec.rb +++ b/spec/jobs/inboxes/fetch_imap_emails_job_spec.rb @@ -38,6 +38,14 @@ RSpec.describe Inboxes::FetchImapEmailsJob do end end + context 'when channel is in backoff' do + it 'does not fetch emails' do + allow(imap_email_channel).to receive(:in_backoff?).and_return(true) + expect(Imap::FetchEmailService).not_to receive(:new) + described_class.perform_now(imap_email_channel) + end + end + context 'when the channel is regular imap' do it 'calls the imap fetch service' do fetch_service = double @@ -56,6 +64,17 @@ RSpec.describe Inboxes::FetchImapEmailsJob do described_class.perform_now(imap_email_channel, 4) expect(fetch_service).to have_received(:perform) end + + it 'clears backoff after successful fetch' do + fetch_service = double + allow(Imap::FetchEmailService).to receive(:new).and_return(fetch_service) + allow(fetch_service).to receive(:perform).and_return([]) + allow(imap_email_channel).to receive(:clear_backoff!) + + described_class.perform_now(imap_email_channel) + + expect(imap_email_channel).to have_received(:clear_backoff!) + end end context 'when the channel is Microsoft' do @@ -69,6 +88,37 @@ RSpec.describe Inboxes::FetchImapEmailsJob do end end + context 'when authentication error is raised' do + it 'calls authorization_error! on the channel' do + allow(Imap::FetchEmailService).to receive(:new).and_raise(Imap::AuthenticationError) + allow(imap_email_channel).to receive(:authorization_error!) + + described_class.perform_now(imap_email_channel) + + expect(imap_email_channel).to have_received(:authorization_error!) + end + end + + context 'when a transient IMAP error is raised' do + it 'calls apply_backoff! on the channel' do + allow(Imap::FetchEmailService).to receive(:new).and_raise(EOFError) + allow(imap_email_channel).to receive(:apply_backoff!) + + described_class.perform_now(imap_email_channel) + + expect(imap_email_channel).to have_received(:apply_backoff!) + end + end + + context 'when lock acquisition fails' do + it 'does not raise an error' do + lock_manager = instance_double(Redis::LockManager, lock: false) + allow(Redis::LockManager).to receive(:new).and_return(lock_manager) + + expect { described_class.perform_now(imap_email_channel) }.not_to raise_error + end + end + context 'when IMAP OAuth errors out' do it 'marks the connection as requiring authorization' do error_response = double diff --git a/spec/models/channel/email_spec.rb b/spec/models/channel/email_spec.rb index 939e8e10a..bdb9f859b 100644 --- a/spec/models/channel/email_spec.rb +++ b/spec/models/channel/email_spec.rb @@ -2,12 +2,14 @@ require 'rails_helper' require Rails.root.join 'spec/models/concerns/reauthorizable_shared.rb' +require Rails.root.join 'spec/models/concerns/backoffable_shared.rb' RSpec.describe Channel::Email do let(:channel) { create(:channel_email) } describe 'concerns' do it_behaves_like 'reauthorizable' + it_behaves_like 'backoffable' context 'when prompt_reauthorization!' do it 'calls channel notifier mail for email' do diff --git a/spec/models/concerns/backoffable_shared.rb b/spec/models/concerns/backoffable_shared.rb new file mode 100644 index 000000000..385f93be7 --- /dev/null +++ b/spec/models/concerns/backoffable_shared.rb @@ -0,0 +1,43 @@ +require 'rails_helper' + +shared_examples_for 'backoffable' do + let(:obj) { FactoryBot.create(described_class.to_s.underscore.tr('/', '_').to_sym) } + + before do + allow(GlobalConfigService).to receive(:load).with('BACKOFF_MAX_INTERVAL_MINUTES', 5).and_return(2) + allow(GlobalConfigService).to receive(:load).with('BACKOFF_MAX_INTERVAL_COUNT', 10).and_return(3) + # max_interval=2, max_retries=(2-1)+3=4; exhausts on 5th apply_backoff! + end + + it 'starts with no backoff' do + expect(obj.in_backoff?).to be false + expect(obj.backoff_retry_count).to eq 0 + end + + it 'ramps backoff on each failure' do + obj.apply_backoff! + expect(obj.backoff_retry_count).to eq 1 + expect(obj.in_backoff?).to be true + end + + it 'caps wait time at max interval' do + 4.times { obj.apply_backoff! } + expect(obj.backoff_retry_count).to eq 4 + expect(obj.in_backoff?).to be true + end + + it 'exhausts backoff and calls prompt_reauthorization! after max retries' do + allow(obj).to receive(:prompt_reauthorization!) + 5.times { obj.apply_backoff! } + expect(obj).to have_received(:prompt_reauthorization!) + expect(obj.backoff_retry_count).to eq 0 + expect(obj.in_backoff?).to be false + end + + it 'clear_backoff! resets retry count and backoff window' do + obj.apply_backoff! + obj.clear_backoff! + expect(obj.in_backoff?).to be false + expect(obj.backoff_retry_count).to eq 0 + end +end