Merge branch 'codex/cw-7431-document-usage' into codex/ai-138-v2-faq-citations

VERSION_CW

@@ -1 +1 @@
-4.16.0
+4.16.1

app/javascript/dashboard/components/widgets/conversation/MessagesView.vue

@@ -33,9 +33,7 @@ import {
 // constants
 import { BUS_EVENTS } from 'shared/constants/busEvents';
 import { REPLY_POLICY } from 'shared/constants/links';
-import wootConstants, {
-  META_RESTRICTION_STATUS_URL,
-} from 'dashboard/constants/globals';
+import wootConstants from 'dashboard/constants/globals';
 import { LOCAL_STORAGE_KEYS } from 'dashboard/constants/localStorage';
 import { INBOX_TYPES } from 'dashboard/helper/inbox';
 
@@ -95,7 +93,6 @@ export default {
       currentUserId: 'getCurrentUserID',
       listLoadingStatus: 'getAllMessagesLoaded',
       currentAccountId: 'getCurrentAccountId',
-      isOnChatwootCloud: 'globalConfig/isOnChatwootCloud',
     }),
     isOpen() {
       return this.currentChat?.status === wootConstants.STATUS_TYPE.OPEN;
@@ -173,13 +170,6 @@ export default {
         instagramInbox
       );
     },
-    isInstagramRestrictionBannerVisible() {
-      return this.isOnChatwootCloud && this.isAnInstagramChannel;
-    },
-    instagramRestrictionStatusUrl() {
-      return META_RESTRICTION_STATUS_URL;
-    },
-
     replyWindowBannerMessage() {
       if (this.isAWhatsAppChannel) {
         return this.$t('CONVERSATION.TWILIO_WHATSAPP_CAN_REPLY');
@@ -464,15 +454,7 @@ export default {
   >
     <div ref="topBannerRef">
       <Banner
-        v-if="isInstagramRestrictionBannerVisible"
-        color-scheme="warning"
-        class="mx-2 mt-2 overflow-hidden rounded-lg"
-        :banner-message="$t('CONVERSATION.INSTAGRAM_RESTRICTION_BANNER')"
-        :href-link="instagramRestrictionStatusUrl"
-        :href-link-text="$t('CONVERSATION.INSTAGRAM_RESTRICTION_STATUS_LINK')"
-      />
-      <Banner
-        v-else-if="!currentChat.can_reply"
+        v-if="!currentChat.can_reply"
         color-scheme="alert"
         class="mx-2 mt-2 overflow-hidden rounded-lg"
         :banner-message="replyWindowBannerMessage"

app/javascript/dashboard/constants/globals.js

@@ -78,5 +78,3 @@ export default {
   },
 };
 export const DEFAULT_REDIRECT_URL = '/app/';
-export const META_RESTRICTION_STATUS_URL =
-  'https://status.chatwoot.com/incident/948346';

app/javascript/dashboard/i18n/locale/en/conversation.json

@@ -44,8 +44,6 @@
     "TWILIO_WHATSAPP_CAN_REPLY": "You can only reply to this conversation using a template message due to",
     "TWILIO_WHATSAPP_24_HOURS_WINDOW": "24 hour message window restriction",
     "OLD_INSTAGRAM_INBOX_REPLY_BANNER": "This Instagram account was migrated to the new Instagram channel inbox. All new messages will show up there. You won’t be able to send messages from this conversation anymore.",
-    "INSTAGRAM_RESTRICTION_BANNER": "Instagram is currently restricted. Some messages or actions may be delayed or unavailable while we restore full support.",
-    "INSTAGRAM_RESTRICTION_STATUS_LINK": "View status update",
     "REPLYING_TO": "You are replying to:",
     "REMOVE_SELECTION": "Remove Selection",
     "DOWNLOAD": "Download",

app/javascript/dashboard/i18n/locale/en/inboxMgmt.json

@@ -58,9 +58,7 @@
         "ERROR_MESSAGE": "There was an error connecting to Instagram, please try again",
         "ERROR_AUTH": "There was an error connecting to Instagram, please try again",
         "NEW_INBOX_SUGGESTION": "This Instagram account was previously linked to a different inbox and has now been migrated here. All new messages will appear here. The old inbox will no longer be able to send or receive messages for this account.",
-        "DUPLICATE_INBOX_BANNER": "This Instagram account was migrated to the new Instagram channel inbox. You won’t be able to send/receive Instagram messages from this inbox anymore.",
-        "SETTINGS_RESTRICTED_WARNING": "Instagram is currently restricted. Some messages or actions may be delayed or unavailable while we restore full support.",
-        "STATUS_LINK": "View status update"
+        "DUPLICATE_INBOX_BANNER": "This Instagram account was migrated to the new Instagram channel inbox. You won’t be able to send/receive Instagram messages from this inbox anymore."
       },
       "TIKTOK": {
         "CONTINUE_WITH_TIKTOK": "Continue with TikTok",

app/javascript/dashboard/routes/dashboard/settings/inbox/Settings.vue

@@ -4,8 +4,6 @@ import { shouldBeUrl } from 'shared/helpers/Validators';
 import { useAlert } from 'dashboard/composables';
 import { useVuelidate } from '@vuelidate/core';
 import Avatar from 'next/avatar/Avatar.vue';
-import Banner from 'dashboard/components-next/banner/Banner.vue';
-import Icon from 'dashboard/components-next/icon/Icon.vue';
 import SettingIntroBanner from 'dashboard/components/widgets/SettingIntroBanner.vue';
 import SettingsToggleSection from 'dashboard/components-next/Settings/SettingsToggleSection.vue';
 import SettingsFieldSection from 'dashboard/components-next/Settings/SettingsFieldSection.vue';
@@ -46,11 +44,9 @@ import SelectInput from 'dashboard/components-next/select/Select.vue';
 import Widget from 'dashboard/modules/widget-preview/components/Widget.vue';
 import AccessToken from 'dashboard/routes/dashboard/settings/profile/AccessToken.vue';
 import { copyTextToClipboard } from 'shared/helpers/clipboard';
-import { META_RESTRICTION_STATUS_URL } from 'dashboard/constants/globals';
 
 export default {
   components: {
-    Banner,
     BotConfiguration,
     CollaboratorsPage,
     ConfigurationPage,
@@ -84,7 +80,6 @@ export default {
     WhatsappManualMigrationBanner,
     Widget,
     AccessToken,
-    Icon,
   },
   mixins: [inboxMixin],
   setup() {
@@ -348,12 +343,6 @@ export default {
     instagramUnauthorized() {
       return this.isAnInstagramChannel && this.inbox.reauthorization_required;
     },
-    showInstagramRestrictionSettingsBanner() {
-      return this.isOnChatwootCloud && this.isAnInstagramChannel;
-    },
-    metaRestrictionStatusUrl() {
-      return META_RESTRICTION_STATUS_URL;
-    },
     tiktokUnauthorized() {
       return this.isATiktokChannel && this.inbox.reauthorization_required;
     },
@@ -820,29 +809,6 @@ export default {
           :class="bannerMaxWidth"
           @start="openWhatsAppManualMigrationDialog"
         />
-        <Banner
-          v-if="showInstagramRestrictionSettingsBanner"
-          color="amber"
-          class="mx-6 mb-4 max-w-4xl"
-        >
-          <div class="flex items-start gap-3 text-start">
-            <Icon
-              icon="i-lucide-triangle-alert"
-              class="flex-shrink-0 size-4 mt-0.5"
-            />
-            <span>
-              {{ $t('INBOX_MGMT.ADD.INSTAGRAM.SETTINGS_RESTRICTED_WARNING') }}
-              <a
-                :href="metaRestrictionStatusUrl"
-                class="link underline"
-                rel="noopener noreferrer nofollow"
-                target="_blank"
-              >
-                {{ $t('INBOX_MGMT.ADD.INSTAGRAM.STATUS_LINK') }}
-              </a>
-            </span>
-          </div>
-        </Banner>
 
         <div
           v-if="selectedTabKey === 'inbox-settings'"

config/app.yml

@@ -1,5 +1,5 @@
 shared: &shared
-  version: '4.16.0'
+  version: '4.16.1'
 
 development:
   <<: *shared

db/migrate/20260721100000_create_captain_message_sources.rb

@@ -0,0 +1,17 @@
+class CreateCaptainMessageSources < ActiveRecord::Migration[7.1]
+  def change
+    create_table :captain_message_sources do |t|
+      t.references :account, null: false, index: true
+      t.references :assistant, null: false, index: true
+      t.references :conversation, null: false, index: true
+      t.references :message, null: false, index: true
+      t.references :document, null: false, index: true
+      t.bigint :assistant_response_id, null: false
+
+      t.timestamps
+    end
+
+    add_index :captain_message_sources, [:message_id, :assistant_response_id],
+              unique: true, name: 'idx_captain_message_sources_on_message_and_response'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2026_07_13_184351) do
+ActiveRecord::Schema[7.1].define(version: 2026_07_21_100000) do
   # These extensions should be enabled to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"
@@ -477,6 +477,23 @@ ActiveRecord::Schema[7.1].define(version: 2026_07_13_184351) do
     t.index ["user_id"], name: "index_captain_message_reports_on_user_id"
   end
 
+  create_table "captain_message_sources", force: :cascade do |t|
+    t.bigint "account_id", null: false
+    t.bigint "assistant_id", null: false
+    t.bigint "conversation_id", null: false
+    t.bigint "message_id", null: false
+    t.bigint "document_id", null: false
+    t.bigint "assistant_response_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_captain_message_sources_on_account_id"
+    t.index ["assistant_id"], name: "index_captain_message_sources_on_assistant_id"
+    t.index ["conversation_id"], name: "index_captain_message_sources_on_conversation_id"
+    t.index ["document_id"], name: "index_captain_message_sources_on_document_id"
+    t.index ["message_id", "assistant_response_id"], name: "idx_captain_message_sources_on_message_and_response", unique: true
+    t.index ["message_id"], name: "index_captain_message_sources_on_message_id"
+  end
+
   create_table "captain_scenarios", force: :cascade do |t|
     t.string "title"
     t.text "description"

enterprise/app/controllers/api/v1/accounts/captain/documents_controller.rb

@@ -11,7 +11,7 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
     @documents = filtered_documents
     @documents_count = @documents.count
     @sync_interval_hours = current_sync_interval&.in_hours&.to_i
-    @documents = with_responses_count(@documents).page(@current_page).per(RESULTS_PER_PAGE)
+    @documents = with_document_usage(@documents).page(@current_page).per(RESULTS_PER_PAGE)
   end
 
   def show; end
@@ -61,14 +61,23 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
     apply_sort(documents, permitted_params[:sort])
   end
 
-  def with_responses_count(scope)
-    scope.left_joins(:responses)
-         .select('captain_documents.*, COUNT(captain_assistant_responses.id) AS responses_count')
-         .group('captain_documents.id')
+  def with_document_usage(scope)
+    response_counts = Captain::AssistantResponse.where(documentable_type: 'Captain::Document')
+                                                .group(:documentable_id)
+                                                .select('documentable_id AS document_id, COUNT(*) AS responses_count')
+    source_counts = Captain::MessageSource.group(:document_id).select(
+      'document_id, COUNT(DISTINCT message_id) AS used_in_answers_count, COUNT(DISTINCT conversation_id) AS used_in_conversations_count'
+    )
+
+    scope.joins("LEFT JOIN (#{response_counts.to_sql}) response_counts ON response_counts.document_id = captain_documents.id")
+         .joins("LEFT JOIN (#{source_counts.to_sql}) source_counts ON source_counts.document_id = captain_documents.id")
+         .select('captain_documents.*, COALESCE(response_counts.responses_count, 0) AS responses_count, ' \
+                 'COALESCE(source_counts.used_in_answers_count, 0) AS used_in_answers_count, ' \
+                 'COALESCE(source_counts.used_in_conversations_count, 0) AS used_in_conversations_count')
   end
 
   def set_document
-    @document = @documents.find(permitted_params[:id])
+    @document = with_document_usage(@documents).find(permitted_params[:id])
   end
 
   def set_assistant

enterprise/app/jobs/captain/conversation/message_builder.rb

@@ -1,4 +1,8 @@
 module Captain::Conversation::MessageBuilder
+  SOURCE_MARKER_PATTERN = /\[\[source:(\d+)\]\]/
+  MALFORMED_SOURCE_MARKER_PATTERN = /\[\[source:[^\]\n]*\]\]?/
+  CUSTOMER_CITATION_PATTERN = /\[\[(\d+)\]\([^\)\n]*\)\]/
+
   private
 
   def collect_previous_messages
@@ -28,8 +32,67 @@ module Captain::Conversation::MessageBuilder
   end
 
   def create_messages
-    validate_message_content!(@response['response'])
-    create_outgoing_message(@response['response'], agent_name: @response['agent_name'])
+    message_content = resolve_citations(@response['response'])
+    validate_message_content!(message_content)
+    create_outgoing_message(message_content, agent_name: @response['agent_name'])
+  end
+
+  def resolve_citations(content)
+    return content if content.blank?
+
+    citation_urls = trusted_citation_urls
+    content = content.gsub(CUSTOMER_CITATION_PATTERN) do
+      citation_for(Regexp.last_match(1), citation_urls)
+    end
+    content = content.gsub(SOURCE_MARKER_PATTERN) do
+      citation_for(Regexp.last_match(1), citation_urls)
+    end
+    content.gsub(MALFORMED_SOURCE_MARKER_PATTERN, '')
+  end
+
+  def citation_for(source_number, citation_urls)
+    url = citation_urls[source_number.to_s]
+    return '' if url.blank?
+
+    "[[#{source_number}](#{url})]"
+  end
+
+  def trusted_citation_urls
+    return {} unless @assistant.config['feature_citation']
+
+    source_map = citation_source_map
+    return {} if source_map.empty?
+
+    document_links = Captain::Document.where(
+      id: source_map.values,
+      account_id: account.id,
+      assistant_id: @assistant.id
+    ).pluck(:id, :external_link).to_h
+
+    source_map.each_with_object({}) do |(source_number, document_id), urls|
+      external_link = document_links[document_id]
+      urls[source_number] = external_link if customer_safe_document_link?(external_link)
+    end
+  end
+
+  def citation_source_map
+    raw_citation_source_map.filter_map do |source_number, document_id|
+      [source_number.to_s, document_id.to_i] if valid_citation_source?(source_number, document_id)
+    end.to_h
+  end
+
+  def raw_citation_source_map
+    context = @run_result&.context || {}
+    state = context[:state] || context['state'] || {}
+    state[:captain_v2_citation_source_map] || state['captain_v2_citation_source_map'] || {}
+  end
+
+  def valid_citation_source?(source_number, document_id)
+    source_number.to_s.match?(/\A[1-9]\d*\z/) && document_id.to_s.match?(/\A\d+\z/)
+  end
+
+  def customer_safe_document_link?(external_link)
+    external_link.present? && !external_link.start_with?('PDF:')
   end
 
   def validate_message_content!(content)

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -7,19 +7,16 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
   retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds
 
-  def perform(conversation, assistant, trigger_message_id = nil)
+  def perform(conversation, assistant)
     @conversation = conversation
     @inbox = conversation.inbox
     @assistant = assistant
-    @trigger_message_id = trigger_message_id if captain_v2_enabled?
 
     return unless conversation_pending?
 
     Current.executed_by = @assistant
 
     if captain_v2_enabled?
-      return unless trigger_message_current?
-
       generate_response_with_v2
     else
       generate_and_process_response
@@ -48,18 +45,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   end
 
   def generate_response_with_v2
-    runner_service = Captain::Assistant::AgentRunnerService.new(
-      assistant: @assistant,
-      conversation: @conversation,
-      trigger_message_id: @trigger_message_id
-    )
+    runner_service = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation)
     message_history = Captain::Conversation::MessageHistoryBuilderService.new(conversation: @conversation).perform
     @response = runner_service.generate_response(message_history: message_history)
     @run_result = runner_service.last_run_result
 
-    return if runner_service.response_discarded?
-    return unless trigger_message_current?
-
     process_response
   end
 
@@ -78,7 +68,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
         # left is the customer-facing follow-up message.
         process_v2_handoff
       end
-      capture_assistant_session(result_message: @handoff_message, credits_consumed: 0.0)
+      capture_assistant_session(result_message: @handoff_message, credits_consumed: 0.0, capture_message_sources: false)
     elsif v1_handoff_requested?
       # V1 only signals via the response string — no state has been touched yet. If
       # the conversation isn't pending anymore, a human took over mid-run; bail out
@@ -87,24 +77,16 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
       process_v1_handoff
     elsif conversation_pending?
-      process_standard_response
+      message = nil
+      ActiveRecord::Base.transaction do
+        message = create_messages
+        Rails.logger.info("[CAPTAIN][ResponseBuilderJob] Incrementing response usage for #{account.id}")
+        account.increment_response_usage
+      end
+      capture_assistant_session(result_message: message, credits_consumed: 1.0, capture_message_sources: captain_v2_enabled?)
     end
   end
 
-  def process_standard_response
-    message = nil
-    ActiveRecord::Base.transaction do
-      next unless trigger_message_current?
-
-      message = create_messages
-      Rails.logger.info("[CAPTAIN][ResponseBuilderJob] Incrementing response usage for #{account.id}")
-      account.increment_response_usage
-    end
-    return unless message
-
-    capture_assistant_session(result_message: message, credits_consumed: 1.0)
-  end
-
   def v1_handoff_requested?
     legacy_v1_handoff_token? || classifier_v1_handoff_requested?
   end
@@ -135,8 +117,6 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   end
 
   def process_v2_handoff
-    return unless trigger_message_current?
-
     # HandoffTool already ran bot_handoff! + OOO inside the agent loop. Preserve
     # waiting_since so this message doesn't clear the timestamp it left in place.
     I18n.with_locale(@assistant.account.locale) do
@@ -162,9 +142,10 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   # Capture runs outside the delivery transaction and never raises (the service
   # swallows its own failures): a session-logging bug must never roll back the
   # customer reply or trigger the top-level handle_error handoff on top of it.
-  def capture_assistant_session(result_message:, credits_consumed:)
+  def capture_assistant_session(result_message:, credits_consumed:, capture_message_sources:)
     Captain::Assistant::SessionCaptureService.new(assistant: @assistant, conversation: @conversation, run_result: @run_result,
-                                                  result_message: result_message, credits_consumed: credits_consumed).capture
+                                                  result_message: result_message, credits_consumed: credits_consumed,
+                                                  capture_message_sources: capture_message_sources).capture
   end
 
   def handle_error(error)
@@ -172,7 +153,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
     @response ||= {}
     @response['action_source'] ||= 'error'
     @response['action_reason'] ||= error_action_reason(error)
-    process_v1_handoff if conversation_pending? && (!captain_v2_enabled? || trigger_message_current?)
+    process_v1_handoff if conversation_pending?
     true
   end
 
@@ -201,14 +182,4 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
     status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) }
     status == 'pending' || status == Conversation.statuses[:pending]
   end
-
-  def trigger_message_current?
-    return true if @trigger_message_id.blank?
-
-    latest_incoming_message_id = Conversation.uncached do
-      @conversation.messages.incoming.maximum(:id)
-    end
-
-    latest_incoming_message_id == @trigger_message_id
-  end
 end

enterprise/app/models/captain/assistant.rb

@@ -108,6 +108,7 @@ class Captain::Assistant < ApplicationRecord
       name: name,
       description: description,
       product_name: config['product_name'] || 'this product',
+      citation_enabled: config['feature_citation'],
       scenarios: scenarios.enabled.map do |scenario|
         {
           title: scenario.title,

enterprise/app/models/captain/document.rb

@@ -34,6 +34,7 @@ class Captain::Document < ApplicationRecord
 
   belongs_to :assistant, class_name: 'Captain::Assistant'
   has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy, as: :documentable
+  has_many :message_sources, class_name: 'Captain::MessageSource', dependent: :destroy_async
   belongs_to :account
   has_one_attached :pdf_file
   store_accessor :metadata, :content_fingerprint, :last_sync_error_code, :sync_step, :openai_file_id

enterprise/app/models/captain/message_source.rb

@@ -0,0 +1,33 @@
+# == Schema Information
+#
+# Table name: captain_message_sources
+#
+#  id                    :bigint           not null, primary key
+#  created_at            :datetime         not null
+#  updated_at            :datetime         not null
+#  account_id            :bigint           not null
+#  assistant_id          :bigint           not null
+#  assistant_response_id :bigint           not null
+#  conversation_id       :bigint           not null
+#  document_id           :bigint           not null
+#  message_id            :bigint           not null
+#
+# Indexes
+#
+#  idx_captain_message_sources_on_message_and_response  (message_id,assistant_response_id) UNIQUE
+#  index_captain_message_sources_on_account_id          (account_id)
+#  index_captain_message_sources_on_assistant_id        (assistant_id)
+#  index_captain_message_sources_on_conversation_id     (conversation_id)
+#  index_captain_message_sources_on_document_id         (document_id)
+#  index_captain_message_sources_on_message_id          (message_id)
+#
+class Captain::MessageSource < ApplicationRecord
+  self.table_name = 'captain_message_sources'
+
+  belongs_to :account
+  belongs_to :assistant, class_name: 'Captain::Assistant'
+  belongs_to :conversation, class_name: '::Conversation'
+  belongs_to :message
+  belongs_to :document, class_name: 'Captain::Document'
+  belongs_to :assistant_response, class_name: 'Captain::AssistantResponse', optional: true
+end

enterprise/app/models/enterprise/concerns/account.rb

@@ -16,6 +16,7 @@ module Enterprise::Concerns::Account
     has_many :captain_documents, dependent: :destroy_async, class_name: 'Captain::Document'
     has_many :captain_custom_tools, dependent: :destroy_async, class_name: 'Captain::CustomTool'
     has_many :captain_agent_sessions, dependent: :destroy_async, class_name: 'Captain::AgentSession'
+    has_many :captain_message_sources, dependent: :destroy_async, class_name: 'Captain::MessageSource'
 
     has_many :copilot_threads, dependent: :destroy_async
     has_many :companies, dependent: :destroy_async

enterprise/app/models/enterprise/concerns/message.rb

@@ -4,5 +4,6 @@ module Enterprise::Concerns::Message
   included do
     has_one :call, dependent: :nullify
     has_many :message_reports, class_name: 'Captain::MessageReport', dependent: :destroy_async
+    has_many :captain_message_sources, class_name: 'Captain::MessageSource', dependent: :destroy_async
   end
 end

enterprise/app/services/captain/assistant/agent_runner_service.rb

@@ -9,12 +9,11 @@ class Captain::Assistant::AgentRunnerService
 
   attr_reader :last_run_result
 
-  def initialize(assistant:, conversation: nil, callbacks: {}, source: nil, trigger_message_id: nil)
+  def initialize(assistant:, conversation: nil, callbacks: {}, source: nil)
     @assistant = assistant
     @conversation = conversation
     @callbacks = callbacks
     @source = source
-    @trigger_message_id = trigger_message_id
     @handoff_tool_called = false
   end
 
@@ -32,10 +31,6 @@ class Captain::Assistant::AgentRunnerService
     error_response(e.message)
   end
 
-  def response_discarded?
-    @response_discarded == true
-  end
-
   private
 
   def build_context(message_history)
@@ -156,9 +151,10 @@ class Captain::Assistant::AgentRunnerService
       track_handoff_usage(tool_name, handoff_tool_name, context_wrapper)
     end
 
-    runner.on_run_complete do |_agent_name, _result, context_wrapper|
-      @response_discarded = trigger_message_stale?
-      write_run_metadata(context_wrapper) if ChatwootApp.otel_enabled?
+    if ChatwootApp.otel_enabled?
+      runner.on_run_complete do |_agent_name, _result, context_wrapper|
+        write_credits_used_metadata(context_wrapper)
+      end
     end
     runner
   end
@@ -173,22 +169,11 @@ class Captain::Assistant::AgentRunnerService
     @handoff_tool_called = true
   end
 
-  def write_run_metadata(context_wrapper)
+  def write_credits_used_metadata(context_wrapper)
     root_span = context_wrapper&.context&.dig(:__otel_tracing, :root_span)
     return unless root_span
 
-    root_span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'discarded'), response_discarded?.to_s)
-    root_span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'credit_used'), (!@handoff_tool_called && !response_discarded?).to_s)
-  end
-
-  def trigger_message_stale?
-    return false if @trigger_message_id.blank? || @conversation.blank?
-
-    latest_incoming_message_id = Conversation.uncached do
-      @conversation.messages.incoming.maximum(:id)
-    end
-
-    latest_incoming_message_id != @trigger_message_id
+    root_span.set_attribute(format(ATTR_LANGFUSE_METADATA, 'credit_used'), @handoff_tool_called ? 'false' : 'true')
   end
 
   def runner

enterprise/app/services/captain/assistant/instrumentation_attribute_provider.rb

@@ -13,8 +13,7 @@ class Captain::Assistant::InstrumentationAttributeProvider
 
   def generation_attributes(_context_wrapper, _chat, message)
     {
-      format(ATTR_LANGFUSE_OBSERVATION_METADATA, 'generation_stage') => generation_stage(message),
-      format(ATTR_LANGFUSE_OBSERVATION_METADATA, 'discarded') => @service.send(:trigger_message_stale?).to_s
+      format(ATTR_LANGFUSE_OBSERVATION_METADATA, 'generation_stage') => generation_stage(message)
     }
   end
 

enterprise/app/services/captain/assistant/runner_state_helper.rb

@@ -23,7 +23,6 @@ module Captain::Assistant::RunnerStateHelper
       timezone: @conversation&.inbox&.timezone.presence || 'UTC'
     }
     state[:source] = @source if @source.present?
-    state[:trigger_message_id] = @trigger_message_id if @trigger_message_id.present?
 
     build_conversation_state(state) if @conversation
     state

enterprise/app/services/captain/assistant/session_capture_service.rb

@@ -1,12 +1,13 @@
 class Captain::Assistant::SessionCaptureService
   SCENARIO_AGENT_REGEX = /\A#{Captain::Scenario::HANDOFF_KEY_PREFIX}_(\d+)_/
 
-  def initialize(assistant:, conversation:, run_result:, result_message:, credits_consumed:)
+  def initialize(assistant:, conversation:, run_result:, result_message:, **options)
     @assistant = assistant
     @conversation = conversation
     @run_result = run_result
     @result_message = result_message
-    @credits_consumed = credits_consumed
+    @credits_consumed = options.fetch(:credits_consumed)
+    @capture_message_sources = options.fetch(:capture_message_sources, false)
   end
 
   def capture
@@ -23,7 +24,7 @@ class Captain::Assistant::SessionCaptureService
   def capture!
     model = @assistant.agent_model
 
-    Captain::AgentSession.create!(
+    session = Captain::AgentSession.create!(
       assistant: @assistant,
       session_type: :assistant,
       subject: @conversation,
@@ -35,6 +36,8 @@ class Captain::Assistant::SessionCaptureService
       scenario_ids: scenario_ids,
       run_context: current_turn_history
     )
+    capture_message_sources(metadata) if @capture_message_sources
+    session
   end
 
   private
@@ -70,6 +73,42 @@ class Captain::Assistant::SessionCaptureService
     ids & @assistant.scenarios.where(id: ids).pluck(:id)
   end
 
+  def capture_message_sources(metadata)
+    sources = Array(metadata[:message_sources])
+    return if sources.empty?
+
+    document_ids = @assistant.documents.where(id: sources.pluck(:document_id)).pluck(:id)
+    return if document_ids.empty?
+
+    insert_message_sources(message_source_rows(sources, document_ids))
+  end
+
+  def message_source_rows(sources, document_ids)
+    timestamp = Time.current
+    sources.filter_map do |source|
+      next unless document_ids.include?(source[:document_id])
+
+      {
+        account_id: @assistant.account_id,
+        assistant_id: @assistant.id,
+        conversation_id: @conversation.id,
+        message_id: @result_message.id,
+        document_id: source[:document_id],
+        assistant_response_id: source[:assistant_response_id],
+        created_at: timestamp,
+        updated_at: timestamp
+      }
+    end
+  end
+
+  def insert_message_sources(rows)
+    return if rows.empty?
+
+    # rubocop:disable Rails/SkipsModelValidations
+    Captain::MessageSource.insert_all(rows, unique_by: :idx_captain_message_sources_on_message_and_response)
+    # rubocop:enable Rails/SkipsModelValidations
+  end
+
   # Trim to the current turn: the last user message and everything after it
   # (assistant replies, tool calls/results, handoff hops).
   def current_turn_history

enterprise/app/services/enterprise/message_templates/hook_execution_service.rb

@@ -31,7 +31,6 @@ module Enterprise::MessageTemplates::HookExecutionService
 
   def schedule_captain_response
     job_args = [conversation, conversation.inbox.captain_assistant]
-    job_args << message.id if conversation.account.feature_enabled?('captain_integration_v2')
 
     if message.attachments.blank?
       Captain::Conversation::ResponseBuilderJob.perform_later(*job_args)

enterprise/app/views/api/v1/models/captain/_document.json.jbuilder

@@ -11,6 +11,10 @@ json.file_size resource.file_size
 json.pdf_document resource.pdf_document?
 responses_count = resource.respond_to?(:responses_count) ? resource.responses_count : resource.responses.count
 json.responses_count responses_count.to_i
+used_in_answers_count = resource.respond_to?(:used_in_answers_count) ? resource.used_in_answers_count : resource.message_sources.distinct.count(:message_id)
+json.used_in_answers_count used_in_answers_count.to_i
+used_in_conversations_count = resource.respond_to?(:used_in_conversations_count) ? resource.used_in_conversations_count : resource.message_sources.distinct.count(:conversation_id)
+json.used_in_conversations_count used_in_conversations_count.to_i
 json.id resource.id
 json.name resource.name
 json.status resource.status

enterprise/lib/captain/prompts/assistant.liquid

@@ -10,6 +10,11 @@ You are {{name}}, a helpful, friendly, and knowledgeable assistant for the produ
 
 Don't digress away from your instructions, and use all the available tools at your disposal for solving customer issues. If you are to state something factual about {{product_name}}, use the `captain--tools--faq_lookup` tool to check the available information first.
 
+{% if citation_enabled -%}
+# Citations
+When you use FAQ content that includes a source marker, add that exact marker at the end of the supported sentence or paragraph. Source markers have the form `[[source:n]]`. Do not create URLs or markdown links. Reuse the same source marker when the same document supports more than one statement. Do not cite conversation context.
+{% endif -%}
+
 {% render 'current_time', current_time: current_time %}
 
 {% render 'core_rules' %}

enterprise/lib/captain/tools/faq_lookup_tool.rb

@@ -14,7 +14,7 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool
       "No relevant FAQs found for: #{query}"
     else
       log_tool_usage('found_results', { query: query, count: responses.size })
-      format_responses(responses)
+      format_responses(tool_context, responses)
     end
   end
 
@@ -26,34 +26,46 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool
     metadata = tool_context.state[:cw_metadata] ||= {}
     metadata[:faq_ids] = Array(metadata[:faq_ids]) | responses.map(&:id)
 
-    document_ids = responses.filter_map { |response| response.documentable_id if response.documentable_type == 'Captain::Document' }
+    document_ids = document_responses(responses).map(&:documentable_id)
     metadata[:document_ids] = Array(metadata[:document_ids]) | document_ids
+    metadata[:message_sources] = Array(metadata[:message_sources]) | message_sources(document_responses(responses))
+
+    document_ids.each { |document_id| citation_source_number(tool_context, document_id) }
   end
 
-  def format_responses(responses)
-    responses.map { |response| format_response(response) }.join
+  def document_responses(responses)
+    responses.select { |response| response.documentable_type == 'Captain::Document' }
   end
 
-  def format_response(response)
+  def message_sources(responses)
+    responses.map { |response| { assistant_response_id: response.id, document_id: response.documentable_id } }
+  end
+
+  def format_responses(tool_context, responses)
+    responses.map { |response| format_response(tool_context, response) }.join
+  end
+
+  def format_response(tool_context, response)
     formatted_response = "
         Question: #{response.question}
         Answer: #{response.answer}
         "
-    if should_show_source?(response)
+    if response.documentable_type == 'Captain::Document'
       formatted_response += "
-          Source: #{response.documentable.external_link}
+          Source marker: [[source:#{citation_source_number(tool_context, response.documentable_id)}]]
           "
     end
 
     formatted_response
   end
 
-  def should_show_source?(response)
-    return false if response.documentable.blank?
-    return false unless response.documentable.try(:external_link)
+  def citation_source_number(tool_context, document_id)
+    source_map = tool_context.state[:captain_v2_citation_source_map] ||= {}
+    existing_number = source_map.find { |_number, id| id.to_i == document_id }&.first
+    return existing_number if existing_number.present?
 
-    # Don't show source if it's a PDF placeholder
-    external_link = response.documentable.external_link
-    !external_link.start_with?('PDF:')
+    source_number = (source_map.keys.map(&:to_i).max || 0) + 1
+    source_map[source_number.to_s] = document_id
+    source_number.to_s
   end
 end

enterprise/lib/captain/tools/handoff_tool.rb

@@ -5,7 +5,6 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool
   def perform(tool_context, reason: nil)
     conversation = find_conversation(tool_context.state)
     return 'Conversation not found' unless conversation
-    return 'Handoff skipped because a newer customer message arrived' unless trigger_message_current?(tool_context.state, conversation)
 
     # Log the handoff with reason
     log_tool_usage('tool_handoff', {
@@ -24,17 +23,6 @@ class Captain::Tools::HandoffTool < Captain::Tools::BasePublicTool
 
   private
 
-  def trigger_message_current?(state, conversation)
-    trigger_message_id = state&.dig(:trigger_message_id)
-    return true if trigger_message_id.blank?
-
-    latest_incoming_message_id = Conversation.uncached do
-      conversation.messages.incoming.maximum(:id)
-    end
-
-    latest_incoming_message_id == trigger_message_id
-  end
-
   def trigger_handoff(tool_context, conversation, reason)
     # post the reason as a private note
     note = conversation.messages.create!(

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatwoot/chatwoot",
-  "version": "4.16.0",
+  "version": "4.16.1",
   "license": "MIT",
   "scripts": {
     "eslint": "eslint app/**/*.{js,vue}",

spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb

@@ -13,7 +13,6 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
     let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
     let(:mock_false_promise_service) { instance_double(Captain::Llm::AssistantFalsePromiseService) }
     let(:assistant_model) { Llm::Models.default_model_for('assistant') }
-    let(:trigger_message) { conversation.messages.find_by!(content: 'Hello') }
 
     before do
       create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -24,7 +23,6 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
       allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
       allow(mock_agent_runner_service).to receive(:last_run_result).and_return(nil)
-      allow(mock_agent_runner_service).to receive(:response_discarded?).and_return(false)
       allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
       allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' })
       allow(Captain::Llm::AssistantFalsePromiseService).to receive(:new).and_return(mock_false_promise_service)
@@ -360,28 +358,14 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       it 'uses Captain::Assistant::AgentRunnerService' do
         expect(Captain::Assistant::AgentRunnerService).to receive(:new).with(
           assistant: assistant,
-          conversation: conversation,
-          trigger_message_id: trigger_message.id
+          conversation: conversation
         )
         expect(Captain::Llm::AssistantChatService).not_to receive(:new)
 
-        described_class.perform_now(conversation, assistant, trigger_message.id)
+        described_class.perform_now(conversation, assistant)
         expect(conversation.messages.last.content).to eq('Hey, welcome to Captain V2')
       end
 
-      it 'discards a response when a newer message arrives during generation' do
-        allow(mock_agent_runner_service).to receive(:generate_response) do
-          create(:message, conversation: conversation, content: 'New context', message_type: :incoming)
-          { 'response' => 'Stale response', 'handoff_tool_called' => false }
-        end
-        allow(mock_agent_runner_service).to receive(:response_discarded?).and_return(true)
-
-        described_class.perform_now(conversation, assistant, trigger_message.id)
-
-        expect(conversation.messages.outgoing.count).to eq(0)
-        expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
-      end
-
       it 'passes message history with resolution markers to agent runner service' do
         same_second = Time.current.change(usec: 0)
         conversation.messages.find_by!(content: 'Hello').update!(created_at: same_second, updated_at: same_second)

spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb

@@ -66,12 +66,13 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
         expect(result).to include('Answer: Go to settings and update email')
       end
 
-      it 'includes source link when document has external_link' do
+      it 'includes a source marker instead of the document link' do
         document.update!(external_link: 'https://help.example.com/password')
 
         result = tool.perform(tool_context, query: 'password')
 
-        expect(result).to include('Source: https://help.example.com/password')
+        expect(result).to include('Source marker: [[source:1]]')
+        expect(result).not_to include('https://help.example.com/password')
       end
 
       it 'logs tool usage for search' do

spec/enterprise/lib/captain/tools/handoff_tool_spec.rb

@@ -28,37 +28,6 @@ RSpec.describe Captain::Tools::HandoffTool, type: :model do
 
   describe '#perform' do
     context 'when conversation exists' do
-      context 'with a trigger message id' do
-        let(:trigger_message) do
-          create(:message, conversation: conversation, account: account, inbox: inbox, message_type: :incoming, created_at: same_second)
-        end
-        let(:same_second) { Time.current.change(usec: 0) }
-        let(:tool_context) do
-          Struct.new(:state).new({ conversation: { id: conversation.id }, trigger_message_id: trigger_message.id })
-        end
-
-        it 'hands off when the trigger message is still the latest message' do
-          trigger_message
-
-          expect do
-            result = tool.perform(tool_context, reason: 'Customer needs specialized support')
-            expect(result).to include('Conversation handed off')
-          end.to change(Message, :count).by(1)
-        end
-
-        it 'skips the handoff when a newer message has arrived' do
-          trigger_message
-          conversation.update!(status: :pending)
-          create(:message, conversation: conversation, account: account, inbox: inbox, message_type: :incoming, created_at: same_second)
-
-          expect do
-            result = tool.perform(tool_context, reason: 'Customer needs specialized support')
-            expect(result).to eq('Handoff skipped because a newer customer message arrived')
-          end.not_to change(Message, :count)
-          expect(conversation.reload.status).to eq('pending')
-        end
-      end
-
       context 'with reason provided' do
         it 'creates a private note with reason and hands off conversation' do
           reason = 'Customer needs specialized support'

spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb

@@ -52,12 +52,6 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
 
       expect(service.instance_variable_get(:@callbacks)).to eq(callbacks)
     end
-
-    it 'accepts the trigger message id' do
-      service = described_class.new(assistant: assistant, conversation: conversation, trigger_message_id: 123)
-
-      expect(service.instance_variable_get(:@trigger_message_id)).to eq(123)
-    end
   end
 
   describe '#generate_response' do
@@ -105,18 +99,6 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
       service.generate_response(message_history: message_history)
     end
 
-    it 'adds the trigger message id to the runner state' do
-      service = described_class.new(assistant: assistant, conversation: conversation, trigger_message_id: 123)
-
-      expect(mock_runner).to receive(:run).with(
-        'I need help with my account',
-        context: hash_including(state: hash_including(trigger_message_id: 123)),
-        max_turns: 10
-      )
-
-      service.generate_response(message_history: message_history)
-    end
-
     context 'when the latest user message is multimodal' do
       let(:multimodal_message_history) do
         [
@@ -464,7 +446,6 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
       attributes = provider.generation_attributes(nil, nil, message)
 
       expect(attributes['langfuse.observation.metadata.generation_stage']).to eq('final_response')
-      expect(attributes['langfuse.observation.metadata.discarded']).to eq('false')
     end
 
     it 'marks tool call generations separately from final responses' do
@@ -475,18 +456,6 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
 
       expect(attributes['langfuse.observation.metadata.generation_stage']).to eq('tool_call')
     end
-
-    it 'marks a generation as discarded when a newer message has arrived' do
-      trigger_message = create(:message, conversation: conversation, message_type: :incoming)
-      runner_service = described_class.new(assistant: assistant, conversation: conversation, trigger_message_id: trigger_message.id)
-      attribute_provider = Captain::Assistant::InstrumentationAttributeProvider.new(runner_service)
-      message = instance_double(RubyLLM::Message, tool_calls: {})
-      create(:message, conversation: conversation, message_type: :incoming)
-
-      attributes = attribute_provider.generation_attributes(nil, nil, message)
-
-      expect(attributes['langfuse.observation.metadata.discarded']).to eq('true')
-    end
   end
 
   describe '#build_state' do
@@ -605,7 +574,6 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
 
       tool_complete_callback.call(Captain::Tools::HandoffTool.new(assistant).name, 'ok', context_wrapper)
 
-      expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.discarded', 'false')
       expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credit_used', 'false')
       run_complete_callback.call('assistant', nil, context_wrapper)
     end
@@ -620,7 +588,6 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
         tool_complete_callback = block
         runner
       end
-      allow(runner).to receive(:on_run_complete).and_return(runner)
 
       service.send(:add_usage_metadata_callback, runner)
 
@@ -632,24 +599,15 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
       expect(context_wrapper.context[:captain_v2_handoff_tool_called]).to be true
     end
 
-    it 'tracks discarded responses when OTEL is disabled' do
-      trigger_message = create(:message, conversation: conversation, message_type: :incoming)
-      service = described_class.new(assistant: assistant, conversation: conversation, trigger_message_id: trigger_message.id)
+    it 'does not register OTEL run callback when OTEL is disabled' do
+      service = described_class.new(assistant: assistant, conversation: conversation)
       runner = instance_double(Agents::AgentRunner)
-      run_complete_callback = nil
 
       allow(ChatwootApp).to receive(:otel_enabled?).and_return(false)
       allow(runner).to receive(:on_tool_complete).and_return(runner)
-      allow(runner).to receive(:on_run_complete) do |&block|
-        run_complete_callback = block
-        runner
-      end
+      expect(runner).not_to receive(:on_run_complete)
 
       service.send(:add_usage_metadata_callback, runner)
-      create(:message, conversation: conversation, message_type: :incoming)
-      run_complete_callback.call('assistant', nil, Struct.new(:context).new({}))
-
-      expect(service.response_discarded?).to be true
     end
 
     it 'sets credit_used=true when handoff tool is not used' do
@@ -671,38 +629,9 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
 
       service.send(:add_usage_metadata_callback, runner)
 
-      expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.discarded', 'false')
       expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credit_used', 'true')
       run_complete_callback.call('assistant', nil, context_wrapper)
     end
-
-    it 'marks the trace discarded and does not use credit when a newer message arrived' do
-      trigger_message = create(:message, conversation: conversation, message_type: :incoming)
-      service = described_class.new(assistant: assistant, conversation: conversation, trigger_message_id: trigger_message.id)
-      runner = instance_double(Agents::AgentRunner)
-      run_complete_callback = nil
-      span_class = Class.new do
-        def set_attribute(*); end
-      end
-      root_span = instance_double(span_class)
-      context_wrapper = Struct.new(:context).new({ __otel_tracing: { root_span: root_span } })
-
-      allow(ChatwootApp).to receive(:otel_enabled?).and_return(true)
-      allow(runner).to receive(:on_tool_complete).and_return(runner)
-      allow(runner).to receive(:on_run_complete) do |&block|
-        run_complete_callback = block
-        runner
-      end
-
-      service.send(:add_usage_metadata_callback, runner)
-      create(:message, conversation: conversation, message_type: :incoming)
-
-      expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.discarded', 'true')
-      expect(root_span).to receive(:set_attribute).with('langfuse.trace.metadata.credit_used', 'false')
-      run_complete_callback.call('assistant', nil, context_wrapper)
-
-      expect(service.response_discarded?).to be true
-    end
   end
 
   describe 'constants' do

spec/enterprise/services/enterprise/message_templates/hook_execution_service_spec.rb

@@ -22,13 +22,9 @@ RSpec.describe MessageTemplates::HookExecutionService do
       end
 
       it 'schedules captain response job for incoming messages on pending conversations' do
-        allow(account).to receive(:feature_enabled?).and_call_original
-        allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(true)
-        allow(Captain::Conversation::ResponseBuilderJob).to receive(:perform_later)
+        expect(Captain::Conversation::ResponseBuilderJob).to receive(:perform_later).with(conversation, assistant)
 
-        message = create(:message, conversation: conversation, message_type: :incoming, account: account)
-
-        expect(Captain::Conversation::ResponseBuilderJob).to have_received(:perform_later).with(conversation, assistant, message.id)
+        create(:message, conversation: conversation, message_type: :incoming, account: account)
       end
     end