draft commit

enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb

@@ -58,6 +58,7 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base
                                                   config: [
                                                     :product_name, :feature_faq, :feature_memory, :feature_citation,
                                                     :feature_contact_attributes,
+                                                    :documentation_sufficiency_gate_enabled,
                                                     :welcome_message, :handoff_message, :resolution_message,
                                                     :instructions, :temperature
                                                   ])

enterprise/app/helpers/captain/chat_generation_recorder.rb

@@ -7,6 +7,27 @@ module Captain::ChatGenerationRecorder
   def record_llm_generation(chat, message)
     return unless valid_llm_message?(message)
 
+    if defer_llm_generation?(message)
+      deferred_llm_generations << [chat, message]
+      return
+    end
+
+    record_llm_generation_span(chat, message)
+  end
+
+  def flush_deferred_llm_generations
+    deferred_llm_generations.each do |chat, message|
+      record_llm_generation_span(chat, message)
+    end
+  ensure
+    deferred_llm_generations.clear
+  end
+
+  def discard_deferred_llm_generations
+    deferred_llm_generations.clear
+  end
+
+  def record_llm_generation_span(chat, message)
     # Create a generation span with model and token info for Langfuse cost calculation.
     # Note: span duration will be near-zero since we create and end it immediately, but token counts are what Langfuse uses for cost calculation.
     tracer.in_span("llm.captain.#{feature_name}.generation") do |span|
@@ -17,6 +38,14 @@ module Captain::ChatGenerationRecorder
     Rails.logger.warn "Failed to record LLM generation: #{e.message}"
   end
 
+  def defer_llm_generation?(message)
+    !message_has_tool_calls?(message)
+  end
+
+  def deferred_llm_generations
+    @deferred_llm_generations ||= []
+  end
+
   # Skip non-LLM messages (e.g., tool results that RubyLLM processes internally).
   # Check for assistant role rather than token presence - some providers/streaming modes
   # may not return token counts, but we still want to capture the generation for evals.

enterprise/app/helpers/captain/chat_helper.rb

@@ -13,7 +13,10 @@ module Captain::ChatHelper
       text, attachments = Captain::OpenAiMessageBuilderService.extract_text_and_attachments(last_content)
 
       response = attachments.any? ? chat.ask(text, with: attachments) : chat.ask(text)
-      build_response(response)
+      built_response = build_response(response)
+      after_chat_response(built_response)
+      flush_deferred_llm_generations
+      built_response
     end
   rescue StandardError => e
     Rails.logger.error "#{self.class.name} Assistant: #{@assistant.id}, Error in chat completion: #{e}"
@@ -57,15 +60,19 @@ module Captain::ChatHelper
 
   def handle_tool_call(tool_call)
     persist_thinking_message(tool_call)
-    start_tool_span(tool_call)
+    start_tool_span(tool_call) unless internally_instrumented_tool?(tool_call)
     (@pending_tool_calls ||= []).push(tool_call)
   end
 
   def handle_tool_result(result)
-    end_tool_span(result)
+    end_tool_span(result) unless internally_instrumented_tool?(@pending_tool_calls&.last)
     persist_tool_completion
   end
 
+  def internally_instrumented_tool?(tool_call)
+    tool_call&.name.to_s == 'search_documentation'
+  end
+
   def add_messages_to_chat(chat)
     conversation_messages[0...-1].each do |msg|
       text, attachments = Captain::OpenAiMessageBuilderService.extract_text_and_attachments(msg[:content])
@@ -91,21 +98,13 @@ module Captain::ChatHelper
     }
   end
 
-  def conversation_messages
-    @messages.reject { |m| m[:role] == 'system' || m[:role] == :system }
-  end
+  def conversation_messages = @messages.reject { |m| m[:role] == 'system' || m[:role] == :system }
 
-  def temperature
-    @assistant&.config&.[]('temperature').to_f || 1
-  end
+  def temperature = @assistant&.config&.[]('temperature').to_f || 1
 
-  def resolved_account_id
-    @account&.id || @assistant&.account_id
-  end
+  def resolved_account_id = @account&.id || @assistant&.account_id
 
-  def resolved_channel_type
-    @conversation&.inbox&.channel_type
-  end
+  def resolved_channel_type = @conversation&.inbox&.channel_type
 
   # Ensures all LLM calls and tool executions within an agentic loop
   # are grouped under a single trace/session in Langfuse.
@@ -123,6 +122,8 @@ module Captain::ChatHelper
     @agent_session_active = false unless already_active
   end
 
+  def after_chat_response(_response) = nil
+
   # Must be implemented by including class to identify the feature for instrumentation.
   # Used for Langfuse tagging and span naming.
   def feature_name

enterprise/app/jobs/captain/conversation/documentation_support_gate.rb

@@ -0,0 +1,86 @@
+module Captain::Conversation::DocumentationSupportGate
+  private
+
+  def check_documentation_support(message_history, chat_service: nil)
+    return unless documentation_support_gate_enabled?
+    return unless customer_reply?
+
+    searches = documentation_searches(message_history)
+    return if documentation_sufficiency_checked_in_tool?(searches)
+
+    review = documentation_support_review(message_history, searches)
+    apply_documentation_support_decision(review, message_history, chat_service)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][ResponseBuilderJob] Documentation support check failed for account=#{account.id} " \
+      "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+  end
+
+  def documentation_support_gate_enabled?
+    ActiveModel::Type::Boolean.new.cast(@assistant.config['documentation_sufficiency_gate_enabled'])
+  end
+
+  def customer_reply?
+    @response.present? &&
+      @response['response'].present? &&
+      @response['response'] != 'conversation_handoff' &&
+      !@response['handoff_tool_called']
+  end
+
+  def documentation_support_review(message_history, searches)
+    Captain::Llm::DocumentationSufficiencyService.new(
+      assistant: @assistant,
+      conversation: @conversation
+    ).evaluate(
+      message_history: message_history,
+      documentation_searches: searches
+    )
+  end
+
+  def documentation_sufficiency_checked_in_tool?(searches)
+    searches.any? { |search| (search[:documentation_sufficiency] || search['documentation_sufficiency']).present? }
+  end
+
+  def documentation_searches(message_history)
+    searches = @response['documentation_searches'].to_a
+    return searches if searches.present?
+
+    [missing_documentation_search(last_user_message(message_history))]
+  end
+
+  def missing_documentation_search(query)
+    {
+      query: query,
+      queries: [query],
+      matches: []
+    }
+  end
+
+  def last_user_message(message_history)
+    message = message_history.reverse.find { |item| (item[:role] || item['role']).to_s == 'user' }
+    message && (message[:content] || message['content']).to_s
+  end
+
+  def apply_documentation_support_decision(review, message_history, chat_service)
+    return unless review['decision'] == 'insufficient'
+
+    if chat_service
+      @response.replace(chat_service.generate_documentation_gap_response(message_history: message_history))
+    else
+      @response['response'] = default_documentation_fallback
+    end
+
+    @response.merge!(
+      'action' => 'continue',
+      'action_reason' => 'missing_docs_bounded_answer',
+      'action_source' => 'documentation_support',
+      'documentation_sufficiency_model' => review['model']
+    )
+  end
+
+  def default_documentation_fallback
+    'I do not have enough information to answer that. Would you like me to connect you with support?'
+  end
+end

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -1,5 +1,6 @@
 class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   include Captain::Conversation::V1ActionClassifier
+  include Captain::Conversation::DocumentationSupportGate
 
   MAX_MESSAGE_LENGTH = 10_000
   retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
@@ -34,17 +35,24 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
   def generate_and_process_response
     message_history = collect_previous_messages
-    @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
+    chat_service = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation)
+    @response = chat_service.generate_response(
       message_history: message_history
-    )
-    classify_v1_response_action(message_history) if conversation_pending?
+    ) do |response|
+      response['documentation_searches'] = chat_service.documentation_searches
+      @response = response
+      check_documentation_support(message_history, chat_service: chat_service) if conversation_pending?
+      classify_v1_response_action(message_history) if conversation_pending?
+    end
     process_response
   end
 
   def generate_response_with_v2
+    message_history = collect_previous_messages
     @response = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation).generate_response(
-      message_history: collect_previous_messages
+      message_history: message_history
     )
+    check_documentation_support(message_history) if conversation_pending?
     process_response
   end
 

enterprise/app/models/captain/assistant_response.rb

@@ -25,6 +25,22 @@
 #
 class Captain::AssistantResponse < ApplicationRecord
   self.table_name = 'captain_assistant_responses'
+  SEARCH_LIMIT = 5
+  SearchMatch = Struct.new(
+    :response,
+    :semantic_distance,
+    keyword_init: true
+  ) do
+    def to_h
+      {
+        response_id: response.id,
+        question: response.question,
+        answer: response.answer,
+        source: response.documentable&.try(:external_link),
+        semantic_distance: semantic_distance
+      }
+    end
+  end
 
   belongs_to :assistant, class_name: 'Captain::Assistant'
   belongs_to :account
@@ -47,8 +63,21 @@ class Captain::AssistantResponse < ApplicationRecord
   enum status: { pending: 0, approved: 1 }
 
   def self.search(query, account_id: nil)
+    search_with_metadata(query, account_id: account_id).map(&:response)
+  end
+
+  def self.search_with_metadata(query, account_id: nil, limit: SEARCH_LIMIT)
+    semantic_search_matches(query, account_id: account_id, limit: limit)
+  end
+
+  def self.semantic_search_matches(query, account_id:, limit:)
     embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(query)
-    nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
+    nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(limit).map do |response|
+      SearchMatch.new(
+        response: response,
+        semantic_distance: response.neighbor_distance&.to_f
+      )
+    end
   end
 
   private

enterprise/app/services/captain/assistant/agent_runner_service.rb

@@ -6,10 +6,7 @@ class Captain::Assistant::AgentRunnerService
   include Captain::Assistant::RunnerCallbacksHelper
   include Captain::Assistant::TracePayloadHelper
 
-  CONVERSATION_STATE_ATTRIBUTES = %i[
-    id display_id inbox_id contact_id status priority
-    label_list custom_attributes additional_attributes
-  ].freeze
+  CONVERSATION_STATE_ATTRIBUTES = %i[id display_id inbox_id contact_id status priority label_list custom_attributes additional_attributes].freeze
 
   CONTACT_STATE_ATTRIBUTES = %i[
     id name email phone_number identifier contact_type
@@ -19,6 +16,7 @@ class Captain::Assistant::AgentRunnerService
   CONTACT_INBOX_STATE_ATTRIBUTES = %i[id hmac_verified].freeze
 
   CAMPAIGN_STATE_ATTRIBUTES = %i[id title message campaign_type description].freeze
+
   def initialize(assistant:, conversation: nil, callbacks: {}, source: nil)
     @assistant = assistant
     @conversation = conversation
@@ -100,6 +98,7 @@ class Captain::Assistant::AgentRunnerService
     response = output.is_a?(Hash) ? output.with_indifferent_access : { 'response' => output.to_s, 'reasoning' => 'Processed by agent' }
     response['agent_name'] = result.context&.dig(:current_agent)
     response['handoff_tool_called'] = result.context&.dig(:captain_v2_handoff_tool_called) || false
+    response['documentation_searches'] = result.context&.dig(:state, :documentation_searches).to_a
     response
   end
 
@@ -115,7 +114,8 @@ class Captain::Assistant::AgentRunnerService
     state = {
       account_id: @assistant.account_id,
       assistant_id: @assistant.id,
-      assistant_config: @assistant.config
+      assistant_config: @assistant.config,
+      documentation_searches: []
     }
     state[:source] = @source if @source.present?
 

enterprise/app/services/captain/documentation_search_service.rb

@@ -0,0 +1,75 @@
+class Captain::DocumentationSearchService
+  TOP_MATCHES_TO_FORMAT = 5
+
+  class << self
+    def serialize(result)
+      result.merge(matches: result[:matches].map(&:to_h))
+    end
+
+    def metadata(result)
+      {
+        match_count: result[:matches].length,
+        top_semantic_distance: result[:matches].first&.semantic_distance
+      }.compact
+    end
+
+    def format_for_tool(result, no_results_message:, documentation_sufficiency: nil)
+      return "#{no_results_message}\n\n#{no_results_instruction}" if result[:matches].empty?
+
+      [documentation_sufficiency_section(documentation_sufficiency), formatted_matches(result)].flatten.compact.join("\n")
+    end
+
+    private
+
+    def formatted_matches(result)
+      result[:matches].first(TOP_MATCHES_TO_FORMAT).map { |match| format_match(match) }
+    end
+
+    def format_match(match)
+      response = match.response
+      lines = ['', "Question: #{response.question}", "Answer: #{response.answer}"]
+      lines << "Source: #{response.documentable.external_link}" if response.documentable.present? && response.documentable.try(:external_link)
+      "#{lines.join("\n")}\n"
+    end
+
+    def documentation_sufficiency_section(documentation_sufficiency)
+      decision = documentation_sufficiency && (documentation_sufficiency[:decision] || documentation_sufficiency['decision'])
+      return if decision.blank?
+
+      if decision == 'sufficient'
+        [
+          'Documentation support: sufficient',
+          'Instruction: Use only the retrieved documentation below to answer the user.'
+        ].join("\n")
+      else
+        [
+          'Documentation support: insufficient',
+          'Instruction: The retrieved documentation does not answer the user question.',
+          'Do not answer the factual question from these results. Ask one clarifying question if useful, or offer a handoff.'
+        ].join("\n")
+      end
+    end
+
+    def no_results_instruction
+      [
+        'Instruction: No documentation matched this query.',
+        'Do not use documentation search results to make factual claims.',
+        'Ask one useful follow-up question or offer a handoff.'
+      ].join(' ')
+    end
+  end
+
+  def initialize(scope:, account_id: nil)
+    @scope = scope
+    @account_id = account_id
+  end
+
+  def search(query)
+    matches = @scope.search_with_metadata(query, account_id: @account_id)
+    {
+      query: query,
+      queries: [query],
+      matches: matches
+    }
+  end
+end

enterprise/app/services/captain/llm/assistant_chat_service.rb

@@ -1,5 +1,6 @@
 class Captain::Llm::AssistantChatService < Llm::BaseAiService
   include Captain::ChatHelper
+  attr_reader :documentation_searches
 
   def initialize(assistant: nil, conversation: nil, source: nil)
     super()
@@ -11,6 +12,7 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
 
     @messages = [system_message]
     @response = ''
+    @documentation_searches = []
     @tools = build_tools
   end
 
@@ -21,16 +23,44 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
   #
   # NOTE: Parameters are provided as keyword arguments to improve clarity and avoid relying on
   # positional ordering.
-  def generate_response(additional_message: nil, message_history: [], role: 'user')
+  def generate_response(additional_message: nil, message_history: [], role: 'user', &after_response)
     @messages += message_history
     @messages << { role: role, content: additional_message } if additional_message.present?
+    @after_response = after_response
     request_chat_completion
+  ensure
+    @after_response = nil
+  end
+
+  def generate_documentation_gap_response(message_history:)
+    previous_messages = @messages
+    previous_tools = @tools
+    previous_after_response = @after_response
+
+    discard_deferred_llm_generations
+    @messages = [system_message, documentation_gap_instruction] + message_history
+    @tools = []
+    @after_response = nil
+
+    request_chat_completion
+  ensure
+    @messages = previous_messages
+    @tools = previous_tools
+    @after_response = previous_after_response
   end
 
   private
 
   def build_tools
-    tools = [Captain::Tools::SearchDocumentationService.new(@assistant, user: nil)]
+    tools = [
+      Captain::Tools::SearchDocumentationService.new(
+        @assistant,
+        user: nil,
+        on_search: ->(search) { @documentation_searches << search },
+        message_history: -> { conversation_messages },
+        conversation: @conversation
+      )
+    ]
     return tools unless custom_tools_enabled?
 
     tools + @assistant.account.captain_custom_tools.enabled.map do |ct|
@@ -49,6 +79,20 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
     }
   end
 
+  def documentation_gap_instruction
+    {
+      role: 'system',
+      content: <<~PROMPT
+        [Documentation Support]
+        The retrieved documentation was not sufficient to answer the user's latest question.
+        Do not answer the factual question or cite the retrieved documentation.
+        Respond briefly in the user's language.
+        Ask one clarifying question if that would help, or offer a handoff.
+        Return the normal JSON response.
+      PROMPT
+    }
+  end
+
   def custom_tools_metadata
     return [] unless custom_tools_enabled?
 
@@ -81,4 +125,8 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
   def feature_name
     'assistant'
   end
+
+  def after_chat_response(response)
+    @after_response&.call(response)
+  end
 end

enterprise/app/services/captain/llm/documentation_sufficiency_service.rb

@@ -0,0 +1,195 @@
+class Captain::Llm::DocumentationSufficiencyService < Llm::BaseAiService
+  include Integrations::LlmInstrumentation
+
+  MODEL = 'gpt-5.4-mini'.freeze
+  MAX_CONTEXT_MESSAGES = 6
+  MAX_SEARCHES = 3
+  MAX_MATCHES_PER_SEARCH = 5
+  MAX_ANSWER_CHARS = 700
+
+  def initialize(assistant:, conversation:)
+    super()
+    @assistant = assistant
+    @conversation = conversation
+    @model = MODEL
+    @temperature = 0.0
+  end
+
+  def evaluate(message_history:, documentation_searches:)
+    user_prompt = inspection_user_prompt(
+      message_history: message_history,
+      documentation_searches: documentation_searches
+    )
+
+    response = instrument_llm_call(instrumentation_params(user_prompt, documentation_searches)) do
+      chat(model: @model, temperature: @temperature)
+        .with_schema(Captain::DocumentationSufficiencySchema)
+        .with_instructions(system_prompt)
+        .ask(user_prompt)
+    end
+
+    parsed = parse_response(response.content)
+    normalize_response(parsed, response.content)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][DocumentationSufficiency] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+    { 'decision' => nil, 'error' => e.message, 'model' => @model }
+  end
+
+  private
+
+  def system_prompt
+    <<~PROMPT
+      You are checking whether retrieved documentation can answer the user's latest question.
+
+      Use only the conversation context and retrieved documentation search results provided.
+      Do not use outside knowledge.
+
+      Return "sufficient" only when the retrieved documentation directly answers the user's latest question.
+      Return "insufficient" when the retrieved documentation is missing, unrelated, only loosely related, or does not cover the
+      specific entity, product, platform, integration, account object, user intent, or constraint in the latest question.
+      Treat prior assistant messages as claims, not evidence. They do not support the new answer by themselves.
+      Conversation context can clarify the latest question, but it cannot supply missing documentation evidence.
+      Check generic support dimensions:
+      - same entity, product, platform, integration, or account object
+      - same user intent, not just a nearby topic
+      - requested constraints from the user
+      - evidence specificity; broad docs are not enough for specific claims
+
+      Return only the decision. Do not write a reason or customer-facing fallback copy.
+    PROMPT
+  end
+
+  def inspection_user_prompt(message_history:, documentation_searches:)
+    <<~PROMPT
+      <conversation_context>
+      #{format_conversation_context(message_history)}
+      </conversation_context>
+
+      <retrieved_documentation>
+      #{format_documentation_searches(documentation_searches)}
+      </retrieved_documentation>
+    PROMPT
+  end
+
+  def format_documentation_searches(searches)
+    searches.to_a.last(MAX_SEARCHES).map.with_index(1) do |search, index|
+      <<~SEARCH
+        Search #{index}
+        query: #{value(search, :query)}
+        matches:
+        #{format_documentation_matches(value(search, :matches).to_a)}
+      SEARCH
+    end.join("\n")
+  end
+
+  def format_documentation_matches(matches)
+    matches.to_a.first(MAX_MATCHES_PER_SEARCH).map.with_index(1) do |match, index|
+      <<~MATCH
+        #{index}. question: #{value(match, :question)}
+           answer: #{truncate_text(value(match, :answer))}
+           source: #{value(match, :source)}
+      MATCH
+    end.join("\n")
+  end
+
+  def value(hash, key) = hash && (hash[key] || hash[key.to_s])
+
+  def normalize_messages(message_history)
+    message_history.filter_map do |message|
+      role = value(message, :role)
+      next if role.blank?
+
+      { role: role.to_s, content: normalize_content(value(message, :content)) }
+    end
+  end
+
+  def normalize_content(content)
+    return content if content.is_a?(String)
+    return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
+
+    content.to_s
+  end
+
+  def text_part?(part)
+    return false unless part.is_a?(Hash)
+
+    (part[:type] || part['type']).to_s == 'text'
+  end
+
+  def format_conversation_context(messages)
+    normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+      content = message[:content].to_s.strip
+      next if content.blank?
+
+      "#{role_label(message[:role])}: #{content}"
+    end.join("\n")
+  end
+
+  def role_label(role) = { 'user' => 'User', 'assistant' => 'Assistant' }.fetch(role, role.to_s.titleize)
+
+  def parse_response(content)
+    return content if content.is_a?(Hash)
+
+    JSON.parse(sanitize_json_response(content))
+  rescue JSON::ParserError, TypeError
+    {}
+  end
+
+  def normalize_response(parsed, raw_content)
+    decision = parsed['decision'].to_s
+    return invalid_response(raw_content) unless Captain::DocumentationSufficiencySchema::DECISIONS.include?(decision)
+
+    {
+      'decision' => decision,
+      'raw_response' => raw_content,
+      'model' => @model
+    }
+  end
+
+  def invalid_response(raw_content)
+    {
+      'decision' => nil,
+      'raw_response' => raw_content,
+      'error' => 'invalid_documentation_sufficiency_response',
+      'model' => @model
+    }
+  end
+
+  def instrumentation_params(user_prompt, documentation_searches)
+    {
+      span_name: 'llm.captain.documentation_sufficiency',
+      model: @model,
+      temperature: @temperature,
+      account_id: @conversation.account_id,
+      conversation_id: @conversation.display_id,
+      feature_name: 'documentation_sufficiency',
+      messages: [
+        { role: 'system', content: system_prompt },
+        { role: 'user', content: user_prompt }
+      ],
+      metadata: {
+        assistant_id: @assistant.id,
+        channel_type: @conversation.inbox&.channel_type,
+        source: 'response_builder'
+      }.merge(search_metadata(documentation_searches))
+    }
+  end
+
+  def search_metadata(documentation_searches)
+    searches = documentation_searches.to_a
+    {
+      search_count: searches.length,
+      match_count: searches.sum { |search| value(search, :matches).to_a.length }
+    }
+  end
+
+  def truncate_text(text)
+    text = text.to_s
+    return text if text.length <= MAX_ANSWER_CHARS
+
+    "#{text.first(MAX_ANSWER_CHARS)}..."
+  end
+end

enterprise/app/services/captain/llm/system_prompts_service.rb

@@ -190,6 +190,7 @@ class Captain::Llm::SystemPromptsService
 
         [Error Handling]
         - If the required information is not found in the provided context, respond with an appropriate message indicating that no relevant data is available.
+        - If documentation search has no results, do not make factual claims from it. Say briefly that you do not have that information, ask one clarifying question if useful, or offer a human handoff.
         - Avoid speculating or providing unverified information.
 
         [Available Actions]
@@ -261,7 +262,9 @@ class Captain::Llm::SystemPromptsService
           response: '',
         }
         ```
-        - If the answer is not provided in context sections, Respond to the customer and ask whether they want to talk to another support agent . If they ask to Chat with another agent, return `conversation_handoff' as the response in JSON response
+        - For product facts, policies, account/service behavior, or how-to questions, call `search_documentation` before saying the answer is unavailable.
+        - If `search_documentation` does not provide the answer, respond to the customer and ask whether they want to talk to another support agent. If they ask to chat with another agent, return `conversation_handoff' as the response in JSON response
+        - If documentation search has no results, do not make factual claims from it. Say briefly that you do not have that information, ask one clarifying question if useful, or offer a human handoff.
         #{'- You MUST provide numbered citations at the appropriate places in the text.' if config['feature_citation']}
 
         #{build_tools_section(custom_tools)}

enterprise/app/services/captain/tools/instrumentation.rb

@@ -3,8 +3,24 @@ module Captain::Tools::Instrumentation
   include Integrations::LlmInstrumentation
 
   def execute(**args)
+    return super unless self.class.instrument_tool_execution?
+
     instrument_tool_call(name, args) do
       super
     end
   end
+
+  def self.prepended(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    def instrument_tool_execution?
+      @instrument_tool_execution != false
+    end
+
+    def skip_tool_execution_instrumentation
+      @instrument_tool_execution = false
+    end
+  end
 end

enterprise/app/services/captain/tools/search_documentation_service.rb

@@ -1,4 +1,14 @@
 class Captain::Tools::SearchDocumentationService < Captain::Tools::BaseTool
+  include Integrations::LlmInstrumentation
+  skip_tool_execution_instrumentation
+
+  def initialize(assistant, user: nil, on_search: nil, message_history: nil, conversation: nil)
+    super(assistant, user: user)
+    @on_search = on_search
+    @message_history = message_history
+    @conversation = conversation
+  end
+
   def self.name
     'search_documentation'
   end
@@ -13,26 +23,70 @@ class Captain::Tools::SearchDocumentationService < Captain::Tools::BaseTool
                        .new(account: assistant.account)
                        .translate(query, target_language: assistant.account.locale_english_name)
 
-    responses = assistant.responses.approved.search(translated_query)
-
-    return 'No FAQs found for the given query' if responses.empty?
-
-    responses.map { |response| format_response(response) }.join
+    instrument_documentation_search(query: translated_query, original_query: query) do
+      format_search_result(search_documentation(translated_query))
+    end
   end
 
   private
 
-  def format_response(response)
-    formatted_response = "
-        Question: #{response.question}
-        Answer: #{response.answer}
-        "
-    if response.documentable.present? && response.documentable.try(:external_link)
-      formatted_response += "
-          Source: #{response.documentable.external_link}
-          "
-    end
+  def instrument_documentation_search(query:, original_query:, &)
+    arguments = { query: query }
+    arguments[:original_query] = original_query if original_query != query
 
-    formatted_response
+    instrument_tool_call('search_documentation', arguments, &)
+  end
+
+  def write_search_metadata(result, documentation_sufficiency = nil)
+    span = OpenTelemetry::Trace.current_span
+    metadata = Captain::DocumentationSearchService.metadata(result)
+    decision = documentation_sufficiency && (documentation_sufficiency[:decision] || documentation_sufficiency['decision'])
+    metadata[:documentation_sufficiency] = decision if decision
+    metadata.each do |key, value|
+      span.set_attribute(format(ATTR_LANGFUSE_METADATA, key), value.to_s)
+      span.set_attribute(format(ATTR_LANGFUSE_OBSERVATION_METADATA, key), value.to_s)
+    end
+  rescue StandardError => e
+    Rails.logger.warn "#{self.class.name}: Failed to write search metadata: #{e.message}"
+  end
+
+  def search_documentation(query)
+    Captain::DocumentationSearchService.new(
+      scope: assistant.responses.approved,
+      account_id: assistant.account_id
+    ).search(query)
+  end
+
+  def format_search_result(result)
+    serialized_result = Captain::DocumentationSearchService.serialize(result)
+    documentation_sufficiency = evaluate_documentation_sufficiency(serialized_result)
+    serialized_result[:documentation_sufficiency] = documentation_sufficiency if documentation_sufficiency.present?
+    write_search_metadata(result, documentation_sufficiency)
+    @on_search&.call(serialized_result)
+
+    Captain::DocumentationSearchService.format_for_tool(
+      result,
+      no_results_message: 'No documentation found for the given query',
+      documentation_sufficiency: documentation_sufficiency
+    )
+  end
+
+  def evaluate_documentation_sufficiency(search)
+    return unless documentation_sufficiency_enabled?
+    return { 'decision' => 'insufficient', 'model' => nil } if search[:matches].blank?
+
+    Captain::Llm::DocumentationSufficiencyService.new(
+      assistant: assistant,
+      conversation: @conversation
+    ).evaluate(
+      message_history: @message_history.call,
+      documentation_searches: [search]
+    )
+  end
+
+  def documentation_sufficiency_enabled?
+    @conversation.present? &&
+      @message_history.respond_to?(:call) &&
+      ActiveModel::Type::Boolean.new.cast(assistant.config['documentation_sufficiency_gate_enabled'])
   end
 end

enterprise/app/services/captain/tools/search_reply_documentation_service.rb

@@ -5,9 +5,10 @@ class Captain::Tools::SearchReplyDocumentationService < RubyLLM::Tool
 
   param :query, desc: 'Search Query', required: true
 
-  def initialize(account:, assistant: nil)
+  def initialize(account:, assistant: nil, on_search: nil)
     @account = account
     @assistant = assistant
+    @on_search = on_search
     super()
   end
 
@@ -22,25 +23,20 @@ class Captain::Tools::SearchReplyDocumentationService < RubyLLM::Tool
                        .new(account: @account)
                        .translate(query, target_language: @account.locale_english_name)
 
-    responses = search_responses(translated_query)
-    return 'No FAQs found for the given query' if responses.empty?
+    result = Captain::DocumentationSearchService.new(
+      scope: search_scope,
+      account_id: @account.id
+    ).search(translated_query)
+    @on_search&.call(Captain::DocumentationSearchService.serialize(result))
 
-    responses.map { |response| format_response(response) }.join
+    Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
   end
 
   private
 
-  def search_responses(query)
-    if @assistant.present?
-      @assistant.responses.approved.search(query, account_id: @account.id)
-    else
-      @account.captain_assistant_responses.approved.search(query, account_id: @account.id)
-    end
-  end
+  def search_scope
+    return @assistant.responses.approved if @assistant.present?
 
-  def format_response(response)
-    result = "\nQuestion: #{response.question}\nAnswer: #{response.answer}\n"
-    result += "Source: #{response.documentable.external_link}\n" if response.documentable.present? && response.documentable.try(:external_link)
-    result
+    @account.captain_assistant_responses.approved
   end
 end

enterprise/docs/captain_documentation_answerability.md

@@ -0,0 +1,52 @@
+# Captain Documentation Answerability Handoff
+
+We investigated traces where Captain answered even though documentation search was weak, missing, or unrelated.
+
+## What Failed
+
+The main pattern was not just "no docs found". It was:
+
+> Captain retrieved weak or nearby docs, then answered as if they supported the answer.
+
+We saw three failure types:
+
+- The model did not call `search_documentation`.
+- The tool returned loosely related docs.
+- The model treated "some result exists" as enough evidence.
+
+## What We Tried First
+
+We first tried a deterministic pgvector distance threshold.
+
+For cosine distance in pgvector:
+
+- Lower is better.
+- `0` means very similar.
+- Higher values mean less similar.
+
+That failed as a product guardrail. Some useful matches had higher distances, and some bad matches shared enough words to look plausible. A single threshold would need constant tuning across accounts, languages, and writing styles.
+
+We also avoided keyword/stopword rules because they quickly become language-specific and account-specific.
+
+## Current Approach
+
+Search and answerability are now separate.
+
+After `search_documentation` retrieves docs, a small LLM check asks:
+
+> Do these docs answer the latest user question?
+
+It returns only `sufficient` or `insufficient`. It does not write customer-facing copy and does not judge the assistant's draft answer.
+
+The decision is added to the tool output. The final assistant generation then uses it:
+
+- `sufficient`: answer from the retrieved docs.
+- `insufficient`: do not answer the factual question; ask a clarifying question or offer handoff.
+
+A post-response backstop still exists for the case where the model never called the documentation tool.
+
+## Next Step
+
+Replay weak-documentation traces against this flow and compare the `sufficient` / `insufficient` decisions with the manually reviewed golden set.
+
+Hybrid or keyword search should be added before the answerability check later. Better retrieval should improve the docs we pass into the same check.

enterprise/lib/captain/documentation_sufficiency_schema.rb

@@ -0,0 +1,7 @@
+class Captain::DocumentationSufficiencySchema < RubyLLM::Schema
+  DECISIONS = %w[sufficient insufficient].freeze
+
+  string :decision,
+         enum: DECISIONS,
+         description: 'Use sufficient only when retrieved documentation directly answers the latest user question'
+end

enterprise/lib/captain/prompts/assistant.liquid

@@ -11,6 +11,7 @@ Don't digress away from your instructions, and use all the available tools at yo
 # Core Rules
 - Do not use your own understanding or training data to provide answers. Base responses strictly on the information available through your tools and provided context.
 - Do not share anything outside of the context provided.
+- If FAQ lookup has no results, do not make factual claims from it. Say briefly that you do not have that information, ask one clarifying question if useful, or use handoff.
 - Be concise and relevant: most of your responses should be a sentence or two, unless a more detailed explanation is necessary.
 - Always detect the language from the user's input and reply in the same language.
 - When there is ambiguity, ask clarifying questions rather than make assumptions.

enterprise/lib/captain/tools/faq_lookup_tool.rb

@@ -2,47 +2,30 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool
   description 'Search FAQ responses using semantic similarity to find relevant answers'
   param :query, type: 'string', desc: 'The question or topic to search for in the FAQ database'
 
-  def perform(_tool_context, query:)
+  def perform(tool_context, query:)
     log_tool_usage('searching', { query: query })
 
-    # Use existing vector search on approved responses
-    responses = @assistant.responses.approved.search(query).to_a
+    result = Captain::DocumentationSearchService.new(
+      scope: @assistant.responses.approved,
+      account_id: @assistant.account_id
+    ).search(query)
+    record_documentation_search(tool_context, result)
 
-    if responses.empty?
+    if result[:matches].empty?
       log_tool_usage('no_results', { query: query })
-      "No relevant FAQs found for: #{query}"
     else
-      log_tool_usage('found_results', { query: query, count: responses.size })
-      format_responses(responses)
+      log_tool_usage('completed', { query: query, count: result[:matches].size })
     end
+
+    Captain::DocumentationSearchService.format_for_tool(result, no_results_message: "No relevant FAQs found for: #{query}")
   end
 
   private
 
-  def format_responses(responses)
-    responses.map { |response| format_response(response) }.join
-  end
+  def record_documentation_search(tool_context, result)
+    searches = tool_context&.state&.dig(:documentation_searches)
+    return unless searches
 
-  def format_response(response)
-    formatted_response = "
-        Question: #{response.question}
-        Answer: #{response.answer}
-        "
-    if should_show_source?(response)
-      formatted_response += "
-          Source: #{response.documentable.external_link}
-          "
-    end
-
-    formatted_response
-  end
-
-  def should_show_source?(response)
-    return false if response.documentable.blank?
-    return false unless response.documentable.try(:external_link)
-
-    # Don't show source if it's a PDF placeholder
-    external_link = response.documentable.external_link
-    !external_link.start_with?('PDF:')
+    searches << Captain::DocumentationSearchService.serialize(result)
   end
 end

spec/enterprise/controllers/api/v1/accounts/captain/assistants_controller_spec.rb

@@ -216,6 +216,16 @@ RSpec.describe 'Api::V1::Accounts::Captain::Assistants', type: :request do
         expect(response).to have_http_status(:success)
         expect(json_response[:config][:feature_citation]).to be(false)
       end
+
+      it 'updates documentation sufficiency gate config' do
+        patch "/api/v1/accounts/#{account.id}/captain/assistants/#{assistant.id}",
+              params: { assistant: { config: { documentation_sufficiency_gate_enabled: true } } },
+              headers: admin.create_new_auth_token,
+              as: :json
+
+        expect(response).to have_http_status(:success)
+        expect(json_response[:config][:documentation_sufficiency_gate_enabled]).to be(true)
+      end
     end
   end
 

spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb

@@ -11,13 +11,22 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
     let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
     let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
     let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
+    let(:mock_documentation_sufficiency_service) { instance_double(Captain::Llm::DocumentationSufficiencyService) }
 
     before do
       create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
 
       allow(inbox).to receive(:captain_active?).and_return(true)
       allow(Captain::Llm::AssistantChatService).to receive(:new).and_return(mock_llm_chat_service)
-      allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
+      allow(mock_llm_chat_service).to receive(:generate_response) do |**_args, &block|
+        response = { 'response' => 'Hey, welcome to Captain Specs' }
+        block&.call(response)
+        response
+      end
+      allow(mock_llm_chat_service).to receive(:generate_documentation_gap_response).and_return(
+        { 'response' => 'I do not have enough information in the documentation. Would you like me to connect you with support?' }
+      )
+      allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([])
       allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
       allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
       allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
@@ -152,6 +161,116 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
         end
       end
 
+      context 'when documentation sufficiency gate is enabled' do
+        let(:empty_search_result) do
+          {
+            query: 'current plan limits',
+            queries: ['current plan limits'],
+            matches: []
+          }
+        end
+
+        before do
+          assistant.update!(config: { 'documentation_sufficiency_gate_enabled' => true })
+          allow(Captain::Llm::DocumentationSufficiencyService).to receive(:new).with(
+            assistant: assistant,
+            conversation: conversation
+          ).and_return(mock_documentation_sufficiency_service)
+        end
+
+        it 'regenerates an unsupported answer with a documentation-gap instruction' do
+          allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([empty_search_result])
+          allow(mock_llm_chat_service).to receive(:generate_response) do |**_args, &block|
+            response = { 'response' => 'Your current plan has unlimited usage.' }
+            block&.call(response)
+            response
+          end
+          expect(mock_llm_chat_service).to receive(:generate_documentation_gap_response).with(
+            message_history: [{ content: 'Hello', role: 'user' }]
+          ).and_return(
+            { 'response' => 'I do not have enough information in the documentation. Would you like me to connect you with support?' }
+          )
+          allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
+            {
+              'decision' => 'insufficient',
+              'model' => 'gpt-4.1'
+            }
+          )
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq(
+            'I do not have enough information in the documentation. Would you like me to connect you with support?'
+          )
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'keeps the assistant response when the gate finds enough support' do
+          allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([empty_search_result])
+          allow(mock_llm_chat_service).to receive(:generate_response) do |**_args, &block|
+            response = { 'response' => 'Billing settings show current plan usage.' }
+            block&.call(response)
+            response
+          end
+          allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
+            { 'decision' => 'sufficient', 'model' => 'gpt-4.1' }
+          )
+          expect(mock_llm_chat_service).not_to receive(:generate_documentation_gap_response)
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq('Billing settings show current plan usage.')
+        end
+
+        it 'does not run a second repair when documentation support was already checked inside the tool' do
+          allow(mock_llm_chat_service).to receive(:documentation_searches).and_return(
+            [empty_search_result.merge(documentation_sufficiency: { 'decision' => 'insufficient', 'model' => 'gpt-5.4-mini' })]
+          )
+          allow(mock_llm_chat_service).to receive(:generate_response) do |**_args, &block|
+            response = { 'response' => 'I do not have that information in the documentation. Would you like a handoff?' }
+            block&.call(response)
+            response
+          end
+          expect(mock_documentation_sufficiency_service).not_to receive(:evaluate)
+          expect(mock_llm_chat_service).not_to receive(:generate_documentation_gap_response)
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq(
+            'I do not have that information in the documentation. Would you like a handoff?'
+          )
+        end
+
+        it 'checks answers even when no documentation search was recorded' do
+          allow(mock_llm_chat_service).to receive(:generate_response) do |**_args, &block|
+            response = { 'response' => 'Your current plan costs $99 per agent per month.' }
+            block&.call(response)
+            response
+          end
+          expect(mock_documentation_sufficiency_service).to receive(:evaluate).with(
+            message_history: [{ content: 'Hello', role: 'user' }],
+            documentation_searches: [
+              {
+                query: 'Hello',
+                queries: ['Hello'],
+                matches: []
+              }
+            ]
+          ).and_return(
+            {
+              'decision' => 'insufficient',
+              'model' => 'gpt-4.1'
+            }
+          )
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq(
+            'I do not have enough information in the documentation. Would you like me to connect you with support?'
+          )
+        end
+      end
+
       it 'does not send a response when the conversation is no longer pending' do
         conversation.open!
 
@@ -366,6 +485,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       allow(Captain::OpenAiMessageBuilderService).to receive(:new).with(message: anything).and_return(mock_message_builder)
       allow(mock_message_builder).to receive(:generate_content).and_return('Hello with image')
       allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Test response' })
+      allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([])
     end
 
     context 'when ActiveStorage::FileNotFoundError occurs' do
@@ -478,6 +598,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       allow(Captain::Llm::AssistantChatService).to receive(:new).and_return(mock_llm_chat_service)
       allow(account).to receive(:feature_enabled?).and_return(false)
       allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
+      allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([])
     end
 
     context 'when handoff occurs outside business hours' do

spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb

@@ -4,16 +4,14 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
   let(:account) { create(:account) }
   let(:assistant) { create(:captain_assistant, account: account) }
   let(:tool) { described_class.new(assistant) }
-  let(:tool_context) { Struct.new(:state).new({}) }
+  let(:documentation_searches) { [] }
+  let(:tool_context) { Struct.new(:state).new({ documentation_searches: documentation_searches }) }
+  let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
 
   before do
-    # Create installation config for OpenAI API key to avoid errors
-    create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
-
-    # Mock embedding service to avoid actual API calls
-    embedding_service = instance_double(Captain::Llm::EmbeddingService)
-    allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
-    allow(embedding_service).to receive(:get_embedding).and_return(Array.new(1536, 0.1))
+    allow(Captain::DocumentationSearchService).to receive(:new)
+      .with(scope: anything, account_id: assistant.account_id)
+      .and_return(documentation_search_service)
   end
 
   describe '#description' do
@@ -32,11 +30,24 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
   end
 
   describe '#perform' do
+    def search_match(response)
+      Captain::AssistantResponse::SearchMatch.new(response: response, semantic_distance: 0.2)
+    end
+
+    def search_result(query:, matches:)
+      {
+        query: query,
+        queries: [query],
+        matches: matches
+      }
+    end
+
     context 'when FAQs exist' do
       let(:document) { create(:captain_document, assistant: assistant) }
       let!(:response1) do
         create(:captain_assistant_response,
                assistant: assistant,
+               account: account,
                question: 'How to reset password?',
                answer: 'Click on forgot password link',
                documentable: document,
@@ -45,15 +56,16 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
       let!(:response2) do
         create(:captain_assistant_response,
                assistant: assistant,
+               account: account,
                question: 'How to change email?',
                answer: 'Go to settings and update email',
                status: 'approved')
       end
 
       before do
-        # Mock nearest_neighbors to return our test responses
-        allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(
-          Captain::AssistantResponse.where(id: [response1.id, response2.id])
+        matches = [response1, response2].map { |response| search_match(response) }
+        allow(documentation_search_service).to receive(:search).and_return(
+          search_result(query: 'password reset', matches: matches)
         )
       end
 
@@ -64,6 +76,7 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
         expect(result).to include('Answer: Click on forgot password link')
         expect(result).to include('Question: How to change email?')
         expect(result).to include('Answer: Go to settings and update email')
+        expect(documentation_searches.first[:matches].first[:question]).to eq('How to reset password?')
       end
 
       it 'includes source link when document has external_link' do
@@ -76,7 +89,10 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
 
       it 'logs tool usage for search' do
         expect(tool).to receive(:log_tool_usage).with('searching', { query: 'password reset' })
-        expect(tool).to receive(:log_tool_usage).with('found_results', { query: 'password reset', count: 2 })
+        expect(tool).to receive(:log_tool_usage).with(
+          'completed',
+          { query: 'password reset', count: 2 }
+        )
 
         tool.perform(tool_context, query: 'password reset')
       end
@@ -84,13 +100,15 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
 
     context 'when no FAQs found' do
       before do
-        # Return empty result set
-        allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
+        allow(documentation_search_service).to receive(:search).and_return(
+          search_result(query: 'nonexistent topic', matches: [])
+        )
       end
 
       it 'returns no results message' do
         result = tool.perform(tool_context, query: 'nonexistent topic')
-        expect(result).to eq('No relevant FAQs found for: nonexistent topic')
+        expect(result).to include('No relevant FAQs found for: nonexistent topic')
+        expect(result).to include('No documentation matched this query')
       end
 
       it 'logs tool usage for no results' do
@@ -103,11 +121,12 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
 
     context 'with blank query' do
       it 'handles empty query' do
-        # Return empty result set
-        allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
+        allow(documentation_search_service).to receive(:search).and_return(
+          search_result(query: '', matches: [])
+        )
 
         result = tool.perform(tool_context, query: '')
-        expect(result).to eq('No relevant FAQs found for: ')
+        expect(result).to include('No relevant FAQs found for: ')
       end
     end
   end

spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb

@@ -167,7 +167,17 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
     it 'processes and formats agent result' do
       result = service.generate_response(message_history: message_history)
 
-      expect(result).to eq({ 'response' => 'Test response', 'agent_name' => nil, 'handoff_tool_called' => false })
+      expect(result).to include('response' => 'Test response', 'agent_name' => nil, 'handoff_tool_called' => false)
+    end
+
+    it 'includes documentation searches from runner state' do
+      searches = [{ query: 'billing', matches: [] }]
+      runner_context = { state: { documentation_searches: searches } }
+      allow(mock_result).to receive(:context).and_return(runner_context)
+
+      result = service.generate_response(message_history: message_history)
+
+      expect(result['documentation_searches']).to eq(searches)
     end
 
     context 'when handoff tool was called during agent execution' do
@@ -179,11 +189,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
       it 'includes handoff_tool_called flag in response' do
         result = service.generate_response(message_history: message_history)
 
-        expect(result).to eq({
-                               'response' => 'Let me connect you',
-                               'agent_name' => nil,
-                               'handoff_tool_called' => true
-                             })
+        expect(result).to include('response' => 'Let me connect you', 'agent_name' => nil, 'handoff_tool_called' => true)
       end
     end
 
@@ -208,12 +214,12 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
       it 'formats string response correctly' do
         result = service.generate_response(message_history: message_history)
 
-        expect(result).to eq({
-                               'response' => 'Simple string response',
-                               'reasoning' => 'Processed by agent',
-                               'agent_name' => nil,
-                               'handoff_tool_called' => false
-                             })
+        expect(result).to include(
+          'response' => 'Simple string response',
+          'reasoning' => 'Processed by agent',
+          'agent_name' => nil,
+          'handoff_tool_called' => false
+        )
       end
     end
 

spec/enterprise/services/captain/documentation_search_service_spec.rb

@@ -0,0 +1,96 @@
+require 'rails_helper'
+
+RSpec.describe Captain::DocumentationSearchService do
+  let(:scope_class) do
+    Class.new do
+      def search_with_metadata(*)
+        []
+      end
+    end
+  end
+  let(:scope) { scope_class.new }
+  let(:service) { described_class.new(scope: scope, account_id: 1) }
+  let(:response) do
+    instance_double(
+      Captain::AssistantResponse,
+      id: 1,
+      question: 'How do plan limits work?',
+      answer: 'Monthly limits are shown in billing settings.',
+      documentable: nil
+    )
+  end
+
+  def search_match(semantic_distance:, response_record: response)
+    Captain::AssistantResponse::SearchMatch.new(
+      response: response_record,
+      semantic_distance: semantic_distance
+    )
+  end
+
+  def search_result(matches:, query: 'billing')
+    {
+      query: query,
+      queries: [query],
+      matches: matches
+    }
+  end
+
+  describe '#search' do
+    it 'returns semantic matches without assigning retrieval quality' do
+      query = 'How do I check limits for my current monthly plan?'
+      match = search_match(semantic_distance: 0.9)
+
+      allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([match])
+
+      result = service.search(query)
+
+      expect(result[:matches]).to eq([match])
+      expect(result[:queries]).to eq([query])
+    end
+
+    it 'returns an empty match list when no documentation matches' do
+      query = 'Where do I find billing settings?'
+
+      allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([])
+
+      result = service.search(query)
+
+      expect(result[:matches]).to eq([])
+      expect(result[:queries]).to eq([query])
+    end
+  end
+
+  describe '.format_for_tool' do
+    it 'adds a bounded-answer instruction when no documentation is found' do
+      result = search_result(query: 'unknown topic', matches: [])
+
+      formatted_result = described_class.format_for_tool(result, no_results_message: 'No FAQs found')
+
+      expect(formatted_result).to include('No FAQs found')
+      expect(formatted_result).to include('No documentation matched this query')
+    end
+  end
+
+  describe '.serialize' do
+    it 'formats search metadata for the response-level support gate' do
+      result = search_result(matches: [search_match(semantic_distance: 0.2)])
+
+      serialized_result = described_class.serialize(result)
+
+      expect(serialized_result[:matches].first[:semantic_distance]).to eq(0.2)
+    end
+  end
+
+  describe '.metadata' do
+    it 'returns compact search metadata' do
+      result = search_result(matches: [search_match(semantic_distance: 0.2)])
+
+      expect(described_class.metadata(result)).to eq(
+        {
+          match_count: 1,
+          top_semantic_distance: 0.2
+        }
+      )
+    end
+  end
+end

spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb

@@ -63,6 +63,43 @@ RSpec.describe Captain::Llm::AssistantChatService do
 
       expect(attributes['langfuse.observation.metadata.generation_stage']).to eq('tool_call')
     end
+
+    it 'runs the response hook before the agent session span closes' do
+      service = described_class.new(assistant: assistant, conversation: conversation)
+      hook_calls = []
+
+      expect(service).to receive(:instrument_agent_session).and_wrap_original do |original, params, &block|
+        original.call(params) do
+          result = block.call
+          hook_calls << :after_hook
+          result
+        end
+      end
+
+      allow(mock_chat).to receive(:ask).and_return(mock_response)
+      service.generate_response(message_history: [{ role: 'user', content: 'Hello' }]) do |response|
+        hook_calls << :hook
+        response
+      end
+
+      expect(hook_calls).to eq(%i[hook after_hook])
+    end
+  end
+
+  describe '#generate_documentation_gap_response' do
+    it 'generates a constrained response without calling tools' do
+      service = described_class.new(assistant: assistant, conversation: conversation)
+
+      expect(mock_chat).not_to receive(:with_tool)
+      expect(mock_chat).to receive(:with_instructions).with(a_string_including('[Documentation Support]')).and_return(mock_chat)
+      expect(mock_chat).to receive(:ask).with('Do your documents auto refresh?').and_return(mock_response)
+
+      response = service.generate_documentation_gap_response(
+        message_history: [{ role: 'user', content: 'Do your documents auto refresh?' }]
+      )
+
+      expect(response['response']).to eq('I can see the image shows a pricing table')
+    end
   end
 
   describe 'image analysis' do

spec/enterprise/services/captain/llm/documentation_sufficiency_service_spec.rb

@@ -0,0 +1,43 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Llm::DocumentationSufficiencyService do
+  let(:account) { create(:account) }
+  let(:assistant) { create(:captain_assistant, account: account) }
+  let(:conversation) { create(:conversation, account: account) }
+  let(:service) { described_class.new(assistant: assistant, conversation: conversation) }
+  let(:mock_chat) { instance_double(RubyLLM::Chat) }
+  let(:mock_response) do
+    instance_double(
+      RubyLLM::Message,
+      content: { 'decision' => 'sufficient' }
+    )
+  end
+
+  before do
+    allow(RubyLLM).to receive(:chat).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
+    allow(mock_chat).to receive(:ask).and_return(mock_response)
+  end
+
+  describe '#evaluate' do
+    it 'uses the documentation support model instead of the global Captain model' do
+      create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-5.5')
+
+      expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.4-mini').and_return(mock_chat)
+
+      result = service.evaluate(
+        message_history: [{ role: 'user', content: 'Who is your mascot?' }],
+        documentation_searches: [
+          {
+            query: 'mascot',
+            matches: [{ question: 'Who is the brand mascot?', answer: 'Robin the bird.' }]
+          }
+        ]
+      )
+
+      expect(result).to include('decision' => 'sufficient', 'model' => 'gpt-5.4-mini')
+    end
+  end
+end

spec/enterprise/services/captain/tools/search_documentation_service_spec.rb

@@ -26,10 +26,14 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
   end
 
   describe '#execute' do
+    let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
+    let(:documentation_sufficiency_service) { instance_double(Captain::Llm::DocumentationSufficiencyService) }
+    let(:translate_query_service) { instance_double(Captain::Llm::TranslateQueryService) }
     let!(:response) do
       create(
         :captain_assistant_response,
         assistant: assistant,
+        account: assistant.account,
         question: question,
         answer: answer,
         status: 'approved'
@@ -37,29 +41,99 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
     end
 
     let(:documentable) { create(:captain_document, external_link: external_link) }
+    let(:recorded_searches) { [] }
+    let(:service) { described_class.new(assistant, on_search: ->(search) { recorded_searches << search }) }
+    let(:match) do
+      Captain::AssistantResponse::SearchMatch.new(
+        response: response,
+        semantic_distance: 0.2
+      )
+    end
+
+    def search_result(matches:)
+      {
+        query: question,
+        queries: [question],
+        matches: matches
+      }
+    end
+
+    before do
+      allow(Captain::Llm::TranslateQueryService).to receive(:new).and_return(translate_query_service)
+      allow(translate_query_service).to receive(:translate).and_return(question)
+      allow(Captain::DocumentationSearchService).to receive(:new)
+        .with(scope: anything, account_id: assistant.account_id)
+        .and_return(documentation_search_service)
+    end
 
     context 'when matching responses exist' do
-      before do
-        response.update(documentable: documentable)
-        allow(Captain::AssistantResponse).to receive(:search).with(question).and_return([response])
-      end
-
       it 'returns formatted responses for the search query' do
+        response.update(documentable: documentable)
+        allow(documentation_search_service).to receive(:search).with(question).and_return(
+          search_result(matches: [match])
+        )
+
         result = service.execute(query: question)
 
         expect(result).to include(question)
         expect(result).to include(answer)
         expect(result).to include(external_link)
+        expect(recorded_searches.first[:matches].first[:question]).to eq(question)
       end
     end
 
     context 'when no matching responses exist' do
-      before do
-        allow(Captain::AssistantResponse).to receive(:search).with(question).and_return([])
+      it 'returns a bounded no-results instruction' do
+        allow(documentation_search_service).to receive(:search).with(question).and_return(
+          search_result(matches: [])
+        )
+
+        result = service.execute(query: question)
+
+        expect(result).to include('No documentation found for the given query')
+        expect(result).to include('No documentation matched this query')
+      end
+    end
+
+    context 'when documentation sufficiency gate is enabled' do
+      let(:conversation) { create(:conversation, account: assistant.account) }
+      let(:message_history) { [{ role: 'user', content: 'Who is your mascot?' }] }
+      let(:service) do
+        described_class.new(
+          assistant,
+          on_search: ->(search) { recorded_searches << search },
+          message_history: -> { message_history },
+          conversation: conversation
+        )
       end
 
-      it 'returns an empty string' do
-        expect(service.execute(query: question)).to eq('No FAQs found for the given query')
+      before do
+        assistant.update!(config: assistant.config.merge('documentation_sufficiency_gate_enabled' => true))
+        allow(Captain::Llm::DocumentationSufficiencyService).to receive(:new).with(
+          assistant: assistant,
+          conversation: conversation
+        ).and_return(documentation_sufficiency_service)
+      end
+
+      it 'returns insufficient documentation support to the final assistant generation' do
+        allow(documentation_search_service).to receive(:search).with(question).and_return(
+          search_result(matches: [match])
+        )
+        allow(documentation_sufficiency_service).to receive(:evaluate).with(
+          message_history: message_history,
+          documentation_searches: [
+            hash_including(
+              query: question,
+              matches: [hash_including(question: question, answer: answer)]
+            )
+          ]
+        ).and_return({ 'decision' => 'insufficient', 'model' => 'gpt-5.4-mini' })
+
+        result = service.execute(query: question)
+
+        expect(result).to include('Documentation support: insufficient')
+        expect(result).to include('Do not answer the factual question from these results')
+        expect(recorded_searches.first[:documentation_sufficiency]).to include('decision' => 'insufficient')
       end
     end
   end