initial commit

enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb

@@ -58,6 +58,7 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base
                                                   config: [
                                                     :product_name, :feature_faq, :feature_memory, :feature_citation,
                                                     :feature_contact_attributes,
+                                                    :documentation_sufficiency_gate_enabled,
                                                     :welcome_message, :handoff_message, :resolution_message,
                                                     :instructions, :temperature
                                                   ])

enterprise/app/jobs/captain/conversation/documentation_sufficiency_handler.rb

@@ -0,0 +1,110 @@
+module Captain::Conversation::DocumentationSufficiencyHandler
+  HIGH_RISK_TERMS = %w[
+    price pricing cost billing bill paid free plan subscription legal compliance policy limit limits maximum minimum
+    available availability current currently roadmap beta early access supported support self-hosted cloud region version
+    provider integration account status
+  ].freeze
+
+  private
+
+  def reset_documentation_searches
+    Current.captain_documentation_searches = []
+  end
+
+  def clear_documentation_searches
+    Current.captain_documentation_searches = nil
+  end
+
+  def inspect_documentation_sufficiency(message_history)
+    searches = documentation_searches_for_inspection(message_history)
+    return unless should_inspect_documentation_sufficiency?(message_history, searches)
+
+    apply_documentation_sufficiency_inspection(documentation_sufficiency_inspection(message_history, searches))
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][ResponseBuilderJob] Documentation sufficiency check failed for account=#{account.id} " \
+      "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+  end
+
+  def should_inspect_documentation_sufficiency?(message_history, searches)
+    return false unless documentation_sufficiency_gate_enabled?
+    return false unless response_has_user_facing_answer?
+    return false if searches.empty?
+
+    documentation_sufficiency_check_needed?(message_history, searches)
+  end
+
+  def response_has_user_facing_answer?
+    @response.present? &&
+      @response['response'].present? &&
+      @response['response'] != 'conversation_handoff' &&
+      !@response['handoff_tool_called']
+  end
+
+  def documentation_searches_for_inspection(message_history)
+    searches = Current.captain_documentation_searches.to_a
+    return searches if searches.present?
+    return [] unless high_risk_conversation?(message_history)
+
+    [synthetic_no_results_search(message_history)]
+  end
+
+  def synthetic_no_results_search(message_history)
+    {
+      query: last_user_message_content(message_history),
+      queries: [last_user_message_content(message_history)],
+      status: 'weak',
+      reason: 'no_documentation_search',
+      matches: []
+    }
+  end
+
+  def documentation_sufficiency_inspection(message_history, searches)
+    Captain::Llm::DocumentationSufficiencyService.new(
+      assistant: @assistant,
+      conversation: @conversation
+    ).evaluate(
+      message_history: message_history,
+      assistant_response: @response['response'],
+      documentation_searches: searches
+    )
+  end
+
+  def documentation_sufficiency_gate_enabled?
+    ActiveModel::Type::Boolean.new.cast(@assistant.config['documentation_sufficiency_gate_enabled'])
+  end
+
+  def documentation_sufficiency_check_needed?(message_history, searches)
+    searches.any? { |search| search[:status] == 'weak' || search['status'] == 'weak' } || high_risk_conversation?(message_history)
+  end
+
+  def high_risk_conversation?(message_history)
+    text = "#{message_history.last(3).map { |message| message[:content] || message['content'] }.join(' ')} #{@response&.dig('response')}".downcase
+    HIGH_RISK_TERMS.any? { |term| text.include?(term) }
+  end
+
+  def last_user_message_content(message_history)
+    user_message = message_history.reverse.find { |message| (message[:role] || message['role']).to_s == 'user' }
+    user_message && (user_message[:content] || user_message['content']).to_s
+  end
+
+  def apply_documentation_sufficiency_inspection(inspection)
+    return unless inspection['decision'] == 'insufficient'
+
+    fallback_response = inspection['fallback_response'].presence || default_documentation_sufficiency_fallback
+    @response.merge!(
+      'response' => fallback_response,
+      'action' => 'continue',
+      'action_reason' => 'missing_docs_bounded_answer',
+      'action_source' => 'documentation_sufficiency',
+      'documentation_sufficiency_reason' => inspection['reason'],
+      'documentation_sufficiency_model' => inspection['model']
+    )
+  end
+
+  def default_documentation_sufficiency_fallback
+    "I couldn't find enough information to answer that confidently. Would you like me to connect you with a support person?"
+  end
+end

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -1,5 +1,6 @@
 class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   include Captain::Conversation::V1ActionClassifier
+  include Captain::Conversation::DocumentationSufficiencyHandler
 
   MAX_MESSAGE_LENGTH = 10_000
   retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
@@ -25,6 +26,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   rescue StandardError => e
     handle_error(e)
   ensure
+    clear_documentation_searches
     Current.executed_by = nil
   end
 
@@ -34,17 +36,22 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
   def generate_and_process_response
     message_history = collect_previous_messages
+    reset_documentation_searches
     @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
       message_history: message_history
     )
+    inspect_documentation_sufficiency(message_history) if conversation_pending?
     classify_v1_response_action(message_history) if conversation_pending?
     process_response
   end
 
   def generate_response_with_v2
+    message_history = collect_previous_messages
+    reset_documentation_searches
     @response = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation).generate_response(
-      message_history: collect_previous_messages
+      message_history: message_history
     )
+    inspect_documentation_sufficiency(message_history) if conversation_pending?
     process_response
   end
 

enterprise/app/models/captain/assistant_response.rb

@@ -25,6 +25,35 @@
 #
 class Captain::AssistantResponse < ApplicationRecord
   self.table_name = 'captain_assistant_responses'
+  SEARCH_LIMIT = 5
+  KEYWORD_SEARCH_LIMIT_MULTIPLIER = 2
+  SEARCH_STOP_WORDS = %w[
+    about after all also and any are but can for from has have how into its may more not now off our out
+    the their them then there these they this was what when where which who why with you your
+  ].freeze
+  SearchMatch = Struct.new(
+    :response,
+    :semantic_distance,
+    :keyword_score,
+    :keyword_coverage,
+    :matched_terms,
+    :retrieval_methods,
+    keyword_init: true
+  ) do
+    def to_h
+      {
+        response_id: response.id,
+        question: response.question,
+        answer: response.answer,
+        source: response.documentable&.try(:external_link),
+        semantic_distance: semantic_distance,
+        keyword_score: keyword_score,
+        keyword_coverage: keyword_coverage,
+        matched_terms: matched_terms,
+        retrieval_methods: retrieval_methods
+      }
+    end
+  end
 
   belongs_to :assistant, class_name: 'Captain::Assistant'
   belongs_to :account
@@ -47,8 +76,105 @@ class Captain::AssistantResponse < ApplicationRecord
   enum status: { pending: 0, approved: 1 }
 
   def self.search(query, account_id: nil)
+    search_with_metadata(query, account_id: account_id).map(&:response)
+  end
+
+  def self.search_with_metadata(query, account_id: nil, limit: SEARCH_LIMIT)
+    semantic_matches = semantic_search_matches(query, account_id: account_id, limit: limit)
+    keyword_matches = keyword_search_matches(query, limit: limit * KEYWORD_SEARCH_LIMIT_MULTIPLIER)
+
+    merge_search_matches(semantic_matches, keyword_matches).first(limit)
+  end
+
+  def self.semantic_search_matches(query, account_id:, limit:)
     embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(query)
-    nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
+    nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(limit).map do |response|
+      SearchMatch.new(
+        response: response,
+        semantic_distance: response.neighbor_distance&.to_f,
+        keyword_score: 0,
+        keyword_coverage: 0.0,
+        matched_terms: [],
+        retrieval_methods: ['semantic']
+      )
+    end
+  end
+
+  def self.keyword_search_matches(query, limit:)
+    terms = search_terms(query)
+    return [] if terms.empty?
+
+    matches = keyword_search_scope(terms).limit(limit).map do |response|
+      matched_terms = matched_terms_for(response, terms)
+      SearchMatch.new(
+        response: response,
+        semantic_distance: nil,
+        keyword_score: matched_terms.size,
+        keyword_coverage: matched_terms.size.to_f / terms.size,
+        matched_terms: matched_terms,
+        retrieval_methods: ['keyword']
+      )
+    end
+    matches.sort_by { |match| [-match.keyword_score, match.response.id] }
+  end
+
+  def self.keyword_search_scope(terms)
+    conditions = []
+    values = []
+
+    terms.each do |term|
+      pattern = "%#{sanitize_sql_like(term)}%"
+      conditions << '(question ILIKE ? OR answer ILIKE ?)'
+      values.push(pattern, pattern)
+    end
+
+    where(conditions.join(' OR '), *values)
+  end
+
+  def self.search_terms(query)
+    query.to_s.downcase.scan(/[[:alnum:]]+/).filter_map do |term|
+      next if term.length < 3
+      next if SEARCH_STOP_WORDS.include?(term)
+
+      term
+    end.uniq
+  end
+
+  def self.matched_terms_for(response, terms)
+    text = "#{response.question} #{response.answer}".downcase
+    terms.select { |term| text.include?(term) }
+  end
+
+  def self.merge_search_matches(semantic_matches, keyword_matches)
+    matches_by_response_id = {}
+
+    semantic_matches.each do |match|
+      matches_by_response_id[match.response.id] = match
+    end
+
+    keyword_matches.each do |keyword_match|
+      existing = matches_by_response_id[keyword_match.response.id]
+      if existing
+        merge_keyword_match!(existing, keyword_match)
+      else
+        matches_by_response_id[keyword_match.response.id] = keyword_match
+      end
+    end
+
+    sort_search_matches(matches_by_response_id.values)
+  end
+
+  def self.merge_keyword_match!(existing, keyword_match)
+    existing.keyword_score = keyword_match.keyword_score
+    existing.keyword_coverage = keyword_match.keyword_coverage
+    existing.matched_terms = keyword_match.matched_terms
+    existing.retrieval_methods |= keyword_match.retrieval_methods
+  end
+
+  def self.sort_search_matches(matches)
+    matches.sort_by do |match|
+      [match.semantic_distance || 1.0, -match.keyword_score, match.response.id]
+    end
   end
 
   private

enterprise/app/services/captain/documentation_search_service.rb

@@ -0,0 +1,155 @@
+class Captain::DocumentationSearchService
+  MAX_ACCEPTABLE_COSINE_DISTANCE = 0.45
+  MIN_ACCEPTABLE_KEYWORD_COVERAGE = 0.4
+  TOP_MATCHES_TO_FORMAT = 5
+  SEARCH_ATTEMPT_LIMIT = 3
+
+  Result = Struct.new(:query, :queries, :matches, :status, :reason, keyword_init: true) do
+    def weak?
+      status == 'weak'
+    end
+
+    def empty?
+      matches.empty?
+    end
+
+    def to_h
+      {
+        query: query,
+        queries: queries,
+        status: status,
+        reason: reason,
+        matches: matches.map(&:to_h)
+      }
+    end
+  end
+
+  def initialize(scope:, account_id: nil)
+    @scope = scope
+    @account_id = account_id
+  end
+
+  def search(query)
+    matches = []
+    queries = []
+
+    search_queries(query).each do |search_query|
+      queries << search_query
+      matches = merge_matches(matches, @scope.search_with_metadata(search_query, account_id: @account_id))
+      break if matches.first && sufficient_match?(matches.first)
+    end
+
+    Result.new(query: query, queries: queries, matches: matches, status: status_for(matches), reason: reason_for(matches))
+  end
+
+  def self.record(result)
+    Current.captain_documentation_searches ||= []
+    Current.captain_documentation_searches << result.to_h
+  end
+
+  def self.format_for_tool(result, no_results_message:)
+    return "#{no_results_message}\n\n#{weak_instruction(result)}" if result.empty?
+
+    sections = [quality_section(result)]
+    sections.concat(result.matches.first(TOP_MATCHES_TO_FORMAT).map { |match| format_match(match) })
+    sections.join("\n")
+  end
+
+  def self.format_match(match)
+    response = match.response
+    formatted_response = "\nQuestion: #{response.question}\nAnswer: #{response.answer}\n"
+    if response.documentable.present? && response.documentable.try(:external_link)
+      formatted_response += "Source: #{response.documentable.external_link}\n"
+    end
+    formatted_response += "Retrieval: #{match.retrieval_methods.join(', ')}"
+    formatted_response += ", semantic_distance=#{format('%.4f', match.semantic_distance)}" if match.semantic_distance.present?
+    formatted_response += ", keyword_coverage=#{format('%.2f', match.keyword_coverage)}" if match.keyword_score.positive?
+    "#{formatted_response}\n"
+  end
+
+  def self.quality_section(result)
+    lines = ["Search quality: #{result.status}", "Search reason: #{result.reason}"]
+    lines << "Search queries: #{result.queries.join(' | ')}" if result.queries.to_a.size > 1
+    lines << weak_instruction(result) if result.weak?
+    lines.join("\n")
+  end
+
+  def self.weak_instruction(_result)
+    [
+      'Instruction: The retrieved documentation is missing or weak. Do not use it to make factual claims.',
+      "Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or ask whether",
+      'the user wants to talk to a support person.'
+    ].join(' ')
+  end
+
+  private
+
+  def search_queries(query)
+    query = query.to_s.strip
+    terms = Captain::AssistantResponse.search_terms(query)
+    keyword_query = terms.join(' ')
+    trailing_query = terms.last(8).join(' ')
+
+    [query, keyword_query, trailing_query].filter_map(&:presence).uniq.first(SEARCH_ATTEMPT_LIMIT)
+  end
+
+  def merge_matches(existing_matches, new_matches)
+    matches_by_response_id = existing_matches.index_by { |match| match.response.id }
+
+    new_matches.each do |new_match|
+      existing_match = matches_by_response_id[new_match.response.id]
+      if existing_match
+        merge_match!(existing_match, new_match)
+      else
+        matches_by_response_id[new_match.response.id] = new_match
+      end
+    end
+
+    matches_by_response_id.values.sort_by do |match|
+      [match.semantic_distance || 1.0, -match.keyword_score, match.response.id]
+    end
+  end
+
+  def merge_match!(existing_match, new_match)
+    existing_match.semantic_distance = best_semantic_distance(existing_match.semantic_distance, new_match.semantic_distance)
+    existing_match.keyword_score = [existing_match.keyword_score, new_match.keyword_score].max
+    existing_match.keyword_coverage = [existing_match.keyword_coverage, new_match.keyword_coverage].max
+    existing_match.matched_terms |= new_match.matched_terms
+    existing_match.retrieval_methods |= new_match.retrieval_methods
+  end
+
+  def best_semantic_distance(existing_distance, new_distance)
+    return new_distance if existing_distance.blank?
+    return existing_distance if new_distance.blank?
+
+    [existing_distance, new_distance].min
+  end
+
+  def status_for(matches)
+    return 'weak' if matches.empty?
+
+    sufficient_match?(matches.first) ? 'sufficient' : 'weak'
+  end
+
+  def reason_for(matches)
+    return 'no_results' if matches.empty?
+
+    top_match = matches.first
+    return 'semantic_match' if top_match.semantic_distance.present? && top_match.semantic_distance <= MAX_ACCEPTABLE_COSINE_DISTANCE
+    return 'keyword_match' if top_match.keyword_coverage >= MIN_ACCEPTABLE_KEYWORD_COVERAGE
+
+    'low_retrieval_confidence'
+  end
+
+  def sufficient_match?(match)
+    semantic_match?(match) || keyword_match?(match)
+  end
+
+  def semantic_match?(match)
+    match.semantic_distance.present? && match.semantic_distance <= MAX_ACCEPTABLE_COSINE_DISTANCE
+  end
+
+  def keyword_match?(match)
+    match.keyword_coverage >= MIN_ACCEPTABLE_KEYWORD_COVERAGE
+  end
+end

enterprise/app/services/captain/llm/documentation_sufficiency_service.rb

@@ -0,0 +1,216 @@
+class Captain::Llm::DocumentationSufficiencyService < Llm::BaseAiService
+  include Integrations::LlmInstrumentation
+
+  MAX_CONTEXT_MESSAGES = 6
+  MAX_SEARCHES = 3
+  MAX_MATCHES_PER_SEARCH = 5
+  MAX_ANSWER_CHARS = 700
+
+  def initialize(assistant:, conversation:)
+    super()
+    @assistant = assistant
+    @conversation = conversation
+    @temperature = 0.0
+  end
+
+  def evaluate(message_history:, assistant_response:, documentation_searches:)
+    user_prompt = inspection_user_prompt(
+      message_history: message_history,
+      assistant_response: assistant_response,
+      documentation_searches: documentation_searches
+    )
+
+    response = instrument_llm_call(instrumentation_params(user_prompt, documentation_searches)) do
+      chat(model: @model, temperature: @temperature)
+        .with_schema(Captain::DocumentationSufficiencySchema)
+        .with_instructions(system_prompt)
+        .ask(user_prompt)
+    end
+
+    parsed = parse_response(response.content)
+    normalize_response(parsed, response.content)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][DocumentationSufficiency] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+    { 'decision' => nil, 'reason' => nil, 'error' => e.message, 'model' => @model }
+  end
+
+  private
+
+  def system_prompt
+    <<~PROMPT
+      You are checking whether a customer-facing assistant response is supported by retrieved documentation.
+
+      Use only the conversation context, assistant response, and retrieved documentation search results provided.
+      Do not use outside knowledge.
+
+      Return "insufficient" when the assistant makes factual claims that are not supported by the retrieved documentation.
+      Treat prior assistant messages as claims, not evidence. They do not support the new answer by themselves.
+      Conversation context can support the answer only when the user explicitly provided the relevant fact, constraint, or artifact.
+      Check generic sufficiency dimensions:
+      - same entity, product, platform, integration, or account object
+      - same user intent, not just a nearby topic
+      - requested constraints such as plan, edition, region, channel, version, provider, billing period, availability, or current status
+      - high-risk claims such as pricing, billing, legal/compliance, limits, availability, platform support, roadmap, or account status
+      - evidence specificity; generic broad docs are not enough for specific claims
+
+      Return "sufficient" when the documentation directly supports the response, or when the response only asks a clarifying
+      question, gives a safe bounded no-answer, offers handoff, or restates user-provided context without adding external claims.
+      If documentation is missing or weak and the response gives factual claims, advice, instructions, examples, links, prices,
+      limits, availability, troubleshooting steps, product behavior, platform behavior, or account-specific statements, return "insufficient".
+
+      If decision is "insufficient", write fallback_response in the user's language. It should be brief, say you could not find
+      enough information to answer confidently, and ask whether the user wants to talk to a support person when appropriate.
+      If decision is not "insufficient", fallback_response must be empty.
+    PROMPT
+  end
+
+  def inspection_user_prompt(message_history:, assistant_response:, documentation_searches:)
+    <<~PROMPT
+      <conversation_context>
+      #{format_conversation_context(message_history)}
+      </conversation_context>
+
+      <retrieved_documentation>
+      #{format_documentation_searches(documentation_searches)}
+      </retrieved_documentation>
+
+      <assistant_response>
+      #{assistant_response}
+      </assistant_response>
+    PROMPT
+  end
+
+  def format_documentation_searches(searches)
+    searches.to_a.last(MAX_SEARCHES).map.with_index(1) do |search, index|
+      matches = search[:matches] || search['matches'] || []
+      <<~SEARCH
+        Search #{index}
+        query: #{search[:query] || search['query']}
+        status: #{search[:status] || search['status']}
+        reason: #{search[:reason] || search['reason']}
+        matches:
+        #{format_documentation_matches(matches)}
+      SEARCH
+    end.join("\n")
+  end
+
+  def format_documentation_matches(matches)
+    matches.to_a.first(MAX_MATCHES_PER_SEARCH).map.with_index(1) do |match, index|
+      <<~MATCH
+        #{index}. question: #{match_value(match, :question)}
+           answer: #{truncate_text(match_value(match, :answer))}
+           source: #{match_value(match, :source)}
+           semantic_distance: #{match_value(match, :semantic_distance)}
+           keyword_coverage: #{match_value(match, :keyword_coverage)}
+           retrieval_methods: #{Array(match_value(match, :retrieval_methods)).join(', ')}
+      MATCH
+    end.join("\n")
+  end
+
+  def match_value(match, key) = match[key] || match[key.to_s]
+
+  def normalize_messages(message_history)
+    message_history.filter_map do |message|
+      role = message[:role] || message['role']
+      next if role.blank?
+
+      { role: role.to_s, content: normalize_content(message[:content] || message['content']) }
+    end
+  end
+
+  def normalize_content(content)
+    return content if content.is_a?(String)
+    return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
+
+    content.to_s
+  end
+
+  def text_part?(part)
+    return false unless part.is_a?(Hash)
+
+    (part[:type] || part['type']).to_s == 'text'
+  end
+
+  def format_conversation_context(messages)
+    normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+      content = message[:content].to_s.strip
+      next if content.blank?
+
+      "#{role_label(message[:role])}: #{content}"
+    end.join("\n")
+  end
+
+  def role_label(role) = { 'user' => 'User', 'assistant' => 'Assistant' }.fetch(role, role.to_s.titleize)
+
+  def parse_response(content)
+    return content if content.is_a?(Hash)
+
+    JSON.parse(sanitize_json_response(content))
+  rescue JSON::ParserError, TypeError
+    {}
+  end
+
+  def normalize_response(parsed, raw_content)
+    decision = parsed['decision'].to_s
+    reason = parsed['reason'].to_s
+    return invalid_response(raw_content) unless Captain::DocumentationSufficiencySchema::DECISIONS.include?(decision)
+
+    {
+      'decision' => decision,
+      'reason' => reason.presence,
+      'fallback_response' => parsed['fallback_response'].to_s,
+      'raw_response' => raw_content,
+      'model' => @model
+    }
+  end
+
+  def invalid_response(raw_content)
+    {
+      'decision' => nil,
+      'reason' => nil,
+      'fallback_response' => nil,
+      'raw_response' => raw_content,
+      'error' => 'invalid_documentation_sufficiency_response',
+      'model' => @model
+    }
+  end
+
+  def instrumentation_params(user_prompt, documentation_searches)
+    {
+      span_name: 'llm.captain.documentation_sufficiency',
+      model: @model,
+      temperature: @temperature,
+      account_id: @conversation.account_id,
+      conversation_id: @conversation.display_id,
+      feature_name: 'documentation_sufficiency',
+      messages: [
+        { role: 'system', content: system_prompt },
+        { role: 'user', content: user_prompt }
+      ],
+      metadata: {
+        assistant_id: @assistant.id,
+        channel_type: @conversation.inbox&.channel_type,
+        source: 'response_builder'
+      }.merge(search_metadata(documentation_searches))
+    }
+  end
+
+  def search_metadata(documentation_searches)
+    searches = documentation_searches.to_a
+    {
+      search_count: searches.length,
+      search_statuses: searches.filter_map { |search| search[:status] || search['status'] }.join(','),
+      search_reasons: searches.filter_map { |search| search[:reason] || search['reason'] }.join(',')
+    }
+  end
+
+  def truncate_text(text)
+    text = text.to_s
+    return text if text.length <= MAX_ANSWER_CHARS
+
+    "#{text.first(MAX_ANSWER_CHARS)}..."
+  end
+end

enterprise/app/services/captain/llm/system_prompts_service.rb

@@ -190,6 +190,8 @@ class Captain::Llm::SystemPromptsService
 
         [Error Handling]
         - If the required information is not found in the provided context, respond with an appropriate message indicating that no relevant data is available.
+        - If a tool response says search quality is weak, do not use that tool response to make factual claims.
+          Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or offer a human handoff.
         - Avoid speculating or providing unverified information.
 
         [Available Actions]
@@ -262,6 +264,7 @@ class Captain::Llm::SystemPromptsService
         }
         ```
         - If the answer is not provided in context sections, Respond to the customer and ask whether they want to talk to another support agent . If they ask to Chat with another agent, return `conversation_handoff' as the response in JSON response
+        - If a tool response says search quality is weak, do not use that tool response to make factual claims. Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or offer a human handoff.
         #{'- You MUST provide numbered citations at the appropriate places in the text.' if config['feature_citation']}
 
         #{build_tools_section(custom_tools)}

enterprise/app/services/captain/tools/search_documentation_service.rb

@@ -13,26 +13,12 @@ class Captain::Tools::SearchDocumentationService < Captain::Tools::BaseTool
                        .new(account: assistant.account)
                        .translate(query, target_language: assistant.account.locale_english_name)
 
-    responses = assistant.responses.approved.search(translated_query)
+    result = Captain::DocumentationSearchService.new(
+      scope: assistant.responses.approved,
+      account_id: assistant.account_id
+    ).search(translated_query)
+    Captain::DocumentationSearchService.record(result)
 
-    return 'No FAQs found for the given query' if responses.empty?
-
-    responses.map { |response| format_response(response) }.join
-  end
-
-  private
-
-  def format_response(response)
-    formatted_response = "
-        Question: #{response.question}
-        Answer: #{response.answer}
-        "
-    if response.documentable.present? && response.documentable.try(:external_link)
-      formatted_response += "
-          Source: #{response.documentable.external_link}
-          "
-    end
-
-    formatted_response
+    Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
   end
 end

enterprise/app/services/captain/tools/search_reply_documentation_service.rb

@@ -22,25 +22,20 @@ class Captain::Tools::SearchReplyDocumentationService < RubyLLM::Tool
                        .new(account: @account)
                        .translate(query, target_language: @account.locale_english_name)
 
-    responses = search_responses(translated_query)
-    return 'No FAQs found for the given query' if responses.empty?
+    result = Captain::DocumentationSearchService.new(
+      scope: search_scope,
+      account_id: @account.id
+    ).search(translated_query)
+    Captain::DocumentationSearchService.record(result)
 
-    responses.map { |response| format_response(response) }.join
+    Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
   end
 
   private
 
-  def search_responses(query)
-    if @assistant.present?
-      @assistant.responses.approved.search(query, account_id: @account.id)
-    else
-      @account.captain_assistant_responses.approved.search(query, account_id: @account.id)
-    end
-  end
+  def search_scope
+    return @assistant.responses.approved if @assistant.present?
 
-  def format_response(response)
-    result = "\nQuestion: #{response.question}\nAnswer: #{response.answer}\n"
-    result += "Source: #{response.documentable.external_link}\n" if response.documentable.present? && response.documentable.try(:external_link)
-    result
+    @account.captain_assistant_responses.approved
   end
 end

enterprise/lib/captain/documentation_sufficiency_schema.rb

@@ -0,0 +1,20 @@
+class Captain::DocumentationSufficiencySchema < RubyLLM::Schema
+  DECISIONS = %w[sufficient insufficient].freeze
+  REASONS = %w[
+    answers_exact_question
+    no_documentation_used
+    bounded_no_answer
+    missing_or_weak_evidence
+    wrong_entity
+    wrong_intent
+    missing_constraint
+    generic_evidence
+    unsupported_high_risk_claim
+  ].freeze
+
+  string :decision,
+         enum: DECISIONS,
+         description: 'Use insufficient for unsupported factual answers; use sufficient only when supported or safely bounded'
+  string :reason, enum: REASONS, description: 'The main reason for the decision'
+  string :fallback_response, description: 'If insufficient, a brief user-facing fallback in the user language; otherwise empty'
+end

enterprise/lib/captain/prompts/assistant.liquid

@@ -11,6 +11,7 @@ Don't digress away from your instructions, and use all the available tools at yo
 # Core Rules
 - Do not use your own understanding or training data to provide answers. Base responses strictly on the information available through your tools and provided context.
 - Do not share anything outside of the context provided.
+- If FAQ lookup says search quality is weak, do not use that result to make factual claims. Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or use handoff.
 - Be concise and relevant: most of your responses should be a sentence or two, unless a more detailed explanation is necessary.
 - Always detect the language from the user's input and reply in the same language.
 - When there is ambiguity, ask clarifying questions rather than make assumptions.

enterprise/lib/captain/tools/faq_lookup_tool.rb

@@ -5,44 +5,18 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool
   def perform(_tool_context, query:)
     log_tool_usage('searching', { query: query })
 
-    # Use existing vector search on approved responses
-    responses = @assistant.responses.approved.search(query).to_a
+    result = Captain::DocumentationSearchService.new(
+      scope: @assistant.responses.approved,
+      account_id: @assistant.account_id
+    ).search(query)
+    Captain::DocumentationSearchService.record(result)
 
-    if responses.empty?
+    if result.empty?
       log_tool_usage('no_results', { query: query })
-      "No relevant FAQs found for: #{query}"
     else
-      log_tool_usage('found_results', { query: query, count: responses.size })
-      format_responses(responses)
-    end
-  end
-
-  private
-
-  def format_responses(responses)
-    responses.map { |response| format_response(response) }.join
-  end
-
-  def format_response(response)
-    formatted_response = "
-        Question: #{response.question}
-        Answer: #{response.answer}
-        "
-    if should_show_source?(response)
-      formatted_response += "
-          Source: #{response.documentable.external_link}
-          "
+      log_tool_usage('found_results', { query: query, count: result.matches.size, status: result.status, reason: result.reason })
     end
 
-    formatted_response
-  end
-
-  def should_show_source?(response)
-    return false if response.documentable.blank?
-    return false unless response.documentable.try(:external_link)
-
-    # Don't show source if it's a PDF placeholder
-    external_link = response.documentable.external_link
-    !external_link.start_with?('PDF:')
+    Captain::DocumentationSearchService.format_for_tool(result, no_results_message: "No relevant FAQs found for: #{query}")
   end
 end

lib/current.rb

@@ -4,6 +4,7 @@ module Current
   thread_mattr_accessor :account_user
   thread_mattr_accessor :executed_by
   thread_mattr_accessor :contact
+  thread_mattr_accessor :captain_documentation_searches
 
   def self.reset
     Current.user = nil
@@ -11,5 +12,6 @@ module Current
     Current.account_user = nil
     Current.executed_by = nil
     Current.contact = nil
+    Current.captain_documentation_searches = nil
   end
 end

spec/enterprise/controllers/api/v1/accounts/captain/assistants_controller_spec.rb

@@ -216,6 +216,16 @@ RSpec.describe 'Api::V1::Accounts::Captain::Assistants', type: :request do
         expect(response).to have_http_status(:success)
         expect(json_response[:config][:feature_citation]).to be(false)
       end
+
+      it 'updates documentation sufficiency gate config' do
+        patch "/api/v1/accounts/#{account.id}/captain/assistants/#{assistant.id}",
+              params: { assistant: { config: { documentation_sufficiency_gate_enabled: true } } },
+              headers: admin.create_new_auth_token,
+              as: :json
+
+        expect(response).to have_http_status(:success)
+        expect(json_response[:config][:documentation_sufficiency_gate_enabled]).to be(true)
+      end
     end
   end
 

spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb

@@ -11,6 +11,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
     let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
     let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
     let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
+    let(:mock_documentation_sufficiency_service) { instance_double(Captain::Llm::DocumentationSufficiencyService) }
 
     before do
       create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -152,6 +153,94 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
         end
       end
 
+      context 'when documentation sufficiency gate is enabled' do
+        let(:weak_search_result) do
+          Captain::DocumentationSearchService::Result.new(
+            query: 'current plan limits',
+            queries: ['current plan limits'],
+            matches: [],
+            status: 'weak',
+            reason: 'no_results'
+          )
+        end
+
+        before do
+          assistant.update!(config: { 'documentation_sufficiency_gate_enabled' => true })
+          allow(Captain::Llm::DocumentationSufficiencyService).to receive(:new).with(
+            assistant: assistant,
+            conversation: conversation
+          ).and_return(mock_documentation_sufficiency_service)
+        end
+
+        it 'replaces an unsupported answer with the bounded fallback from the gate' do
+          allow(mock_llm_chat_service).to receive(:generate_response) do
+            Captain::DocumentationSearchService.record(weak_search_result)
+            { 'response' => 'Your current plan has unlimited usage.' }
+          end
+          allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
+            {
+              'decision' => 'insufficient',
+              'reason' => 'missing_constraint',
+              'fallback_response' => "I couldn't find enough information to answer that confidently. Would you like support?",
+              'model' => 'gpt-4.1'
+            }
+          )
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq(
+            "I couldn't find enough information to answer that confidently. Would you like support?"
+          )
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'keeps the assistant response when the gate finds enough support' do
+          allow(mock_llm_chat_service).to receive(:generate_response) do
+            Captain::DocumentationSearchService.record(weak_search_result)
+            { 'response' => 'Billing settings show current plan usage.' }
+          end
+          allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
+            { 'decision' => 'sufficient', 'reason' => 'answers_exact_question', 'fallback_response' => '', 'model' => 'gpt-4.1' }
+          )
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq('Billing settings show current plan usage.')
+        end
+
+        it 'checks high-risk answers even when no documentation search was recorded' do
+          allow(mock_llm_chat_service).to receive(:generate_response).and_return(
+            { 'response' => 'Your current plan costs $99 per agent per month.' }
+          )
+          expect(mock_documentation_sufficiency_service).to receive(:evaluate).with(
+            message_history: [{ content: 'Hello', role: 'user' }],
+            assistant_response: 'Your current plan costs $99 per agent per month.',
+            documentation_searches: [
+              {
+                query: 'Hello',
+                queries: ['Hello'],
+                status: 'weak',
+                reason: 'no_documentation_search',
+                matches: []
+              }
+            ]
+          ).and_return(
+            {
+              'decision' => 'insufficient',
+              'reason' => 'unsupported_high_risk_claim',
+              'fallback_response' => "I couldn't find enough information to answer that confidently. Would you like support?",
+              'model' => 'gpt-4.1'
+            }
+          )
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.last.content).to eq(
+            "I couldn't find enough information to answer that confidently. Would you like support?"
+          )
+        end
+      end
+
       it 'does not send a response when the conversation is no longer pending' do
         conversation.open!
 

spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb

@@ -5,15 +5,12 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
   let(:assistant) { create(:captain_assistant, account: account) }
   let(:tool) { described_class.new(assistant) }
   let(:tool_context) { Struct.new(:state).new({}) }
+  let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
 
   before do
-    # Create installation config for OpenAI API key to avoid errors
-    create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
-
-    # Mock embedding service to avoid actual API calls
-    embedding_service = instance_double(Captain::Llm::EmbeddingService)
-    allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
-    allow(embedding_service).to receive(:get_embedding).and_return(Array.new(1536, 0.1))
+    allow(Captain::DocumentationSearchService).to receive(:new)
+      .with(scope: anything, account_id: assistant.account_id)
+      .and_return(documentation_search_service)
   end
 
   describe '#description' do
@@ -37,6 +34,7 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
       let!(:response1) do
         create(:captain_assistant_response,
                assistant: assistant,
+               account: account,
                question: 'How to reset password?',
                answer: 'Click on forgot password link',
                documentable: document,
@@ -45,16 +43,31 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
       let!(:response2) do
         create(:captain_assistant_response,
                assistant: assistant,
+               account: account,
                question: 'How to change email?',
                answer: 'Go to settings and update email',
                status: 'approved')
       end
 
       before do
-        # Mock nearest_neighbors to return our test responses
-        allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(
-          Captain::AssistantResponse.where(id: [response1.id, response2.id])
+        matches = [response1, response2].map do |response|
+          Captain::AssistantResponse::SearchMatch.new(
+            response: response,
+            semantic_distance: 0.2,
+            keyword_score: 0,
+            keyword_coverage: 0.0,
+            matched_terms: [],
+            retrieval_methods: ['semantic']
+          )
+        end
+        search_result = Captain::DocumentationSearchService::Result.new(
+          query: 'password reset',
+          queries: ['password reset'],
+          matches: matches,
+          status: 'sufficient',
+          reason: 'semantic_match'
         )
+        allow(documentation_search_service).to receive(:search).and_return(search_result)
       end
 
       it 'searches FAQs and returns formatted responses' do
@@ -76,7 +89,10 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
 
       it 'logs tool usage for search' do
         expect(tool).to receive(:log_tool_usage).with('searching', { query: 'password reset' })
-        expect(tool).to receive(:log_tool_usage).with('found_results', { query: 'password reset', count: 2 })
+        expect(tool).to receive(:log_tool_usage).with(
+          'found_results',
+          { query: 'password reset', count: 2, status: 'sufficient', reason: 'semantic_match' }
+        )
 
         tool.perform(tool_context, query: 'password reset')
       end
@@ -84,13 +100,20 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
 
     context 'when no FAQs found' do
       before do
-        # Return empty result set
-        allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
+        search_result = Captain::DocumentationSearchService::Result.new(
+          query: 'nonexistent topic',
+          queries: ['nonexistent topic'],
+          matches: [],
+          status: 'weak',
+          reason: 'no_results'
+        )
+        allow(documentation_search_service).to receive(:search).and_return(search_result)
       end
 
       it 'returns no results message' do
         result = tool.perform(tool_context, query: 'nonexistent topic')
-        expect(result).to eq('No relevant FAQs found for: nonexistent topic')
+        expect(result).to include('No relevant FAQs found for: nonexistent topic')
+        expect(result).to include('Do not use it to make factual claims')
       end
 
       it 'logs tool usage for no results' do
@@ -103,11 +126,17 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
 
     context 'with blank query' do
       it 'handles empty query' do
-        # Return empty result set
-        allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
+        search_result = Captain::DocumentationSearchService::Result.new(
+          query: '',
+          queries: [],
+          matches: [],
+          status: 'weak',
+          reason: 'no_results'
+        )
+        allow(documentation_search_service).to receive(:search).and_return(search_result)
 
         result = tool.perform(tool_context, query: '')
-        expect(result).to eq('No relevant FAQs found for: ')
+        expect(result).to include('No relevant FAQs found for: ')
       end
     end
   end

spec/enterprise/services/captain/documentation_search_service_spec.rb

@@ -0,0 +1,102 @@
+require 'rails_helper'
+
+RSpec.describe Captain::DocumentationSearchService do
+  let(:scope_class) do
+    Class.new do
+      def search_with_metadata(*)
+        []
+      end
+    end
+  end
+  let(:scope) { scope_class.new }
+  let(:service) { described_class.new(scope: scope, account_id: 1) }
+  let(:response) do
+    instance_double(
+      Captain::AssistantResponse,
+      id: 1,
+      question: 'How do plan limits work?',
+      answer: 'Monthly limits are shown in billing settings.',
+      documentable: nil
+    )
+  end
+
+  def search_match(semantic_distance:, keyword_coverage:, keyword_score: 0, response_record: response)
+    Captain::AssistantResponse::SearchMatch.new(
+      response: response_record,
+      semantic_distance: semantic_distance,
+      keyword_score: keyword_score,
+      keyword_coverage: keyword_coverage,
+      matched_terms: [],
+      retrieval_methods: ['semantic']
+    )
+  end
+
+  describe '#search' do
+    it 'retries with generic query variants when the original query has weak matches' do
+      query = 'How do I check limits for my current monthly plan?'
+      weak_match = search_match(semantic_distance: 0.9, keyword_coverage: 0.0)
+      sufficient_match = search_match(semantic_distance: 0.8, keyword_coverage: 0.4, keyword_score: 2)
+
+      allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([weak_match])
+      allow(scope).to receive(:search_with_metadata)
+        .with('check limits current monthly plan', account_id: 1)
+        .and_return([sufficient_match])
+
+      result = service.search(query)
+
+      expect(result.status).to eq('sufficient')
+      expect(result.reason).to eq('keyword_match')
+      expect(result.queries).to eq([query, 'check limits current monthly plan'])
+      expect(result.matches.first.keyword_coverage).to eq(0.4)
+    end
+
+    it 'stops after the first query when retrieval confidence is sufficient' do
+      query = 'Where do I find billing settings?'
+      sufficient_match = search_match(semantic_distance: 0.2, keyword_coverage: 0.0)
+
+      allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([sufficient_match])
+
+      result = service.search(query)
+
+      expect(result.status).to eq('sufficient')
+      expect(result.reason).to eq('semantic_match')
+      expect(result.queries).to eq([query])
+    end
+  end
+
+  describe '.format_for_tool' do
+    it 'adds a bounded-answer instruction when no documentation is found' do
+      result = described_class::Result.new(
+        query: 'unknown topic',
+        queries: ['unknown topic'],
+        matches: [],
+        status: 'weak',
+        reason: 'no_results'
+      )
+
+      formatted_result = described_class.format_for_tool(result, no_results_message: 'No FAQs found')
+
+      expect(formatted_result).to include('No FAQs found')
+      expect(formatted_result).to include('Do not use it to make factual claims')
+    end
+  end
+
+  describe '.record' do
+    it 'stores search metadata on Current for the response-level sufficiency gate' do
+      result = described_class::Result.new(
+        query: 'billing',
+        queries: ['billing'],
+        matches: [search_match(semantic_distance: 0.2, keyword_coverage: 0.0)],
+        status: 'sufficient',
+        reason: 'semantic_match'
+      )
+
+      described_class.record(result)
+
+      expect(Current.captain_documentation_searches.first[:status]).to eq('sufficient')
+      expect(Current.captain_documentation_searches.first[:matches].first[:semantic_distance]).to eq(0.2)
+    ensure
+      Current.captain_documentation_searches = nil
+    end
+  end
+end

spec/enterprise/services/captain/tools/search_documentation_service_spec.rb

@@ -26,10 +26,13 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
   end
 
   describe '#execute' do
+    let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
+    let(:translate_query_service) { instance_double(Captain::Llm::TranslateQueryService) }
     let!(:response) do
       create(
         :captain_assistant_response,
         assistant: assistant,
+        account: assistant.account,
         question: question,
         answer: answer,
         status: 'approved'
@@ -37,14 +40,37 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
     end
 
     let(:documentable) { create(:captain_document, external_link: external_link) }
+    let(:match) do
+      Captain::AssistantResponse::SearchMatch.new(
+        response: response,
+        semantic_distance: 0.2,
+        keyword_score: 0,
+        keyword_coverage: 0.0,
+        matched_terms: [],
+        retrieval_methods: ['semantic']
+      )
+    end
+
+    before do
+      allow(Captain::Llm::TranslateQueryService).to receive(:new).and_return(translate_query_service)
+      allow(translate_query_service).to receive(:translate).and_return(question)
+      allow(Captain::DocumentationSearchService).to receive(:new)
+        .with(scope: anything, account_id: assistant.account_id)
+        .and_return(documentation_search_service)
+    end
 
     context 'when matching responses exist' do
-      before do
-        response.update(documentable: documentable)
-        allow(Captain::AssistantResponse).to receive(:search).with(question).and_return([response])
-      end
-
       it 'returns formatted responses for the search query' do
+        response.update(documentable: documentable)
+        search_result = Captain::DocumentationSearchService::Result.new(
+          query: question,
+          queries: [question],
+          matches: [match],
+          status: 'sufficient',
+          reason: 'semantic_match'
+        )
+        allow(documentation_search_service).to receive(:search).with(question).and_return(search_result)
+
         result = service.execute(query: question)
 
         expect(result).to include(question)
@@ -54,12 +80,20 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
     end
 
     context 'when no matching responses exist' do
-      before do
-        allow(Captain::AssistantResponse).to receive(:search).with(question).and_return([])
-      end
+      it 'returns a bounded no-results instruction' do
+        search_result = Captain::DocumentationSearchService::Result.new(
+          query: question,
+          queries: [question],
+          matches: [],
+          status: 'weak',
+          reason: 'no_results'
+        )
+        allow(documentation_search_service).to receive(:search).with(question).and_return(search_result)
 
-      it 'returns an empty string' do
-        expect(service.execute(query: question)).to eq('No FAQs found for the given query')
+        result = service.execute(query: question)
+
+        expect(result).to include('No FAQs found for the given query')
+        expect(result).to include('Do not use it to make factual claims')
       end
     end
   end