diff --git a/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb b/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb
index df9dfe5cc..dc13b1c7f 100644
--- a/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb
+++ b/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb
@@ -58,6 +58,7 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base
config: [
:product_name, :feature_faq, :feature_memory, :feature_citation,
:feature_contact_attributes,
+ :documentation_sufficiency_gate_enabled,
:welcome_message, :handoff_message, :resolution_message,
:instructions, :temperature
])
diff --git a/enterprise/app/jobs/captain/conversation/documentation_sufficiency_handler.rb b/enterprise/app/jobs/captain/conversation/documentation_sufficiency_handler.rb
new file mode 100644
index 000000000..c47a1835d
--- /dev/null
+++ b/enterprise/app/jobs/captain/conversation/documentation_sufficiency_handler.rb
@@ -0,0 +1,110 @@
+module Captain::Conversation::DocumentationSufficiencyHandler
+ HIGH_RISK_TERMS = %w[
+ price pricing cost billing bill paid free plan subscription legal compliance policy limit limits maximum minimum
+ available availability current currently roadmap beta early access supported support self-hosted cloud region version
+ provider integration account status
+ ].freeze
+
+ private
+
+ def reset_documentation_searches
+ Current.captain_documentation_searches = []
+ end
+
+ def clear_documentation_searches
+ Current.captain_documentation_searches = nil
+ end
+
+ def inspect_documentation_sufficiency(message_history)
+ searches = documentation_searches_for_inspection(message_history)
+ return unless should_inspect_documentation_sufficiency?(message_history, searches)
+
+ apply_documentation_sufficiency_inspection(documentation_sufficiency_inspection(message_history, searches))
+ rescue StandardError => e
+ ChatwootExceptionTracker.new(e, account: account).capture_exception
+ Rails.logger.warn(
+ "[CAPTAIN][ResponseBuilderJob] Documentation sufficiency check failed for account=#{account.id} " \
+ "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
+ )
+ end
+
+ def should_inspect_documentation_sufficiency?(message_history, searches)
+ return false unless documentation_sufficiency_gate_enabled?
+ return false unless response_has_user_facing_answer?
+ return false if searches.empty?
+
+ documentation_sufficiency_check_needed?(message_history, searches)
+ end
+
+ def response_has_user_facing_answer?
+ @response.present? &&
+ @response['response'].present? &&
+ @response['response'] != 'conversation_handoff' &&
+ !@response['handoff_tool_called']
+ end
+
+ def documentation_searches_for_inspection(message_history)
+ searches = Current.captain_documentation_searches.to_a
+ return searches if searches.present?
+ return [] unless high_risk_conversation?(message_history)
+
+ [synthetic_no_results_search(message_history)]
+ end
+
+ def synthetic_no_results_search(message_history)
+ {
+ query: last_user_message_content(message_history),
+ queries: [last_user_message_content(message_history)],
+ status: 'weak',
+ reason: 'no_documentation_search',
+ matches: []
+ }
+ end
+
+ def documentation_sufficiency_inspection(message_history, searches)
+ Captain::Llm::DocumentationSufficiencyService.new(
+ assistant: @assistant,
+ conversation: @conversation
+ ).evaluate(
+ message_history: message_history,
+ assistant_response: @response['response'],
+ documentation_searches: searches
+ )
+ end
+
+ def documentation_sufficiency_gate_enabled?
+ ActiveModel::Type::Boolean.new.cast(@assistant.config['documentation_sufficiency_gate_enabled'])
+ end
+
+ def documentation_sufficiency_check_needed?(message_history, searches)
+ searches.any? { |search| search[:status] == 'weak' || search['status'] == 'weak' } || high_risk_conversation?(message_history)
+ end
+
+ def high_risk_conversation?(message_history)
+ text = "#{message_history.last(3).map { |message| message[:content] || message['content'] }.join(' ')} #{@response&.dig('response')}".downcase
+ HIGH_RISK_TERMS.any? { |term| text.include?(term) }
+ end
+
+ def last_user_message_content(message_history)
+ user_message = message_history.reverse.find { |message| (message[:role] || message['role']).to_s == 'user' }
+ user_message && (user_message[:content] || user_message['content']).to_s
+ end
+
+ def apply_documentation_sufficiency_inspection(inspection)
+ return unless inspection['decision'] == 'insufficient'
+
+ fallback_response = inspection['fallback_response'].presence || default_documentation_sufficiency_fallback
+ @response.merge!(
+ 'response' => fallback_response,
+ 'action' => 'continue',
+ 'action_reason' => 'missing_docs_bounded_answer',
+ 'action_source' => 'documentation_sufficiency',
+ 'documentation_sufficiency_reason' => inspection['reason'],
+ 'documentation_sufficiency_model' => inspection['model']
+ )
+ end
+
+ def default_documentation_sufficiency_fallback
+ "I couldn't find enough information to answer that confidently. Would you like me to connect you with a support person?"
+ end
+end
diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
index 5050f11b2..f8f65a1ed 100644
--- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb
+++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
@@ -1,5 +1,6 @@
class Captain::Conversation::ResponseBuilderJob < ApplicationJob
include Captain::Conversation::V1ActionClassifier
+ include Captain::Conversation::DocumentationSufficiencyHandler
MAX_MESSAGE_LENGTH = 10_000
retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
@@ -25,6 +26,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
rescue StandardError => e
handle_error(e)
ensure
+ clear_documentation_searches
Current.executed_by = nil
end
@@ -34,17 +36,22 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
def generate_and_process_response
message_history = collect_previous_messages
+ reset_documentation_searches
@response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
message_history: message_history
)
+ inspect_documentation_sufficiency(message_history) if conversation_pending?
classify_v1_response_action(message_history) if conversation_pending?
process_response
end
def generate_response_with_v2
+ message_history = collect_previous_messages
+ reset_documentation_searches
@response = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation).generate_response(
- message_history: collect_previous_messages
+ message_history: message_history
)
+ inspect_documentation_sufficiency(message_history) if conversation_pending?
process_response
end
diff --git a/enterprise/app/models/captain/assistant_response.rb b/enterprise/app/models/captain/assistant_response.rb
index db2ac058a..0af5c41d5 100644
--- a/enterprise/app/models/captain/assistant_response.rb
+++ b/enterprise/app/models/captain/assistant_response.rb
@@ -25,6 +25,35 @@
#
class Captain::AssistantResponse < ApplicationRecord
self.table_name = 'captain_assistant_responses'
+ SEARCH_LIMIT = 5
+ KEYWORD_SEARCH_LIMIT_MULTIPLIER = 2
+ SEARCH_STOP_WORDS = %w[
+ about after all also and any are but can for from has have how into its may more not now off our out
+ the their them then there these they this was what when where which who why with you your
+ ].freeze
+ SearchMatch = Struct.new(
+ :response,
+ :semantic_distance,
+ :keyword_score,
+ :keyword_coverage,
+ :matched_terms,
+ :retrieval_methods,
+ keyword_init: true
+ ) do
+ def to_h
+ {
+ response_id: response.id,
+ question: response.question,
+ answer: response.answer,
+ source: response.documentable&.try(:external_link),
+ semantic_distance: semantic_distance,
+ keyword_score: keyword_score,
+ keyword_coverage: keyword_coverage,
+ matched_terms: matched_terms,
+ retrieval_methods: retrieval_methods
+ }
+ end
+ end
belongs_to :assistant, class_name: 'Captain::Assistant'
belongs_to :account
@@ -47,8 +76,105 @@ class Captain::AssistantResponse < ApplicationRecord
enum status: { pending: 0, approved: 1 }
def self.search(query, account_id: nil)
+ search_with_metadata(query, account_id: account_id).map(&:response)
+ end
+
+ def self.search_with_metadata(query, account_id: nil, limit: SEARCH_LIMIT)
+ semantic_matches = semantic_search_matches(query, account_id: account_id, limit: limit)
+ keyword_matches = keyword_search_matches(query, limit: limit * KEYWORD_SEARCH_LIMIT_MULTIPLIER)
+
+ merge_search_matches(semantic_matches, keyword_matches).first(limit)
+ end
+
+ def self.semantic_search_matches(query, account_id:, limit:)
embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(query)
- nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(5)
+ nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(limit).map do |response|
+ SearchMatch.new(
+ response: response,
+ semantic_distance: response.neighbor_distance&.to_f,
+ keyword_score: 0,
+ keyword_coverage: 0.0,
+ matched_terms: [],
+ retrieval_methods: ['semantic']
+ )
+ end
+ end
+
+ def self.keyword_search_matches(query, limit:)
+ terms = search_terms(query)
+ return [] if terms.empty?
+
+ matches = keyword_search_scope(terms).limit(limit).map do |response|
+ matched_terms = matched_terms_for(response, terms)
+ SearchMatch.new(
+ response: response,
+ semantic_distance: nil,
+ keyword_score: matched_terms.size,
+ keyword_coverage: matched_terms.size.to_f / terms.size,
+ matched_terms: matched_terms,
+ retrieval_methods: ['keyword']
+ )
+ end
+ matches.sort_by { |match| [-match.keyword_score, match.response.id] }
+ end
+
+ def self.keyword_search_scope(terms)
+ conditions = []
+ values = []
+
+ terms.each do |term|
+ pattern = "%#{sanitize_sql_like(term)}%"
+ conditions << '(question ILIKE ? OR answer ILIKE ?)'
+ values.push(pattern, pattern)
+ end
+
+ where(conditions.join(' OR '), *values)
+ end
+
+ def self.search_terms(query)
+ query.to_s.downcase.scan(/[[:alnum:]]+/).filter_map do |term|
+ next if term.length < 3
+ next if SEARCH_STOP_WORDS.include?(term)
+
+ term
+ end.uniq
+ end
+
+ def self.matched_terms_for(response, terms)
+ text = "#{response.question} #{response.answer}".downcase
+ terms.select { |term| text.include?(term) }
+ end
+
+ def self.merge_search_matches(semantic_matches, keyword_matches)
+ matches_by_response_id = {}
+
+ semantic_matches.each do |match|
+ matches_by_response_id[match.response.id] = match
+ end
+
+ keyword_matches.each do |keyword_match|
+ existing = matches_by_response_id[keyword_match.response.id]
+ if existing
+ merge_keyword_match!(existing, keyword_match)
+ else
+ matches_by_response_id[keyword_match.response.id] = keyword_match
+ end
+ end
+
+ sort_search_matches(matches_by_response_id.values)
+ end
+
+ def self.merge_keyword_match!(existing, keyword_match)
+ existing.keyword_score = keyword_match.keyword_score
+ existing.keyword_coverage = keyword_match.keyword_coverage
+ existing.matched_terms = keyword_match.matched_terms
+ existing.retrieval_methods |= keyword_match.retrieval_methods
+ end
+
+ def self.sort_search_matches(matches)
+ matches.sort_by do |match|
+ [match.semantic_distance || 1.0, -match.keyword_score, match.response.id]
+ end
end
private
diff --git a/enterprise/app/services/captain/documentation_search_service.rb b/enterprise/app/services/captain/documentation_search_service.rb
new file mode 100644
index 000000000..ff2827e32
--- /dev/null
+++ b/enterprise/app/services/captain/documentation_search_service.rb
@@ -0,0 +1,155 @@
+class Captain::DocumentationSearchService
+ MAX_ACCEPTABLE_COSINE_DISTANCE = 0.45
+ MIN_ACCEPTABLE_KEYWORD_COVERAGE = 0.4
+ TOP_MATCHES_TO_FORMAT = 5
+ SEARCH_ATTEMPT_LIMIT = 3
+
+ Result = Struct.new(:query, :queries, :matches, :status, :reason, keyword_init: true) do
+ def weak?
+ status == 'weak'
+ end
+
+ def empty?
+ matches.empty?
+ end
+
+ def to_h
+ {
+ query: query,
+ queries: queries,
+ status: status,
+ reason: reason,
+ matches: matches.map(&:to_h)
+ }
+ end
+ end
+
+ def initialize(scope:, account_id: nil)
+ @scope = scope
+ @account_id = account_id
+ end
+
+ def search(query)
+ matches = []
+ queries = []
+
+ search_queries(query).each do |search_query|
+ queries << search_query
+ matches = merge_matches(matches, @scope.search_with_metadata(search_query, account_id: @account_id))
+ break if matches.first && sufficient_match?(matches.first)
+ end
+
+ Result.new(query: query, queries: queries, matches: matches, status: status_for(matches), reason: reason_for(matches))
+ end
+
+ def self.record(result)
+ Current.captain_documentation_searches ||= []
+ Current.captain_documentation_searches << result.to_h
+ end
+
+ def self.format_for_tool(result, no_results_message:)
+ return "#{no_results_message}\n\n#{weak_instruction(result)}" if result.empty?
+
+ sections = [quality_section(result)]
+ sections.concat(result.matches.first(TOP_MATCHES_TO_FORMAT).map { |match| format_match(match) })
+ sections.join("\n")
+ end
+
+ def self.format_match(match)
+ response = match.response
+ formatted_response = "\nQuestion: #{response.question}\nAnswer: #{response.answer}\n"
+ if response.documentable.present? && response.documentable.try(:external_link)
+ formatted_response += "Source: #{response.documentable.external_link}\n"
+ end
+ formatted_response += "Retrieval: #{match.retrieval_methods.join(', ')}"
+ formatted_response += ", semantic_distance=#{format('%.4f', match.semantic_distance)}" if match.semantic_distance.present?
+ formatted_response += ", keyword_coverage=#{format('%.2f', match.keyword_coverage)}" if match.keyword_score.positive?
+ "#{formatted_response}\n"
+ end
+
+ def self.quality_section(result)
+ lines = ["Search quality: #{result.status}", "Search reason: #{result.reason}"]
+ lines << "Search queries: #{result.queries.join(' | ')}" if result.queries.to_a.size > 1
+ lines << weak_instruction(result) if result.weak?
+ lines.join("\n")
+ end
+
+ def self.weak_instruction(_result)
+ [
+ 'Instruction: The retrieved documentation is missing or weak. Do not use it to make factual claims.',
+ "Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or ask whether",
+ 'the user wants to talk to a support person.'
+ ].join(' ')
+ end
+
+ private
+
+ def search_queries(query)
+ query = query.to_s.strip
+ terms = Captain::AssistantResponse.search_terms(query)
+ keyword_query = terms.join(' ')
+ trailing_query = terms.last(8).join(' ')
+
+ [query, keyword_query, trailing_query].filter_map(&:presence).uniq.first(SEARCH_ATTEMPT_LIMIT)
+ end
+
+ def merge_matches(existing_matches, new_matches)
+ matches_by_response_id = existing_matches.index_by { |match| match.response.id }
+
+ new_matches.each do |new_match|
+ existing_match = matches_by_response_id[new_match.response.id]
+ if existing_match
+ merge_match!(existing_match, new_match)
+ else
+ matches_by_response_id[new_match.response.id] = new_match
+ end
+ end
+
+ matches_by_response_id.values.sort_by do |match|
+ [match.semantic_distance || 1.0, -match.keyword_score, match.response.id]
+ end
+ end
+
+ def merge_match!(existing_match, new_match)
+ existing_match.semantic_distance = best_semantic_distance(existing_match.semantic_distance, new_match.semantic_distance)
+ existing_match.keyword_score = [existing_match.keyword_score, new_match.keyword_score].max
+ existing_match.keyword_coverage = [existing_match.keyword_coverage, new_match.keyword_coverage].max
+ existing_match.matched_terms |= new_match.matched_terms
+ existing_match.retrieval_methods |= new_match.retrieval_methods
+ end
+
+ def best_semantic_distance(existing_distance, new_distance)
+ return new_distance if existing_distance.blank?
+ return existing_distance if new_distance.blank?
+
+ [existing_distance, new_distance].min
+ end
+
+ def status_for(matches)
+ return 'weak' if matches.empty?
+
+ sufficient_match?(matches.first) ? 'sufficient' : 'weak'
+ end
+
+ def reason_for(matches)
+ return 'no_results' if matches.empty?
+
+ top_match = matches.first
+ return 'semantic_match' if top_match.semantic_distance.present? && top_match.semantic_distance <= MAX_ACCEPTABLE_COSINE_DISTANCE
+ return 'keyword_match' if top_match.keyword_coverage >= MIN_ACCEPTABLE_KEYWORD_COVERAGE
+
+ 'low_retrieval_confidence'
+ end
+
+ def sufficient_match?(match)
+ semantic_match?(match) || keyword_match?(match)
+ end
+
+ def semantic_match?(match)
+ match.semantic_distance.present? && match.semantic_distance <= MAX_ACCEPTABLE_COSINE_DISTANCE
+ end
+
+ def keyword_match?(match)
+ match.keyword_coverage >= MIN_ACCEPTABLE_KEYWORD_COVERAGE
+ end
+end
diff --git a/enterprise/app/services/captain/llm/documentation_sufficiency_service.rb b/enterprise/app/services/captain/llm/documentation_sufficiency_service.rb
new file mode 100644
index 000000000..9a0db5d89
--- /dev/null
+++ b/enterprise/app/services/captain/llm/documentation_sufficiency_service.rb
@@ -0,0 +1,216 @@
+class Captain::Llm::DocumentationSufficiencyService < Llm::BaseAiService
+ include Integrations::LlmInstrumentation
+
+ MAX_CONTEXT_MESSAGES = 6
+ MAX_SEARCHES = 3
+ MAX_MATCHES_PER_SEARCH = 5
+ MAX_ANSWER_CHARS = 700
+
+ def initialize(assistant:, conversation:)
+ super()
+ @assistant = assistant
+ @conversation = conversation
+ @temperature = 0.0
+ end
+
+ def evaluate(message_history:, assistant_response:, documentation_searches:)
+ user_prompt = inspection_user_prompt(
+ message_history: message_history,
+ assistant_response: assistant_response,
+ documentation_searches: documentation_searches
+ )
+
+ response = instrument_llm_call(instrumentation_params(user_prompt, documentation_searches)) do
+ chat(model: @model, temperature: @temperature)
+ .with_schema(Captain::DocumentationSufficiencySchema)
+ .with_instructions(system_prompt)
+ .ask(user_prompt)
+ end
+
+ parsed = parse_response(response.content)
+ normalize_response(parsed, response.content)
+ rescue StandardError => e
+ ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
+ Rails.logger.warn(
+ "[CAPTAIN][DocumentationSufficiency] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
+ )
+ { 'decision' => nil, 'reason' => nil, 'error' => e.message, 'model' => @model }
+ end
+
+ private
+
+ def system_prompt
+ <<~PROMPT
+ You are checking whether a customer-facing assistant response is supported by retrieved documentation.
+
+ Use only the conversation context, assistant response, and retrieved documentation search results provided.
+ Do not use outside knowledge.
+
+ Return "insufficient" when the assistant makes factual claims that are not supported by the retrieved documentation.
+ Treat prior assistant messages as claims, not evidence. They do not support the new answer by themselves.
+ Conversation context can support the answer only when the user explicitly provided the relevant fact, constraint, or artifact.
+ Check generic sufficiency dimensions:
+ - same entity, product, platform, integration, or account object
+ - same user intent, not just a nearby topic
+ - requested constraints such as plan, edition, region, channel, version, provider, billing period, availability, or current status
+ - high-risk claims such as pricing, billing, legal/compliance, limits, availability, platform support, roadmap, or account status
+ - evidence specificity; generic broad docs are not enough for specific claims
+
+ Return "sufficient" when the documentation directly supports the response, or when the response only asks a clarifying
+ question, gives a safe bounded no-answer, offers handoff, or restates user-provided context without adding external claims.
+ If documentation is missing or weak and the response gives factual claims, advice, instructions, examples, links, prices,
+ limits, availability, troubleshooting steps, product behavior, platform behavior, or account-specific statements, return "insufficient".
+
+ If decision is "insufficient", write fallback_response in the user's language. It should be brief, say you could not find
+ enough information to answer confidently, and ask whether the user wants to talk to a support person when appropriate.
+ If decision is not "insufficient", fallback_response must be empty.
+ PROMPT
+ end
+
+ def inspection_user_prompt(message_history:, assistant_response:, documentation_searches:)
+ <<~PROMPT
+
+ #{format_conversation_context(message_history)}
+
+
+
+ #{format_documentation_searches(documentation_searches)}
+
+
+
+ #{assistant_response}
+
+ PROMPT
+ end
+
+ def format_documentation_searches(searches)
+ searches.to_a.last(MAX_SEARCHES).map.with_index(1) do |search, index|
+ matches = search[:matches] || search['matches'] || []
+ <<~SEARCH
+ Search #{index}
+ query: #{search[:query] || search['query']}
+ status: #{search[:status] || search['status']}
+ reason: #{search[:reason] || search['reason']}
+ matches:
+ #{format_documentation_matches(matches)}
+ SEARCH
+ end.join("\n")
+ end
+
+ def format_documentation_matches(matches)
+ matches.to_a.first(MAX_MATCHES_PER_SEARCH).map.with_index(1) do |match, index|
+ <<~MATCH
+ #{index}. question: #{match_value(match, :question)}
+ answer: #{truncate_text(match_value(match, :answer))}
+ source: #{match_value(match, :source)}
+ semantic_distance: #{match_value(match, :semantic_distance)}
+ keyword_coverage: #{match_value(match, :keyword_coverage)}
+ retrieval_methods: #{Array(match_value(match, :retrieval_methods)).join(', ')}
+ MATCH
+ end.join("\n")
+ end
+
+ def match_value(match, key) = match[key] || match[key.to_s]
+
+ def normalize_messages(message_history)
+ message_history.filter_map do |message|
+ role = message[:role] || message['role']
+ next if role.blank?
+
+ { role: role.to_s, content: normalize_content(message[:content] || message['content']) }
+ end
+ end
+
+ def normalize_content(content)
+ return content if content.is_a?(String)
+ return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
+
+ content.to_s
+ end
+
+ def text_part?(part)
+ return false unless part.is_a?(Hash)
+
+ (part[:type] || part['type']).to_s == 'text'
+ end
+
+ def format_conversation_context(messages)
+ normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+ content = message[:content].to_s.strip
+ next if content.blank?
+
+ "#{role_label(message[:role])}: #{content}"
+ end.join("\n")
+ end
+
+ def role_label(role) = { 'user' => 'User', 'assistant' => 'Assistant' }.fetch(role, role.to_s.titleize)
+
+ def parse_response(content)
+ return content if content.is_a?(Hash)
+
+ JSON.parse(sanitize_json_response(content))
+ rescue JSON::ParserError, TypeError
+ {}
+ end
+
+ def normalize_response(parsed, raw_content)
+ decision = parsed['decision'].to_s
+ reason = parsed['reason'].to_s
+ return invalid_response(raw_content) unless Captain::DocumentationSufficiencySchema::DECISIONS.include?(decision)
+
+ {
+ 'decision' => decision,
+ 'reason' => reason.presence,
+ 'fallback_response' => parsed['fallback_response'].to_s,
+ 'raw_response' => raw_content,
+ 'model' => @model
+ }
+ end
+
+ def invalid_response(raw_content)
+ {
+ 'decision' => nil,
+ 'reason' => nil,
+ 'fallback_response' => nil,
+ 'raw_response' => raw_content,
+ 'error' => 'invalid_documentation_sufficiency_response',
+ 'model' => @model
+ }
+ end
+
+ def instrumentation_params(user_prompt, documentation_searches)
+ {
+ span_name: 'llm.captain.documentation_sufficiency',
+ model: @model,
+ temperature: @temperature,
+ account_id: @conversation.account_id,
+ conversation_id: @conversation.display_id,
+ feature_name: 'documentation_sufficiency',
+ messages: [
+ { role: 'system', content: system_prompt },
+ { role: 'user', content: user_prompt }
+ ],
+ metadata: {
+ assistant_id: @assistant.id,
+ channel_type: @conversation.inbox&.channel_type,
+ source: 'response_builder'
+ }.merge(search_metadata(documentation_searches))
+ }
+ end
+
+ def search_metadata(documentation_searches)
+ searches = documentation_searches.to_a
+ {
+ search_count: searches.length,
+ search_statuses: searches.filter_map { |search| search[:status] || search['status'] }.join(','),
+ search_reasons: searches.filter_map { |search| search[:reason] || search['reason'] }.join(',')
+ }
+ end
+
+ def truncate_text(text)
+ text = text.to_s
+ return text if text.length <= MAX_ANSWER_CHARS
+
+ "#{text.first(MAX_ANSWER_CHARS)}..."
+ end
+end
diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb
index 9520330f6..8cf91cb12 100644
--- a/enterprise/app/services/captain/llm/system_prompts_service.rb
+++ b/enterprise/app/services/captain/llm/system_prompts_service.rb
@@ -190,6 +190,8 @@ class Captain::Llm::SystemPromptsService
[Error Handling]
- If the required information is not found in the provided context, respond with an appropriate message indicating that no relevant data is available.
+ - If a tool response says search quality is weak, do not use that tool response to make factual claims.
+ Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or offer a human handoff.
- Avoid speculating or providing unverified information.
[Available Actions]
@@ -262,6 +264,7 @@ class Captain::Llm::SystemPromptsService
}
```
- If the answer is not provided in context sections, Respond to the customer and ask whether they want to talk to another support agent . If they ask to Chat with another agent, return `conversation_handoff' as the response in JSON response
+ - If a tool response says search quality is weak, do not use that tool response to make factual claims. Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or offer a human handoff.
#{'- You MUST provide numbered citations at the appropriate places in the text.' if config['feature_citation']}
#{build_tools_section(custom_tools)}
diff --git a/enterprise/app/services/captain/tools/search_documentation_service.rb b/enterprise/app/services/captain/tools/search_documentation_service.rb
index e4a237186..0e6f254e1 100644
--- a/enterprise/app/services/captain/tools/search_documentation_service.rb
+++ b/enterprise/app/services/captain/tools/search_documentation_service.rb
@@ -13,26 +13,12 @@ class Captain::Tools::SearchDocumentationService < Captain::Tools::BaseTool
.new(account: assistant.account)
.translate(query, target_language: assistant.account.locale_english_name)
- responses = assistant.responses.approved.search(translated_query)
+ result = Captain::DocumentationSearchService.new(
+ scope: assistant.responses.approved,
+ account_id: assistant.account_id
+ ).search(translated_query)
+ Captain::DocumentationSearchService.record(result)
- return 'No FAQs found for the given query' if responses.empty?
-
- responses.map { |response| format_response(response) }.join
- end
-
- private
-
- def format_response(response)
- formatted_response = "
- Question: #{response.question}
- Answer: #{response.answer}
- "
- if response.documentable.present? && response.documentable.try(:external_link)
- formatted_response += "
- Source: #{response.documentable.external_link}
- "
- end
-
- formatted_response
+ Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
end
end
diff --git a/enterprise/app/services/captain/tools/search_reply_documentation_service.rb b/enterprise/app/services/captain/tools/search_reply_documentation_service.rb
index 24c3fd379..82a6eaf36 100644
--- a/enterprise/app/services/captain/tools/search_reply_documentation_service.rb
+++ b/enterprise/app/services/captain/tools/search_reply_documentation_service.rb
@@ -22,25 +22,20 @@ class Captain::Tools::SearchReplyDocumentationService < RubyLLM::Tool
.new(account: @account)
.translate(query, target_language: @account.locale_english_name)
- responses = search_responses(translated_query)
- return 'No FAQs found for the given query' if responses.empty?
+ result = Captain::DocumentationSearchService.new(
+ scope: search_scope,
+ account_id: @account.id
+ ).search(translated_query)
+ Captain::DocumentationSearchService.record(result)
- responses.map { |response| format_response(response) }.join
+ Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
end
private
- def search_responses(query)
- if @assistant.present?
- @assistant.responses.approved.search(query, account_id: @account.id)
- else
- @account.captain_assistant_responses.approved.search(query, account_id: @account.id)
- end
- end
+ def search_scope
+ return @assistant.responses.approved if @assistant.present?
- def format_response(response)
- result = "\nQuestion: #{response.question}\nAnswer: #{response.answer}\n"
- result += "Source: #{response.documentable.external_link}\n" if response.documentable.present? && response.documentable.try(:external_link)
- result
+ @account.captain_assistant_responses.approved
end
end
diff --git a/enterprise/lib/captain/documentation_sufficiency_schema.rb b/enterprise/lib/captain/documentation_sufficiency_schema.rb
new file mode 100644
index 000000000..c1fd1bdc2
--- /dev/null
+++ b/enterprise/lib/captain/documentation_sufficiency_schema.rb
@@ -0,0 +1,20 @@
+class Captain::DocumentationSufficiencySchema < RubyLLM::Schema
+ DECISIONS = %w[sufficient insufficient].freeze
+ REASONS = %w[
+ answers_exact_question
+ no_documentation_used
+ bounded_no_answer
+ missing_or_weak_evidence
+ wrong_entity
+ wrong_intent
+ missing_constraint
+ generic_evidence
+ unsupported_high_risk_claim
+ ].freeze
+
+ string :decision,
+ enum: DECISIONS,
+ description: 'Use insufficient for unsupported factual answers; use sufficient only when supported or safely bounded'
+ string :reason, enum: REASONS, description: 'The main reason for the decision'
+ string :fallback_response, description: 'If insufficient, a brief user-facing fallback in the user language; otherwise empty'
+end
diff --git a/enterprise/lib/captain/prompts/assistant.liquid b/enterprise/lib/captain/prompts/assistant.liquid
index 61fb368ae..003faf5c9 100644
--- a/enterprise/lib/captain/prompts/assistant.liquid
+++ b/enterprise/lib/captain/prompts/assistant.liquid
@@ -11,6 +11,7 @@ Don't digress away from your instructions, and use all the available tools at yo
# Core Rules
- Do not use your own understanding or training data to provide answers. Base responses strictly on the information available through your tools and provided context.
- Do not share anything outside of the context provided.
+- If FAQ lookup says search quality is weak, do not use that result to make factual claims. Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or use handoff.
- Be concise and relevant: most of your responses should be a sentence or two, unless a more detailed explanation is necessary.
- Always detect the language from the user's input and reply in the same language.
- When there is ambiguity, ask clarifying questions rather than make assumptions.
diff --git a/enterprise/lib/captain/tools/faq_lookup_tool.rb b/enterprise/lib/captain/tools/faq_lookup_tool.rb
index 93dd90259..61da3f750 100644
--- a/enterprise/lib/captain/tools/faq_lookup_tool.rb
+++ b/enterprise/lib/captain/tools/faq_lookup_tool.rb
@@ -5,44 +5,18 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool
def perform(_tool_context, query:)
log_tool_usage('searching', { query: query })
- # Use existing vector search on approved responses
- responses = @assistant.responses.approved.search(query).to_a
+ result = Captain::DocumentationSearchService.new(
+ scope: @assistant.responses.approved,
+ account_id: @assistant.account_id
+ ).search(query)
+ Captain::DocumentationSearchService.record(result)
- if responses.empty?
+ if result.empty?
log_tool_usage('no_results', { query: query })
- "No relevant FAQs found for: #{query}"
else
- log_tool_usage('found_results', { query: query, count: responses.size })
- format_responses(responses)
- end
- end
-
- private
-
- def format_responses(responses)
- responses.map { |response| format_response(response) }.join
- end
-
- def format_response(response)
- formatted_response = "
- Question: #{response.question}
- Answer: #{response.answer}
- "
- if should_show_source?(response)
- formatted_response += "
- Source: #{response.documentable.external_link}
- "
+ log_tool_usage('found_results', { query: query, count: result.matches.size, status: result.status, reason: result.reason })
end
- formatted_response
- end
-
- def should_show_source?(response)
- return false if response.documentable.blank?
- return false unless response.documentable.try(:external_link)
-
- # Don't show source if it's a PDF placeholder
- external_link = response.documentable.external_link
- !external_link.start_with?('PDF:')
+ Captain::DocumentationSearchService.format_for_tool(result, no_results_message: "No relevant FAQs found for: #{query}")
end
end
diff --git a/lib/current.rb b/lib/current.rb
index 3376099f8..0848544a0 100644
--- a/lib/current.rb
+++ b/lib/current.rb
@@ -4,6 +4,7 @@ module Current
thread_mattr_accessor :account_user
thread_mattr_accessor :executed_by
thread_mattr_accessor :contact
+ thread_mattr_accessor :captain_documentation_searches
def self.reset
Current.user = nil
@@ -11,5 +12,6 @@ module Current
Current.account_user = nil
Current.executed_by = nil
Current.contact = nil
+ Current.captain_documentation_searches = nil
end
end
diff --git a/spec/enterprise/controllers/api/v1/accounts/captain/assistants_controller_spec.rb b/spec/enterprise/controllers/api/v1/accounts/captain/assistants_controller_spec.rb
index 4689defaf..1584a7d40 100644
--- a/spec/enterprise/controllers/api/v1/accounts/captain/assistants_controller_spec.rb
+++ b/spec/enterprise/controllers/api/v1/accounts/captain/assistants_controller_spec.rb
@@ -216,6 +216,16 @@ RSpec.describe 'Api::V1::Accounts::Captain::Assistants', type: :request do
expect(response).to have_http_status(:success)
expect(json_response[:config][:feature_citation]).to be(false)
end
+
+ it 'updates documentation sufficiency gate config' do
+ patch "/api/v1/accounts/#{account.id}/captain/assistants/#{assistant.id}",
+ params: { assistant: { config: { documentation_sufficiency_gate_enabled: true } } },
+ headers: admin.create_new_auth_token,
+ as: :json
+
+ expect(response).to have_http_status(:success)
+ expect(json_response[:config][:documentation_sufficiency_gate_enabled]).to be(true)
+ end
end
end
diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
index 8fac81d60..f020b085d 100644
--- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
+++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
@@ -11,6 +11,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
+ let(:mock_documentation_sufficiency_service) { instance_double(Captain::Llm::DocumentationSufficiencyService) }
before do
create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -152,6 +153,94 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
end
end
+ context 'when documentation sufficiency gate is enabled' do
+ let(:weak_search_result) do
+ Captain::DocumentationSearchService::Result.new(
+ query: 'current plan limits',
+ queries: ['current plan limits'],
+ matches: [],
+ status: 'weak',
+ reason: 'no_results'
+ )
+ end
+
+ before do
+ assistant.update!(config: { 'documentation_sufficiency_gate_enabled' => true })
+ allow(Captain::Llm::DocumentationSufficiencyService).to receive(:new).with(
+ assistant: assistant,
+ conversation: conversation
+ ).and_return(mock_documentation_sufficiency_service)
+ end
+
+ it 'replaces an unsupported answer with the bounded fallback from the gate' do
+ allow(mock_llm_chat_service).to receive(:generate_response) do
+ Captain::DocumentationSearchService.record(weak_search_result)
+ { 'response' => 'Your current plan has unlimited usage.' }
+ end
+ allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
+ {
+ 'decision' => 'insufficient',
+ 'reason' => 'missing_constraint',
+ 'fallback_response' => "I couldn't find enough information to answer that confidently. Would you like support?",
+ 'model' => 'gpt-4.1'
+ }
+ )
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.messages.outgoing.last.content).to eq(
+ "I couldn't find enough information to answer that confidently. Would you like support?"
+ )
+ expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+ end
+
+ it 'keeps the assistant response when the gate finds enough support' do
+ allow(mock_llm_chat_service).to receive(:generate_response) do
+ Captain::DocumentationSearchService.record(weak_search_result)
+ { 'response' => 'Billing settings show current plan usage.' }
+ end
+ allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
+ { 'decision' => 'sufficient', 'reason' => 'answers_exact_question', 'fallback_response' => '', 'model' => 'gpt-4.1' }
+ )
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.messages.outgoing.last.content).to eq('Billing settings show current plan usage.')
+ end
+
+ it 'checks high-risk answers even when no documentation search was recorded' do
+ allow(mock_llm_chat_service).to receive(:generate_response).and_return(
+ { 'response' => 'Your current plan costs $99 per agent per month.' }
+ )
+ expect(mock_documentation_sufficiency_service).to receive(:evaluate).with(
+ message_history: [{ content: 'Hello', role: 'user' }],
+ assistant_response: 'Your current plan costs $99 per agent per month.',
+ documentation_searches: [
+ {
+ query: 'Hello',
+ queries: ['Hello'],
+ status: 'weak',
+ reason: 'no_documentation_search',
+ matches: []
+ }
+ ]
+ ).and_return(
+ {
+ 'decision' => 'insufficient',
+ 'reason' => 'unsupported_high_risk_claim',
+ 'fallback_response' => "I couldn't find enough information to answer that confidently. Would you like support?",
+ 'model' => 'gpt-4.1'
+ }
+ )
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.messages.outgoing.last.content).to eq(
+ "I couldn't find enough information to answer that confidently. Would you like support?"
+ )
+ end
+ end
+
it 'does not send a response when the conversation is no longer pending' do
conversation.open!
diff --git a/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb b/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb
index ccae44ac2..798533ae1 100644
--- a/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb
+++ b/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb
@@ -5,15 +5,12 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
let(:assistant) { create(:captain_assistant, account: account) }
let(:tool) { described_class.new(assistant) }
let(:tool_context) { Struct.new(:state).new({}) }
+ let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
before do
- # Create installation config for OpenAI API key to avoid errors
- create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
-
- # Mock embedding service to avoid actual API calls
- embedding_service = instance_double(Captain::Llm::EmbeddingService)
- allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
- allow(embedding_service).to receive(:get_embedding).and_return(Array.new(1536, 0.1))
+ allow(Captain::DocumentationSearchService).to receive(:new)
+ .with(scope: anything, account_id: assistant.account_id)
+ .and_return(documentation_search_service)
end
describe '#description' do
@@ -37,6 +34,7 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
let!(:response1) do
create(:captain_assistant_response,
assistant: assistant,
+ account: account,
question: 'How to reset password?',
answer: 'Click on forgot password link',
documentable: document,
@@ -45,16 +43,31 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
let!(:response2) do
create(:captain_assistant_response,
assistant: assistant,
+ account: account,
question: 'How to change email?',
answer: 'Go to settings and update email',
status: 'approved')
end
before do
- # Mock nearest_neighbors to return our test responses
- allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(
- Captain::AssistantResponse.where(id: [response1.id, response2.id])
+ matches = [response1, response2].map do |response|
+ Captain::AssistantResponse::SearchMatch.new(
+ response: response,
+ semantic_distance: 0.2,
+ keyword_score: 0,
+ keyword_coverage: 0.0,
+ matched_terms: [],
+ retrieval_methods: ['semantic']
+ )
+ end
+ search_result = Captain::DocumentationSearchService::Result.new(
+ query: 'password reset',
+ queries: ['password reset'],
+ matches: matches,
+ status: 'sufficient',
+ reason: 'semantic_match'
)
+ allow(documentation_search_service).to receive(:search).and_return(search_result)
end
it 'searches FAQs and returns formatted responses' do
@@ -76,7 +89,10 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
it 'logs tool usage for search' do
expect(tool).to receive(:log_tool_usage).with('searching', { query: 'password reset' })
- expect(tool).to receive(:log_tool_usage).with('found_results', { query: 'password reset', count: 2 })
+ expect(tool).to receive(:log_tool_usage).with(
+ 'found_results',
+ { query: 'password reset', count: 2, status: 'sufficient', reason: 'semantic_match' }
+ )
tool.perform(tool_context, query: 'password reset')
end
@@ -84,13 +100,20 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
context 'when no FAQs found' do
before do
- # Return empty result set
- allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
+ search_result = Captain::DocumentationSearchService::Result.new(
+ query: 'nonexistent topic',
+ queries: ['nonexistent topic'],
+ matches: [],
+ status: 'weak',
+ reason: 'no_results'
+ )
+ allow(documentation_search_service).to receive(:search).and_return(search_result)
end
it 'returns no results message' do
result = tool.perform(tool_context, query: 'nonexistent topic')
- expect(result).to eq('No relevant FAQs found for: nonexistent topic')
+ expect(result).to include('No relevant FAQs found for: nonexistent topic')
+ expect(result).to include('Do not use it to make factual claims')
end
it 'logs tool usage for no results' do
@@ -103,11 +126,17 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
context 'with blank query' do
it 'handles empty query' do
- # Return empty result set
- allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
+ search_result = Captain::DocumentationSearchService::Result.new(
+ query: '',
+ queries: [],
+ matches: [],
+ status: 'weak',
+ reason: 'no_results'
+ )
+ allow(documentation_search_service).to receive(:search).and_return(search_result)
result = tool.perform(tool_context, query: '')
- expect(result).to eq('No relevant FAQs found for: ')
+ expect(result).to include('No relevant FAQs found for: ')
end
end
end
diff --git a/spec/enterprise/services/captain/documentation_search_service_spec.rb b/spec/enterprise/services/captain/documentation_search_service_spec.rb
new file mode 100644
index 000000000..95ebb413a
--- /dev/null
+++ b/spec/enterprise/services/captain/documentation_search_service_spec.rb
@@ -0,0 +1,102 @@
+require 'rails_helper'
+
+RSpec.describe Captain::DocumentationSearchService do
+ let(:scope_class) do
+ Class.new do
+ def search_with_metadata(*)
+ []
+ end
+ end
+ end
+ let(:scope) { scope_class.new }
+ let(:service) { described_class.new(scope: scope, account_id: 1) }
+ let(:response) do
+ instance_double(
+ Captain::AssistantResponse,
+ id: 1,
+ question: 'How do plan limits work?',
+ answer: 'Monthly limits are shown in billing settings.',
+ documentable: nil
+ )
+ end
+
+ def search_match(semantic_distance:, keyword_coverage:, keyword_score: 0, response_record: response)
+ Captain::AssistantResponse::SearchMatch.new(
+ response: response_record,
+ semantic_distance: semantic_distance,
+ keyword_score: keyword_score,
+ keyword_coverage: keyword_coverage,
+ matched_terms: [],
+ retrieval_methods: ['semantic']
+ )
+ end
+
+ describe '#search' do
+ it 'retries with generic query variants when the original query has weak matches' do
+ query = 'How do I check limits for my current monthly plan?'
+ weak_match = search_match(semantic_distance: 0.9, keyword_coverage: 0.0)
+ sufficient_match = search_match(semantic_distance: 0.8, keyword_coverage: 0.4, keyword_score: 2)
+
+ allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([weak_match])
+ allow(scope).to receive(:search_with_metadata)
+ .with('check limits current monthly plan', account_id: 1)
+ .and_return([sufficient_match])
+
+ result = service.search(query)
+
+ expect(result.status).to eq('sufficient')
+ expect(result.reason).to eq('keyword_match')
+ expect(result.queries).to eq([query, 'check limits current monthly plan'])
+ expect(result.matches.first.keyword_coverage).to eq(0.4)
+ end
+
+ it 'stops after the first query when retrieval confidence is sufficient' do
+ query = 'Where do I find billing settings?'
+ sufficient_match = search_match(semantic_distance: 0.2, keyword_coverage: 0.0)
+
+ allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([sufficient_match])
+
+ result = service.search(query)
+
+ expect(result.status).to eq('sufficient')
+ expect(result.reason).to eq('semantic_match')
+ expect(result.queries).to eq([query])
+ end
+ end
+
+ describe '.format_for_tool' do
+ it 'adds a bounded-answer instruction when no documentation is found' do
+ result = described_class::Result.new(
+ query: 'unknown topic',
+ queries: ['unknown topic'],
+ matches: [],
+ status: 'weak',
+ reason: 'no_results'
+ )
+
+ formatted_result = described_class.format_for_tool(result, no_results_message: 'No FAQs found')
+
+ expect(formatted_result).to include('No FAQs found')
+ expect(formatted_result).to include('Do not use it to make factual claims')
+ end
+ end
+
+ describe '.record' do
+ it 'stores search metadata on Current for the response-level sufficiency gate' do
+ result = described_class::Result.new(
+ query: 'billing',
+ queries: ['billing'],
+ matches: [search_match(semantic_distance: 0.2, keyword_coverage: 0.0)],
+ status: 'sufficient',
+ reason: 'semantic_match'
+ )
+
+ described_class.record(result)
+
+ expect(Current.captain_documentation_searches.first[:status]).to eq('sufficient')
+ expect(Current.captain_documentation_searches.first[:matches].first[:semantic_distance]).to eq(0.2)
+ ensure
+ Current.captain_documentation_searches = nil
+ end
+ end
+end
diff --git a/spec/enterprise/services/captain/tools/search_documentation_service_spec.rb b/spec/enterprise/services/captain/tools/search_documentation_service_spec.rb
index 41cec35f5..daac50b78 100644
--- a/spec/enterprise/services/captain/tools/search_documentation_service_spec.rb
+++ b/spec/enterprise/services/captain/tools/search_documentation_service_spec.rb
@@ -26,10 +26,13 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
end
describe '#execute' do
+ let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
+ let(:translate_query_service) { instance_double(Captain::Llm::TranslateQueryService) }
let!(:response) do
create(
:captain_assistant_response,
assistant: assistant,
+ account: assistant.account,
question: question,
answer: answer,
status: 'approved'
@@ -37,14 +40,37 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
end
let(:documentable) { create(:captain_document, external_link: external_link) }
+ let(:match) do
+ Captain::AssistantResponse::SearchMatch.new(
+ response: response,
+ semantic_distance: 0.2,
+ keyword_score: 0,
+ keyword_coverage: 0.0,
+ matched_terms: [],
+ retrieval_methods: ['semantic']
+ )
+ end
+
+ before do
+ allow(Captain::Llm::TranslateQueryService).to receive(:new).and_return(translate_query_service)
+ allow(translate_query_service).to receive(:translate).and_return(question)
+ allow(Captain::DocumentationSearchService).to receive(:new)
+ .with(scope: anything, account_id: assistant.account_id)
+ .and_return(documentation_search_service)
+ end
context 'when matching responses exist' do
- before do
- response.update(documentable: documentable)
- allow(Captain::AssistantResponse).to receive(:search).with(question).and_return([response])
- end
-
it 'returns formatted responses for the search query' do
+ response.update(documentable: documentable)
+ search_result = Captain::DocumentationSearchService::Result.new(
+ query: question,
+ queries: [question],
+ matches: [match],
+ status: 'sufficient',
+ reason: 'semantic_match'
+ )
+ allow(documentation_search_service).to receive(:search).with(question).and_return(search_result)
+
result = service.execute(query: question)
expect(result).to include(question)
@@ -54,12 +80,20 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
end
context 'when no matching responses exist' do
- before do
- allow(Captain::AssistantResponse).to receive(:search).with(question).and_return([])
- end
+ it 'returns a bounded no-results instruction' do
+ search_result = Captain::DocumentationSearchService::Result.new(
+ query: question,
+ queries: [question],
+ matches: [],
+ status: 'weak',
+ reason: 'no_results'
+ )
+ allow(documentation_search_service).to receive(:search).with(question).and_return(search_result)
- it 'returns an empty string' do
- expect(service.execute(query: question)).to eq('No FAQs found for the given query')
+ result = service.execute(query: question)
+
+ expect(result).to include('No FAQs found for the given query')
+ expect(result).to include('Do not use it to make factual claims')
end
end
end