akadmin/chatwoot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
aakashb95
2026-06-09 18:56:44 +05:30
parent 8b7aca1b72
commit 50ed5f5efb
18 changed files with 261 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files
enterprise/app/jobs/captain/conversation/documentation_sufficiency_handler.rb
-110
View File
@@ -1,110 +0,0 @@
module Captain::Conversation::DocumentationSufficiencyHandler
HIGH_RISK_TERMS = %w[
price pricing cost billing bill paid free plan subscription legal compliance policy limit limits maximum minimum
available availability current currently roadmap beta early access supported support self-hosted cloud region version
provider integration account status
].freeze
private
def reset_documentation_searches
Current.captain_documentation_searches = []
end
def clear_documentation_searches
Current.captain_documentation_searches = nil
end
def inspect_documentation_sufficiency(message_history)
searches = documentation_searches_for_inspection(message_history)
return unless should_inspect_documentation_sufficiency?(message_history, searches)
apply_documentation_sufficiency_inspection(documentation_sufficiency_inspection(message_history, searches))
rescue StandardError => e
ChatwootExceptionTracker.new(e, account: account).capture_exception
Rails.logger.warn(
"[CAPTAIN][ResponseBuilderJob] Documentation sufficiency check failed for account=#{account.id} " \
"conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
)
end
def should_inspect_documentation_sufficiency?(message_history, searches)
return false unless documentation_sufficiency_gate_enabled?
return false unless response_has_user_facing_answer?
return false if searches.empty?
documentation_sufficiency_check_needed?(message_history, searches)
end
def response_has_user_facing_answer?
@response.present? &&
@response['response'].present? &&
@response['response'] != 'conversation_handoff' &&
!@response['handoff_tool_called']
end
def documentation_searches_for_inspection(message_history)
searches = Current.captain_documentation_searches.to_a
return searches if searches.present?
return [] unless high_risk_conversation?(message_history)
[synthetic_no_results_search(message_history)]
end
def synthetic_no_results_search(message_history)
{
query: last_user_message_content(message_history),
queries: [last_user_message_content(message_history)],
status: 'weak',
reason: 'no_documentation_search',
matches: []
}
end
def documentation_sufficiency_inspection(message_history, searches)
Captain::Llm::DocumentationSufficiencyService.new(
assistant: @assistant,
conversation: @conversation
).evaluate(
message_history: message_history,
assistant_response: @response['response'],
documentation_searches: searches
)
end
def documentation_sufficiency_gate_enabled?
ActiveModel::Type::Boolean.new.cast(@assistant.config['documentation_sufficiency_gate_enabled'])
end
def documentation_sufficiency_check_needed?(message_history, searches)
searches.any? { |search| search[:status] == 'weak' || search['status'] == 'weak' } || high_risk_conversation?(message_history)
end
def high_risk_conversation?(message_history)
text = "#{message_history.last(3).map { |message| message[:content] || message['content'] }.join(' ')} #{@response&.dig('response')}".downcase
HIGH_RISK_TERMS.any? { |term| text.include?(term) }
end
def last_user_message_content(message_history)
user_message = message_history.reverse.find { |message| (message[:role] || message['role']).to_s == 'user' }
user_message && (user_message[:content] || user_message['content']).to_s
end
def apply_documentation_sufficiency_inspection(inspection)
return unless inspection['decision'] == 'insufficient'
fallback_response = inspection['fallback_response'].presence || default_documentation_sufficiency_fallback
@response.merge!(
'response' => fallback_response,
'action' => 'continue',
'action_reason' => 'missing_docs_bounded_answer',
'action_source' => 'documentation_sufficiency',
'documentation_sufficiency_reason' => inspection['reason'],
'documentation_sufficiency_model' => inspection['model']
)
end
def default_documentation_sufficiency_fallback
"I couldn't find enough information to answer that confidently. Would you like me to connect you with a support person?"
end
end
enterprise/app/jobs/captain/conversation/documentation_support_gate.rb
+78
View File
@@ -0,0 +1,78 @@
module Captain::Conversation::DocumentationSupportGate
private
def check_documentation_support(message_history)
return unless documentation_gate_enabled?
return unless customer_reply?
review = review_documentation_support(message_history, documentation_evidence(message_history))
apply_documentation_fallback(review)
rescue StandardError => e
ChatwootExceptionTracker.new(e, account: account).capture_exception
Rails.logger.warn(
"[CAPTAIN][ResponseBuilderJob] Documentation support check failed for account=#{account.id} " \
"conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
)
end
def documentation_gate_enabled?
ActiveModel::Type::Boolean.new.cast(@assistant.config['documentation_sufficiency_gate_enabled'])
end
def customer_reply?
@response.present? &&
@response['response'].present? &&
@response['response'] != 'conversation_handoff' &&
!@response['handoff_tool_called']
end
def documentation_evidence(message_history)
searches = @response['documentation_searches'].to_a
return searches if searches.present?
[no_documentation_search(last_user_message(message_history))]
end
def no_documentation_search(query)
{
query: query,
queries: [query],
status: 'weak',
reason: 'no_documentation_search',
matches: []
}
end
def review_documentation_support(message_history, evidence)
Captain::Llm::DocumentationSufficiencyService.new(
assistant: @assistant,
conversation: @conversation
).evaluate(
message_history: message_history,
assistant_response: @response['response'],
documentation_searches: evidence
)
end
def last_user_message(message_history)
message = message_history.reverse.find { |item| (item[:role] || item['role']).to_s == 'user' }
message && (message[:content] || message['content']).to_s
end
def apply_documentation_fallback(review)
return unless review['decision'] == 'insufficient'
@response.merge!(
'response' => review['fallback_response'].presence || default_documentation_fallback,
'action' => 'continue',
'action_reason' => 'missing_docs_bounded_answer',
'action_source' => 'documentation_support',
'documentation_sufficiency_reason' => review['reason'],
'documentation_sufficiency_model' => review['model']
)
end
def default_documentation_fallback
"I couldn't find enough information to answer that confidently. Would you like me to connect you with a support person?"
end
end
enterprise/app/jobs/captain/conversation/response_builder_job.rb
+6 -7
View File
@@ -1,6 +1,6 @@
class Captain::Conversation::ResponseBuilderJob < ApplicationJob
include Captain::Conversation::V1ActionClassifier
include Captain::Conversation::DocumentationSufficiencyHandler
include Captain::Conversation::DocumentationSupportGate
MAX_MESSAGE_LENGTH = 10_000
retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
@@ -26,7 +26,6 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
rescue StandardError => e
handle_error(e)
ensure
clear_documentation_searches
Current.executed_by = nil
end
@@ -36,22 +35,22 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
def generate_and_process_response
message_history = collect_previous_messages
reset_documentation_searches
@response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
chat_service = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation)
@response = chat_service.generate_response(
message_history: message_history
)
inspect_documentation_sufficiency(message_history) if conversation_pending?
@response['documentation_searches'] = chat_service.documentation_searches
check_documentation_support(message_history) if conversation_pending?
classify_v1_response_action(message_history) if conversation_pending?
process_response
end
def generate_response_with_v2
message_history = collect_previous_messages
reset_documentation_searches
@response = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation).generate_response(
message_history: message_history
)
inspect_documentation_sufficiency(message_history) if conversation_pending?
check_documentation_support(message_history) if conversation_pending?
process_response
end
enterprise/app/models/captain/assistant_response.rb
+3 -100
View File
@@ -26,18 +26,9 @@
class Captain::AssistantResponse < ApplicationRecord
self.table_name = 'captain_assistant_responses'
SEARCH_LIMIT = 5
KEYWORD_SEARCH_LIMIT_MULTIPLIER = 2
SEARCH_STOP_WORDS = %w[
about after all also and any are but can for from has have how into its may more not now off our out
the their them then there these they this was what when where which who why with you your
].freeze
SearchMatch = Struct.new(
:response,
:semantic_distance,
:keyword_score,
:keyword_coverage,
:matched_terms,
:retrieval_methods,
keyword_init: true
) do
def to_h
@@ -46,11 +37,7 @@ class Captain::AssistantResponse < ApplicationRecord
question: response.question,
answer: response.answer,
source: response.documentable&.try(:external_link),
semantic_distance: semantic_distance,
keyword_score: keyword_score,
keyword_coverage: keyword_coverage,
matched_terms: matched_terms,
retrieval_methods: retrieval_methods
semantic_distance: semantic_distance
}
end
end
@@ -80,10 +67,7 @@ class Captain::AssistantResponse < ApplicationRecord
end
def self.search_with_metadata(query, account_id: nil, limit: SEARCH_LIMIT)
semantic_matches = semantic_search_matches(query, account_id: account_id, limit: limit)
keyword_matches = keyword_search_matches(query, limit: limit * KEYWORD_SEARCH_LIMIT_MULTIPLIER)
merge_search_matches(semantic_matches, keyword_matches).first(limit)
semantic_search_matches(query, account_id: account_id, limit: limit)
end
def self.semantic_search_matches(query, account_id:, limit:)
@@ -91,92 +75,11 @@ class Captain::AssistantResponse < ApplicationRecord
nearest_neighbors(:embedding, embedding, distance: 'cosine').limit(limit).map do |response|
SearchMatch.new(
response: response,
semantic_distance: response.neighbor_distance&.to_f,
keyword_score: 0,
keyword_coverage: 0.0,
matched_terms: [],
retrieval_methods: ['semantic']
semantic_distance: response.neighbor_distance&.to_f
)
end
end
def self.keyword_search_matches(query, limit:)
terms = search_terms(query)
return [] if terms.empty?
matches = keyword_search_scope(terms).limit(limit).map do |response|
matched_terms = matched_terms_for(response, terms)
SearchMatch.new(
response: response,
semantic_distance: nil,
keyword_score: matched_terms.size,
keyword_coverage: matched_terms.size.to_f / terms.size,
matched_terms: matched_terms,
retrieval_methods: ['keyword']
)
end
matches.sort_by { |match| [-match.keyword_score, match.response.id] }
end
def self.keyword_search_scope(terms)
conditions = []
values = []
terms.each do |term|
pattern = "%#{sanitize_sql_like(term)}%"
conditions << '(question ILIKE ? OR answer ILIKE ?)'
values.push(pattern, pattern)
end
where(conditions.join(' OR '), *values)
end
def self.search_terms(query)
query.to_s.downcase.scan(/[[:alnum:]]+/).filter_map do |term|
next if term.length < 3
next if SEARCH_STOP_WORDS.include?(term)
term
end.uniq
end
def self.matched_terms_for(response, terms)
text = "#{response.question} #{response.answer}".downcase
terms.select { |term| text.include?(term) }
end
def self.merge_search_matches(semantic_matches, keyword_matches)
matches_by_response_id = {}
semantic_matches.each do |match|
matches_by_response_id[match.response.id] = match
end
keyword_matches.each do |keyword_match|
existing = matches_by_response_id[keyword_match.response.id]
if existing
merge_keyword_match!(existing, keyword_match)
else
matches_by_response_id[keyword_match.response.id] = keyword_match
end
end
sort_search_matches(matches_by_response_id.values)
end
def self.merge_keyword_match!(existing, keyword_match)
existing.keyword_score = keyword_match.keyword_score
existing.keyword_coverage = keyword_match.keyword_coverage
existing.matched_terms = keyword_match.matched_terms
existing.retrieval_methods |= keyword_match.retrieval_methods
end
def self.sort_search_matches(matches)
matches.sort_by do |match|
[match.semantic_distance || 1.0, -match.keyword_score, match.response.id]
end
end
private
def ensure_status
enterprise/app/services/captain/assistant/agent_runner_service.rb
+5 -5
View File
@@ -6,10 +6,7 @@ class Captain::Assistant::AgentRunnerService
include Captain::Assistant::RunnerCallbacksHelper
include Captain::Assistant::TracePayloadHelper
CONVERSATION_STATE_ATTRIBUTES = %i[
id display_id inbox_id contact_id status priority
label_list custom_attributes additional_attributes
].freeze
CONVERSATION_STATE_ATTRIBUTES = %i[id display_id inbox_id contact_id status priority label_list custom_attributes additional_attributes].freeze
CONTACT_STATE_ATTRIBUTES = %i[
id name email phone_number identifier contact_type
@@ -19,6 +16,7 @@ class Captain::Assistant::AgentRunnerService
CONTACT_INBOX_STATE_ATTRIBUTES = %i[id hmac_verified].freeze
CAMPAIGN_STATE_ATTRIBUTES = %i[id title message campaign_type description].freeze
def initialize(assistant:, conversation: nil, callbacks: {}, source: nil)
@assistant = assistant
@conversation = conversation
@@ -100,6 +98,7 @@ class Captain::Assistant::AgentRunnerService
response = output.is_a?(Hash) ? output.with_indifferent_access : { 'response' => output.to_s, 'reasoning' => 'Processed by agent' }
response['agent_name'] = result.context&.dig(:current_agent)
response['handoff_tool_called'] = result.context&.dig(:captain_v2_handoff_tool_called) || false
response['documentation_searches'] = result.context&.dig(:state, :documentation_searches).to_a
response
end
@@ -115,7 +114,8 @@ class Captain::Assistant::AgentRunnerService
state = {
account_id: @assistant.account_id,
assistant_id: @assistant.id,
assistant_config: @assistant.config
assistant_config: @assistant.config,
documentation_searches: []
}
state[:source] = @source if @source.present?
enterprise/app/services/captain/documentation_search_service.rb
+24 -98
View File
@@ -1,28 +1,8 @@
class Captain::DocumentationSearchService
MAX_ACCEPTABLE_COSINE_DISTANCE = 0.45
MIN_ACCEPTABLE_KEYWORD_COVERAGE = 0.4
# pgvector cosine distance: lower is closer. This only marks retrieval confidence;
# final answer support is checked by Captain::Llm::DocumentationSufficiencyService.
CLOSE_MATCH_DISTANCE = 0.45
TOP_MATCHES_TO_FORMAT = 5
SEARCH_ATTEMPT_LIMIT = 3
Result = Struct.new(:query, :queries, :matches, :status, :reason, keyword_init: true) do
def weak?
status == 'weak'
end
def empty?
matches.empty?
end
def to_h
{
query: query,
queries: queries,
status: status,
reason: reason,
matches: matches.map(&:to_h)
}
end
end
def initialize(scope:, account_id: nil)
@scope = scope
@@ -30,28 +10,25 @@ class Captain::DocumentationSearchService
end
def search(query)
matches = []
queries = []
search_queries(query).each do |search_query|
queries << search_query
matches = merge_matches(matches, @scope.search_with_metadata(search_query, account_id: @account_id))
break if matches.first && sufficient_match?(matches.first)
end
Result.new(query: query, queries: queries, matches: matches, status: status_for(matches), reason: reason_for(matches))
matches = @scope.search_with_metadata(query, account_id: @account_id)
{
query: query,
queries: [query],
matches: matches,
status: status_for(matches),
reason: reason_for(matches)
}
end
def self.record(result)
Current.captain_documentation_searches ||= []
Current.captain_documentation_searches << result.to_h
def self.serialize(result)
result.merge(matches: result[:matches].map(&:to_h))
end
def self.format_for_tool(result, no_results_message:)
return "#{no_results_message}\n\n#{weak_instruction(result)}" if result.empty?
return "#{no_results_message}\n\n#{weak_instruction}" if result[:matches].empty?
sections = [quality_section(result)]
sections.concat(result.matches.first(TOP_MATCHES_TO_FORMAT).map { |match| format_match(match) })
sections.concat(result[:matches].first(TOP_MATCHES_TO_FORMAT).map { |match| format_match(match) })
sections.join("\n")
end
@@ -61,20 +38,19 @@ class Captain::DocumentationSearchService
if response.documentable.present? && response.documentable.try(:external_link)
formatted_response += "Source: #{response.documentable.external_link}\n"
end
formatted_response += "Retrieval: #{match.retrieval_methods.join(', ')}"
formatted_response += 'Retrieval: semantic'
formatted_response += ", semantic_distance=#{format('%.4f', match.semantic_distance)}" if match.semantic_distance.present?
formatted_response += ", keyword_coverage=#{format('%.2f', match.keyword_coverage)}" if match.keyword_score.positive?
"#{formatted_response}\n"
end
def self.quality_section(result)
lines = ["Search quality: #{result.status}", "Search reason: #{result.reason}"]
lines << "Search queries: #{result.queries.join(' | ')}" if result.queries.to_a.size > 1
lines << weak_instruction(result) if result.weak?
lines = ["Search quality: #{result[:status]}", "Search reason: #{result[:reason]}"]
lines << "Search queries: #{result[:queries].join(' | ')}" if result[:queries].to_a.size > 1
lines << weak_instruction if result[:status] == 'weak'
lines.join("\n")
end
def self.weak_instruction(_result)
def self.weak_instruction
[
'Instruction: The retrieved documentation is missing or weak. Do not use it to make factual claims.',
"Say you couldn't find enough information to answer confidently, ask one clarifying question if useful, or ask whether",
@@ -84,72 +60,22 @@ class Captain::DocumentationSearchService
private
def search_queries(query)
query = query.to_s.strip
terms = Captain::AssistantResponse.search_terms(query)
keyword_query = terms.join(' ')
trailing_query = terms.last(8).join(' ')
[query, keyword_query, trailing_query].filter_map(&:presence).uniq.first(SEARCH_ATTEMPT_LIMIT)
end
def merge_matches(existing_matches, new_matches)
matches_by_response_id = existing_matches.index_by { |match| match.response.id }
new_matches.each do |new_match|
existing_match = matches_by_response_id[new_match.response.id]
if existing_match
merge_match!(existing_match, new_match)
else
matches_by_response_id[new_match.response.id] = new_match
end
end
matches_by_response_id.values.sort_by do |match|
[match.semantic_distance || 1.0, -match.keyword_score, match.response.id]
end
end
def merge_match!(existing_match, new_match)
existing_match.semantic_distance = best_semantic_distance(existing_match.semantic_distance, new_match.semantic_distance)
existing_match.keyword_score = [existing_match.keyword_score, new_match.keyword_score].max
existing_match.keyword_coverage = [existing_match.keyword_coverage, new_match.keyword_coverage].max
existing_match.matched_terms |= new_match.matched_terms
existing_match.retrieval_methods |= new_match.retrieval_methods
end
def best_semantic_distance(existing_distance, new_distance)
return new_distance if existing_distance.blank?
return existing_distance if new_distance.blank?
[existing_distance, new_distance].min
end
def status_for(matches)
return 'weak' if matches.empty?
sufficient_match?(matches.first) ? 'sufficient' : 'weak'
close_match?(matches.first) ? 'found' : 'weak'
end
def reason_for(matches)
return 'no_results' if matches.empty?
top_match = matches.first
return 'semantic_match' if top_match.semantic_distance.present? && top_match.semantic_distance <= MAX_ACCEPTABLE_COSINE_DISTANCE
return 'keyword_match' if top_match.keyword_coverage >= MIN_ACCEPTABLE_KEYWORD_COVERAGE
return 'semantic_match' if close_match?(top_match)
'low_retrieval_confidence'
end
def sufficient_match?(match)
semantic_match?(match) || keyword_match?(match)
end
def semantic_match?(match)
match.semantic_distance.present? && match.semantic_distance <= MAX_ACCEPTABLE_COSINE_DISTANCE
end
def keyword_match?(match)
match.keyword_coverage >= MIN_ACCEPTABLE_KEYWORD_COVERAGE
def close_match?(match)
match.semantic_distance.present? && match.semantic_distance <= CLOSE_MATCH_DISTANCE
end
end
enterprise/app/services/captain/llm/assistant_chat_service.rb
+9 -1
View File
@@ -1,5 +1,6 @@
class Captain::Llm::AssistantChatService < Llm::BaseAiService
include Captain::ChatHelper
attr_reader :documentation_searches
def initialize(assistant: nil, conversation: nil, source: nil)
super()
@@ -11,6 +12,7 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
@messages = [system_message]
@response = ''
@documentation_searches = []
@tools = build_tools
end
@@ -30,7 +32,13 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
private
def build_tools
tools = [Captain::Tools::SearchDocumentationService.new(@assistant, user: nil)]
tools = [
Captain::Tools::SearchDocumentationService.new(
@assistant,
user: nil,
on_search: ->(search) { @documentation_searches << search }
)
]
return tools unless custom_tools_enabled?
tools + @assistant.account.captain_custom_tools.enabled.map do |ct|
enterprise/app/services/captain/llm/documentation_sufficiency_service.rb
+9 -9
View File
@@ -49,17 +49,19 @@ class Captain::Llm::DocumentationSufficiencyService < Llm::BaseAiService
Return "insufficient" when the assistant makes factual claims that are not supported by the retrieved documentation.
Treat prior assistant messages as claims, not evidence. They do not support the new answer by themselves.
Conversation context can support the answer only when the user explicitly provided the relevant fact, constraint, or artifact.
Check generic sufficiency dimensions:
Check generic support dimensions:
- same entity, product, platform, integration, or account object
- same user intent, not just a nearby topic
- requested constraints such as plan, edition, region, channel, version, provider, billing period, availability, or current status
- high-risk claims such as pricing, billing, legal/compliance, limits, availability, platform support, roadmap, or account status
- evidence specificity; generic broad docs are not enough for specific claims
- requested constraints from the user
- specific claims in the response
- evidence specificity; broad docs are not enough for specific claims
Return "sufficient" when the documentation directly supports the response, or when the response only asks a clarifying
Return "sufficient" only when the documentation directly supports the response, or when the response only asks a clarifying
question, gives a safe bounded no-answer, offers handoff, or restates user-provided context without adding external claims.
If documentation is missing or weak and the response gives factual claims, advice, instructions, examples, links, prices,
limits, availability, troubleshooting steps, product behavior, platform behavior, or account-specific statements, return "insufficient".
A response is not a safe bounded no-answer if it includes unsupported facts, steps, recommendations, examples, or links.
A response does not answer the exact question unless the retrieved documentation supports the specific claims it makes.
If documentation is missing or weak and the response gives factual claims, advice, instructions, examples, links,
product behavior, platform behavior, or account-specific statements, return "insufficient".
If decision is "insufficient", write fallback_response in the user's language. It should be brief, say you could not find
enough information to answer confidently, and ask whether the user wants to talk to a support person when appropriate.
@@ -104,8 +106,6 @@ class Captain::Llm::DocumentationSufficiencyService < Llm::BaseAiService
answer: #{truncate_text(match_value(match, :answer))}
source: #{match_value(match, :source)}
semantic_distance: #{match_value(match, :semantic_distance)}
keyword_coverage: #{match_value(match, :keyword_coverage)}
retrieval_methods: #{Array(match_value(match, :retrieval_methods)).join(', ')}
MATCH
end.join("\n")
end
enterprise/app/services/captain/tools/search_documentation_service.rb
+6 -1
View File
@@ -1,4 +1,9 @@
class Captain::Tools::SearchDocumentationService < Captain::Tools::BaseTool
def initialize(assistant, user: nil, on_search: nil)
super(assistant, user: user)
@on_search = on_search
end
def self.name
'search_documentation'
end
@@ -17,7 +22,7 @@ class Captain::Tools::SearchDocumentationService < Captain::Tools::BaseTool
scope: assistant.responses.approved,
account_id: assistant.account_id
).search(translated_query)
Captain::DocumentationSearchService.record(result)
@on_search&.call(Captain::DocumentationSearchService.serialize(result))
Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
end
enterprise/app/services/captain/tools/search_reply_documentation_service.rb
+3 -2
View File
@@ -5,9 +5,10 @@ class Captain::Tools::SearchReplyDocumentationService < RubyLLM::Tool
param :query, desc: 'Search Query', required: true
def initialize(account:, assistant: nil)
def initialize(account:, assistant: nil, on_search: nil)
@account = account
@assistant = assistant
@on_search = on_search
super()
end
@@ -26,7 +27,7 @@ class Captain::Tools::SearchReplyDocumentationService < RubyLLM::Tool
scope: search_scope,
account_id: @account.id
).search(translated_query)
Captain::DocumentationSearchService.record(result)
@on_search&.call(Captain::DocumentationSearchService.serialize(result))
Captain::DocumentationSearchService.format_for_tool(result, no_results_message: 'No FAQs found for the given query')
end
enterprise/lib/captain/documentation_sufficiency_schema.rb
+1 -1
View File
@@ -9,7 +9,7 @@ class Captain::DocumentationSufficiencySchema < RubyLLM::Schema
wrong_intent
missing_constraint
generic_evidence
unsupported_high_risk_claim
unsupported_specific_claim
].freeze
string :decision,
enterprise/lib/captain/tools/faq_lookup_tool.rb
+13 -4
View File
@@ -2,21 +2,30 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool
description 'Search FAQ responses using semantic similarity to find relevant answers'
param :query, type: 'string', desc: 'The question or topic to search for in the FAQ database'
def perform(_tool_context, query:)
def perform(tool_context, query:)
log_tool_usage('searching', { query: query })
result = Captain::DocumentationSearchService.new(
scope: @assistant.responses.approved,
account_id: @assistant.account_id
).search(query)
Captain::DocumentationSearchService.record(result)
record_documentation_search(tool_context, result)
if result.empty?
if result[:matches].empty?
log_tool_usage('no_results', { query: query })
else
log_tool_usage('found_results', { query: query, count: result.matches.size, status: result.status, reason: result.reason })
log_tool_usage('found_results', { query: query, count: result[:matches].size, status: result[:status], reason: result[:reason] })
end
Captain::DocumentationSearchService.format_for_tool(result, no_results_message: "No relevant FAQs found for: #{query}")
end
private
def record_documentation_search(tool_context, result)
searches = tool_context&.state&.dig(:documentation_searches)
return unless searches
searches << Captain::DocumentationSearchService.serialize(result)
end
end
lib/current.rb
-2
View File
@@ -4,7 +4,6 @@ module Current
thread_mattr_accessor :account_user
thread_mattr_accessor :executed_by
thread_mattr_accessor :contact
thread_mattr_accessor :captain_documentation_searches
def self.reset
Current.user = nil
@@ -12,6 +11,5 @@ module Current
Current.account_user = nil
Current.executed_by = nil
Current.contact = nil
Current.captain_documentation_searches = nil
end
end
spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
+13 -10
View File
@@ -19,6 +19,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
allow(inbox).to receive(:captain_active?).and_return(true)
allow(Captain::Llm::AssistantChatService).to receive(:new).and_return(mock_llm_chat_service)
allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([])
allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
@@ -155,13 +156,13 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
context 'when documentation sufficiency gate is enabled' do
let(:weak_search_result) do
Captain::DocumentationSearchService::Result.new(
{
query: 'current plan limits',
queries: ['current plan limits'],
matches: [],
status: 'weak',
reason: 'no_results'
)
}
end
before do
@@ -173,10 +174,10 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
end
it 'replaces an unsupported answer with the bounded fallback from the gate' do
allow(mock_llm_chat_service).to receive(:generate_response) do
Captain::DocumentationSearchService.record(weak_search_result)
allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([weak_search_result])
allow(mock_llm_chat_service).to receive(:generate_response).and_return(
{ 'response' => 'Your current plan has unlimited usage.' }
end
)
allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
{
'decision' => 'insufficient',
@@ -195,10 +196,10 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
end
it 'keeps the assistant response when the gate finds enough support' do
allow(mock_llm_chat_service).to receive(:generate_response) do
Captain::DocumentationSearchService.record(weak_search_result)
allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([weak_search_result])
allow(mock_llm_chat_service).to receive(:generate_response).and_return(
{ 'response' => 'Billing settings show current plan usage.' }
end
)
allow(mock_documentation_sufficiency_service).to receive(:evaluate).and_return(
{ 'decision' => 'sufficient', 'reason' => 'answers_exact_question', 'fallback_response' => '', 'model' => 'gpt-4.1' }
)
@@ -208,7 +209,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
expect(conversation.messages.outgoing.last.content).to eq('Billing settings show current plan usage.')
end
it 'checks high-risk answers even when no documentation search was recorded' do
it 'checks answers even when no documentation search was recorded' do
allow(mock_llm_chat_service).to receive(:generate_response).and_return(
{ 'response' => 'Your current plan costs $99 per agent per month.' }
)
@@ -227,7 +228,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
).and_return(
{
'decision' => 'insufficient',
'reason' => 'unsupported_high_risk_claim',
'reason' => 'unsupported_specific_claim',
'fallback_response' => "I couldn't find enough information to answer that confidently. Would you like support?",
'model' => 'gpt-4.1'
}
@@ -455,6 +456,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
allow(Captain::OpenAiMessageBuilderService).to receive(:new).with(message: anything).and_return(mock_message_builder)
allow(mock_message_builder).to receive(:generate_content).and_return('Hello with image')
allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Test response' })
allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([])
end
context 'when ActiveStorage::FileNotFoundError occurs' do
@@ -567,6 +569,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
allow(Captain::Llm::AssistantChatService).to receive(:new).and_return(mock_llm_chat_service)
allow(account).to receive(:feature_enabled?).and_return(false)
allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
allow(mock_llm_chat_service).to receive(:documentation_searches).and_return([])
end
context 'when handoff occurs outside business hours' do
spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb
+25 -33
View File
@@ -4,7 +4,8 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
let(:account) { create(:account) }
let(:assistant) { create(:captain_assistant, account: account) }
let(:tool) { described_class.new(assistant) }
let(:tool_context) { Struct.new(:state).new({}) }
let(:documentation_searches) { [] }
let(:tool_context) { Struct.new(:state).new({ documentation_searches: documentation_searches }) }
let(:documentation_search_service) { instance_double(Captain::DocumentationSearchService) }
before do
@@ -29,6 +30,20 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
end
describe '#perform' do
def search_match(response)
Captain::AssistantResponse::SearchMatch.new(response: response, semantic_distance: 0.2)
end
def search_result(query:, matches:, status:, reason:)
{
query: query,
queries: [query],
matches: matches,
status: status,
reason: reason
}
end
context 'when FAQs exist' do
let(:document) { create(:captain_document, assistant: assistant) }
let!(:response1) do
@@ -50,24 +65,10 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
end
before do
matches = [response1, response2].map do |response|
Captain::AssistantResponse::SearchMatch.new(
response: response,
semantic_distance: 0.2,
keyword_score: 0,
keyword_coverage: 0.0,
matched_terms: [],
retrieval_methods: ['semantic']
)
end
search_result = Captain::DocumentationSearchService::Result.new(
query: 'password reset',
queries: ['password reset'],
matches: matches,
status: 'sufficient',
reason: 'semantic_match'
matches = [response1, response2].map { |response| search_match(response) }
allow(documentation_search_service).to receive(:search).and_return(
search_result(query: 'password reset', matches: matches, status: 'found', reason: 'semantic_match')
)
allow(documentation_search_service).to receive(:search).and_return(search_result)
end
it 'searches FAQs and returns formatted responses' do
@@ -77,6 +78,7 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
expect(result).to include('Answer: Click on forgot password link')
expect(result).to include('Question: How to change email?')
expect(result).to include('Answer: Go to settings and update email')
expect(documentation_searches.first[:status]).to eq('found')
end
it 'includes source link when document has external_link' do
@@ -91,7 +93,7 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
expect(tool).to receive(:log_tool_usage).with('searching', { query: 'password reset' })
expect(tool).to receive(:log_tool_usage).with(
'found_results',
{ query: 'password reset', count: 2, status: 'sufficient', reason: 'semantic_match' }
{ query: 'password reset', count: 2, status: 'found', reason: 'semantic_match' }
)
tool.perform(tool_context, query: 'password reset')
@@ -100,14 +102,9 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
context 'when no FAQs found' do
before do
search_result = Captain::DocumentationSearchService::Result.new(
query: 'nonexistent topic',
queries: ['nonexistent topic'],
matches: [],
status: 'weak',
reason: 'no_results'
allow(documentation_search_service).to receive(:search).and_return(
search_result(query: 'nonexistent topic', matches: [], status: 'weak', reason: 'no_results')
)
allow(documentation_search_service).to receive(:search).and_return(search_result)
end
it 'returns no results message' do
@@ -126,14 +123,9 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
context 'with blank query' do
it 'handles empty query' do
search_result = Captain::DocumentationSearchService::Result.new(
query: '',
queries: [],
matches: [],
status: 'weak',
reason: 'no_results'
allow(documentation_search_service).to receive(:search).and_return(
search_result(query: '', matches: [], status: 'weak', reason: 'no_results')
)
allow(documentation_search_service).to receive(:search).and_return(search_result)
result = tool.perform(tool_context, query: '')
expect(result).to include('No relevant FAQs found for: ')
spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb
+18 -12
View File
@@ -167,7 +167,17 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
it 'processes and formats agent result' do
result = service.generate_response(message_history: message_history)
expect(result).to eq({ 'response' => 'Test response', 'agent_name' => nil, 'handoff_tool_called' => false })
expect(result).to include('response' => 'Test response', 'agent_name' => nil, 'handoff_tool_called' => false)
end
it 'includes documentation searches from runner state' do
searches = [{ query: 'billing', matches: [], status: 'weak', reason: 'no_results' }]
runner_context = { state: { documentation_searches: searches } }
allow(mock_result).to receive(:context).and_return(runner_context)
result = service.generate_response(message_history: message_history)
expect(result['documentation_searches']).to eq(searches)
end
context 'when handoff tool was called during agent execution' do
@@ -179,11 +189,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
it 'includes handoff_tool_called flag in response' do
result = service.generate_response(message_history: message_history)
expect(result).to eq({
'response' => 'Let me connect you',
'agent_name' => nil,
'handoff_tool_called' => true
})
expect(result).to include('response' => 'Let me connect you', 'agent_name' => nil, 'handoff_tool_called' => true)
end
end
@@ -208,12 +214,12 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
it 'formats string response correctly' do
result = service.generate_response(message_history: message_history)
expect(result).to eq({
'response' => 'Simple string response',
'reasoning' => 'Processed by agent',
'agent_name' => nil,
'handoff_tool_called' => false
})
expect(result).to include(
'response' => 'Simple string response',
'reasoning' => 'Processed by agent',
'agent_name' => nil,
'handoff_tool_called' => false
)
end
end
spec/enterprise/services/captain/documentation_search_service_spec.rb
+30 -43
View File
@@ -20,59 +20,54 @@ RSpec.describe Captain::DocumentationSearchService do
)
end
def search_match(semantic_distance:, keyword_coverage:, keyword_score: 0, response_record: response)
def search_match(semantic_distance:, response_record: response)
Captain::AssistantResponse::SearchMatch.new(
response: response_record,
semantic_distance: semantic_distance,
keyword_score: keyword_score,
keyword_coverage: keyword_coverage,
matched_terms: [],
retrieval_methods: ['semantic']
semantic_distance: semantic_distance
)
end
def search_result(matches:, status:, reason:, query: 'billing')
{
query: query,
queries: [query],
matches: matches,
status: status,
reason: reason
}
end
describe '#search' do
it 'retries with generic query variants when the original query has weak matches' do
it 'marks weak semantic matches as weak' do
query = 'How do I check limits for my current monthly plan?'
weak_match = search_match(semantic_distance: 0.9, keyword_coverage: 0.0)
sufficient_match = search_match(semantic_distance: 0.8, keyword_coverage: 0.4, keyword_score: 2)
weak_match = search_match(semantic_distance: 0.9)
allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([weak_match])
allow(scope).to receive(:search_with_metadata)
.with('check limits current monthly plan', account_id: 1)
.and_return([sufficient_match])
result = service.search(query)
expect(result.status).to eq('sufficient')
expect(result.reason).to eq('keyword_match')
expect(result.queries).to eq([query, 'check limits current monthly plan'])
expect(result.matches.first.keyword_coverage).to eq(0.4)
expect(result[:status]).to eq('weak')
expect(result[:reason]).to eq('low_retrieval_confidence')
expect(result[:queries]).to eq([query])
end
it 'stops after the first query when retrieval confidence is sufficient' do
it 'marks close semantic matches as found' do
query = 'Where do I find billing settings?'
sufficient_match = search_match(semantic_distance: 0.2, keyword_coverage: 0.0)
close_match = search_match(semantic_distance: 0.2)
allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([sufficient_match])
allow(scope).to receive(:search_with_metadata).with(query, account_id: 1).and_return([close_match])
result = service.search(query)
expect(result.status).to eq('sufficient')
expect(result.reason).to eq('semantic_match')
expect(result.queries).to eq([query])
expect(result[:status]).to eq('found')
expect(result[:reason]).to eq('semantic_match')
expect(result[:queries]).to eq([query])
end
end
describe '.format_for_tool' do
it 'adds a bounded-answer instruction when no documentation is found' do
result = described_class::Result.new(
query: 'unknown topic',
queries: ['unknown topic'],
matches: [],
status: 'weak',
reason: 'no_results'
)
result = search_result(query: 'unknown topic', matches: [], status: 'weak', reason: 'no_results')
formatted_result = described_class.format_for_tool(result, no_results_message: 'No FAQs found')
@@ -81,22 +76,14 @@ RSpec.describe Captain::DocumentationSearchService do
end
end
describe '.record' do
it 'stores search metadata on Current for the response-level sufficiency gate' do
result = described_class::Result.new(
query: 'billing',
queries: ['billing'],
matches: [search_match(semantic_distance: 0.2, keyword_coverage: 0.0)],
status: 'sufficient',
reason: 'semantic_match'
)
describe '.serialize' do
it 'formats search metadata for the response-level support gate' do
result = search_result(matches: [search_match(semantic_distance: 0.2)], status: 'found', reason: 'semantic_match')
described_class.record(result)
serialized_result = described_class.serialize(result)
expect(Current.captain_documentation_searches.first[:status]).to eq('sufficient')
expect(Current.captain_documentation_searches.first[:matches].first[:semantic_distance]).to eq(0.2)
ensure
Current.captain_documentation_searches = nil
expect(serialized_result[:status]).to eq('found')
expect(serialized_result[:matches].first[:semantic_distance]).to eq(0.2)
end
end
end
spec/enterprise/services/captain/tools/search_documentation_service_spec.rb
+18 -19
View File
@@ -40,17 +40,25 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
end
let(:documentable) { create(:captain_document, external_link: external_link) }
let(:recorded_searches) { [] }
let(:service) { described_class.new(assistant, on_search: ->(search) { recorded_searches << search }) }
let(:match) do
Captain::AssistantResponse::SearchMatch.new(
response: response,
semantic_distance: 0.2,
keyword_score: 0,
keyword_coverage: 0.0,
matched_terms: [],
retrieval_methods: ['semantic']
semantic_distance: 0.2
)
end
def search_result(matches:, status:, reason:)
{
query: question,
queries: [question],
matches: matches,
status: status,
reason: reason
}
end
before do
allow(Captain::Llm::TranslateQueryService).to receive(:new).and_return(translate_query_service)
allow(translate_query_service).to receive(:translate).and_return(question)
@@ -62,33 +70,24 @@ RSpec.describe Captain::Tools::SearchDocumentationService do
context 'when matching responses exist' do
it 'returns formatted responses for the search query' do
response.update(documentable: documentable)
search_result = Captain::DocumentationSearchService::Result.new(
query: question,
queries: [question],
matches: [match],
status: 'sufficient',
reason: 'semantic_match'
allow(documentation_search_service).to receive(:search).with(question).and_return(
search_result(matches: [match], status: 'found', reason: 'semantic_match')
)
allow(documentation_search_service).to receive(:search).with(question).and_return(search_result)
result = service.execute(query: question)
expect(result).to include(question)
expect(result).to include(answer)
expect(result).to include(external_link)
expect(recorded_searches.first[:status]).to eq('found')
end
end
context 'when no matching responses exist' do
it 'returns a bounded no-results instruction' do
search_result = Captain::DocumentationSearchService::Result.new(
query: question,
queries: [question],
matches: [],
status: 'weak',
reason: 'no_results'
allow(documentation_search_service).to receive(:search).with(question).and_return(
search_result(matches: [], status: 'weak', reason: 'no_results')
)
allow(documentation_search_service).to receive(:search).with(question).and_return(search_result)
result = service.execute(query: question)