akadmin/chatwoot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(captain): refine FAQ suggestion generation

Browse Source
This commit is contained in:
aakashb95
2026-07-17 15:24:50 +05:30
parent 916b510ecf
commit 8600e9b170
6 changed files with 66 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
enterprise/app/jobs/captain/llm/conversation_faq_job.rb
+15
View File
@@ -0,0 +1,15 @@
class Captain::Llm::ConversationFaqJob < ApplicationJob
queue_as :low
def perform(conversation)
inbox = conversation.inbox
return unless conversation.resolved?
return unless inbox.captain_active?
assistant = inbox.captain_assistant
return if assistant.config['feature_faq'].blank?
Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_suggestions
end
end
enterprise/app/listeners/captain_listener.rb
+1 -1
View File
@@ -8,6 +8,6 @@ class CaptainListener < BaseListener
return unless conversation.inbox.captain_active?
Captain::Llm::ContactNotesService.new(assistant, conversation).generate_and_update_notes if assistant.config['feature_memory'].present?
Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_and_deduplicate if assistant.config['feature_faq'].present?
Captain::Llm::ConversationFaqJob.perform_later(conversation) if assistant.config['feature_faq'].present?
end
end
enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb
+4 -4
View File
@@ -57,12 +57,12 @@ class Captain::Llm::ConversationFaqPromptsService
PROMPT
end
def equivalence_classifier
def same_faq
<<~PROMPT
Decide whether the candidate and existing entries represent the same reusable FAQ.
Decide whether the new FAQ and existing FAQ are the same.
Return `same_faq` as true only when both questions have the same user intent and both answers give the same substantive guidance.
Wording, grammar, level of detail, and examples may differ. Return false when either entry adds, removes, contradicts, or changes a condition,
Return `same_faq` as true only when both questions ask the same thing and both answers give the same guidance.
Wording, grammar, level of detail, and examples may differ. Return false when either FAQ adds, removes, contradicts, or changes a condition,
policy, procedure, audience, product, plan, time frame, or outcome. Related FAQs are not the same FAQ. When uncertain, return false.
Return only valid JSON in this exact structure:
enterprise/app/services/captain/llm/conversation_faq_service.rb
+12 -7
View File
@@ -13,7 +13,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
@embedding_service = Captain::Llm::EmbeddingService.new(account_id: conversation.account_id)
end
def generate_and_deduplicate
def generate_suggestions
return [] if no_human_interaction?
generate.map { |faq| route_candidate(faq) }
@@ -31,6 +31,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
embedding = embedding_service.get_embedding(candidate_text(faq))
return discard_observation(faq) if matching_record(approved_faqs_for_language, faq, embedding)
return discard_observation(faq) if matching_record(dismissed_suggestions_for_language, faq, embedding)
suggestion = matching_record(open_suggestions_for_language, faq, embedding)
suggestion ||= assistant.faq_suggestions.create!(
@@ -66,8 +67,8 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
candidate: candidate.slice('question', 'answer'),
existing: { question: existing_record.question, answer: existing_record.answer }
}
prompt = Captain::Llm::ConversationFaqPromptsService.equivalence_classifier
response = instrument_llm_call(equivalence_instrumentation_params(prompt, comparison)) do
prompt = Captain::Llm::ConversationFaqPromptsService.same_faq
response = instrument_llm_call(match_instrumentation_params(prompt, comparison)) do
chat
.with_params(response_format: { type: 'json_object' })
.with_instructions(prompt)
@@ -79,7 +80,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
JSON.parse(response_content).fetch('same_faq', false) == true
rescue JSON::ParserError, RubyLLM::Error => e
Rails.logger.error "FAQ equivalence classification failed: #{e.message}"
Rails.logger.error "FAQ match failed: #{e.message}"
false
end
@@ -114,6 +115,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
assistant.faq_suggestions.where(account_id: conversation.account_id).open.by_language(faq_language)
end
def dismissed_suggestions_for_language
assistant.faq_suggestions.where(account_id: conversation.account_id).dismissed.by_language(faq_language)
end
def approved_faqs_for_language
return assistant.responses.approved if faq_language == account_language
@@ -153,14 +158,14 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
}
end
def equivalence_instrumentation_params(prompt, comparison)
def match_instrumentation_params(prompt, comparison)
{
span_name: 'llm.captain.faq_equivalence',
span_name: 'llm.captain.faq_match',
model: model,
temperature: temperature,
account_id: conversation.account_id,
conversation_id: conversation.display_id,
feature_name: 'conversation_faq_deduplication',
feature_name: 'conversation_faq_match',
messages: [
{ role: 'system', content: prompt },
{ role: 'user', content: comparison.to_json }
spec/enterprise/listeners/captain_listener_spec.rb
+3 -6
View File
@@ -29,7 +29,7 @@ describe CaptainListener do
.to receive(:new)
.with(assistant, conversation)
.and_return(instance_double(Captain::Llm::ContactNotesService, generate_and_update_notes: nil))
expect(Captain::Llm::ConversationFaqService).not_to receive(:new)
expect(Captain::Llm::ConversationFaqJob).not_to receive(:perform_later)
listener.conversation_resolved(event)
end
@@ -42,11 +42,8 @@ describe CaptainListener do
assistant.save!
end
it 'generates and deduplicates FAQs' do
expect(Captain::Llm::ConversationFaqService)
.to receive(:new)
.with(assistant, conversation)
.and_return(instance_double(Captain::Llm::ConversationFaqService, generate_and_deduplicate: false))
it 'enqueues FAQ suggestion generation' do
expect(Captain::Llm::ConversationFaqJob).to receive(:perform_later).with(conversation)
expect(Captain::Llm::ContactNotesService).not_to receive(:new)
listener.conversation_resolved(event)
spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb
+31 -31
View File
@@ -28,7 +28,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
allow(mock_chat).to receive(:ask).and_return(mock_response)
end
describe '#generate_and_deduplicate' do
describe '#generate_suggestions' do
context 'when successful' do
before do
allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two)
@@ -39,7 +39,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
model: Llm::Models.default_model_for('conversation_faq_generation')
).and_return(mock_chat)
described_class.new(captain_assistant, conversation).generate_and_deduplicate
described_class.new(captain_assistant, conversation).generate_suggestions
end
it 'uses the conversation FAQ default ahead of the legacy global installation model' do
@@ -49,7 +49,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
model: Llm::Models.default_model_for('conversation_faq_generation')
).and_return(mock_chat)
described_class.new(captain_assistant, conversation).generate_and_deduplicate
described_class.new(captain_assistant, conversation).generate_suggestions
end
it 'keeps account conversation FAQ model overrides ahead of the feature default' do
@@ -58,7 +58,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-mini').and_return(mock_chat)
described_class.new(captain_assistant, conversation).generate_and_deduplicate
described_class.new(captain_assistant, conversation).generate_suggestions
end
it 'resolves the feature model from the conversation account' do
@@ -67,7 +67,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
account: conversation.account
).and_call_original
described_class.new(captain_assistant, conversation).generate_and_deduplicate
described_class.new(captain_assistant, conversation).generate_suggestions
end
it 'sends only customer and human support agent messages to the LLM' do
@@ -85,7 +85,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
create(:message, conversation: conversation, account: conversation.account, inbox: conversation.inbox,
message_type: :activity, content: 'Activity message')
service.generate_and_deduplicate
service.generate_suggestions
expected_content = satisfy do |content|
content.include?('User: Customer question') &&
@@ -105,7 +105,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
sender: nil, message_type: :outgoing, content: 'Human replied from the native app',
content_attributes: { external_echo: true })
service.generate_and_deduplicate
service.generate_suggestions
expected_content = satisfy do |content|
content.include?('User: Customer asks in a native channel') &&
@@ -134,19 +134,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do
block.call
end
service.generate_and_deduplicate
service.generate_suggestions
end
it 'creates suggestions instead of trusted FAQs for valid conversation content' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(captain_assistant.faq_suggestions, :count).by(2)
expect(Captain::FaqObservation.count).to eq(2)
expect(captain_assistant.responses.count).to be_zero
end
it 'saves open suggestions with one attached source each' do
service.generate_and_deduplicate
service.generate_suggestions
expect(
captain_assistant.faq_suggestions.pluck(:question, :answer, :status, :source_count, :language)
).to contain_exactly(
@@ -163,12 +163,12 @@ RSpec.describe Captain::Llm::ConversationFaqService do
let(:conversation) { create(:conversation) }
it 'returns an empty array without generating FAQs' do
expect(service.generate_and_deduplicate).to eq([])
expect(service.generate_suggestions).to eq([])
end
it 'does not call the LLM API' do
expect(RubyLLM).not_to receive(:chat)
service.generate_and_deduplicate
service.generate_suggestions
end
end
@@ -177,19 +177,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do
create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account,
question: 'Similar question', answer: 'Similar answer', embedding: embedding_one)
end
let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
before do
existing_response
allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
allow(mock_chat).to receive(:ask) do |input|
input.start_with?('{') ? equivalence_response : mock_response
input.start_with?('{') ? match_response : mock_response
end
end
it 'discards candidates the LLM confirms are covered by an approved FAQ' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(Captain::FaqObservation.discarded, :count).by(2)
expect(captain_assistant.faq_suggestions.count).to be_zero
end
@@ -212,19 +212,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do
suggestion.update!(source_count: suggestion.observations.attached.count)
end
end
let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
before do
existing_suggestion
allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
allow(mock_chat).to receive(:ask) do |input|
input.start_with?('{') ? equivalence_response : mock_response
input.start_with?('{') ? match_response : mock_response
end
end
it 'attaches the observation and increments the source count' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(existing_suggestion.observations, :count).by(1)
expect(existing_suggestion.reload.source_count).to eq(2)
expect(captain_assistant.faq_suggestions.count).to eq(1)
@@ -245,7 +245,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'creates a separate suggestion in the conversation language' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(captain_assistant.faq_suggestions, :count).by(1)
expect(captain_assistant.faq_suggestions.pluck(:language)).to contain_exactly('en', 'pt')
@@ -267,19 +267,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do
source_count: 1
)
end
let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
before do
existing_suggestion
allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
allow(mock_chat).to receive(:ask) do |input|
input.start_with?('{') ? equivalence_response : mock_response
input.start_with?('{') ? match_response : mock_response
end
end
it 'attaches the observation to the existing base-language suggestion' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(existing_suggestion.observations, :count).by(1)
expect(existing_suggestion.reload.source_count).to eq(2)
@@ -301,7 +301,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'does not discard a candidate in another language' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(captain_assistant.faq_suggestions, :count).by(1)
expect(Captain::FaqObservation.discarded.count).to be_zero
expect(captain_assistant.faq_suggestions.last.language).to eq('pt')
@@ -320,19 +320,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do
question: 'Como ativo o recurso?', answer: 'Ative nas configuracoes.',
embedding: embedding_one)
end
let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
before do
existing_response
allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
allow(mock_chat).to receive(:ask) do |input|
input.start_with?('{') ? equivalence_response : mock_response
input.start_with?('{') ? match_response : mock_response
end
end
it 'deduplicates against approved FAQs in the same base language' do
expect do
service.generate_and_deduplicate
service.generate_suggestions
end.to change(Captain::FaqObservation.discarded, :count).by(2)
expect(captain_assistant.faq_suggestions.count).to be_zero
end
@@ -346,7 +346,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'returns empty array and logs the error' do
expect(Rails.logger).to receive(:error).with('LLM API Error: API Error')
expect(service.generate_and_deduplicate).to eq([])
expect(service.generate_suggestions).to eq([])
end
end
@@ -361,7 +361,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
it 'handles JSON parsing errors gracefully' do
expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/)
expect(service.generate_and_deduplicate).to eq([])
expect(service.generate_suggestions).to eq([])
end
end
@@ -375,7 +375,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
end
it 'returns empty array' do
expect(service.generate_and_deduplicate).to eq([])
expect(service.generate_suggestions).to eq([])
end
end
end
@@ -398,7 +398,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
.at_least(:once)
.and_call_original
service.generate_and_deduplicate
service.generate_suggestions
end
end
@@ -420,7 +420,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
.at_least(:once)
.and_call_original
service.generate_and_deduplicate
service.generate_suggestions
end
end
end