diff --git a/enterprise/app/jobs/captain/llm/conversation_faq_job.rb b/enterprise/app/jobs/captain/llm/conversation_faq_job.rb new file mode 100644 index 000000000..fa0fe45dd --- /dev/null +++ b/enterprise/app/jobs/captain/llm/conversation_faq_job.rb @@ -0,0 +1,15 @@ +class Captain::Llm::ConversationFaqJob < ApplicationJob + queue_as :low + + def perform(conversation) + inbox = conversation.inbox + + return unless conversation.resolved? + return unless inbox.captain_active? + + assistant = inbox.captain_assistant + return if assistant.config['feature_faq'].blank? + + Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_suggestions + end +end diff --git a/enterprise/app/listeners/captain_listener.rb b/enterprise/app/listeners/captain_listener.rb index 879fb2b7e..611bf7a68 100644 --- a/enterprise/app/listeners/captain_listener.rb +++ b/enterprise/app/listeners/captain_listener.rb @@ -8,6 +8,6 @@ class CaptainListener < BaseListener return unless conversation.inbox.captain_active? Captain::Llm::ContactNotesService.new(assistant, conversation).generate_and_update_notes if assistant.config['feature_memory'].present? - Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_and_deduplicate if assistant.config['feature_faq'].present? + Captain::Llm::ConversationFaqJob.perform_later(conversation) if assistant.config['feature_faq'].present? end end diff --git a/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb b/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb index dd0df4a6c..155e10a06 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb @@ -57,12 +57,12 @@ class Captain::Llm::ConversationFaqPromptsService PROMPT end - def equivalence_classifier + def same_faq <<~PROMPT - Decide whether the candidate and existing entries represent the same reusable FAQ. + Decide whether the new FAQ and existing FAQ are the same. - Return `same_faq` as true only when both questions have the same user intent and both answers give the same substantive guidance. - Wording, grammar, level of detail, and examples may differ. Return false when either entry adds, removes, contradicts, or changes a condition, + Return `same_faq` as true only when both questions ask the same thing and both answers give the same guidance. + Wording, grammar, level of detail, and examples may differ. Return false when either FAQ adds, removes, contradicts, or changes a condition, policy, procedure, audience, product, plan, time frame, or outcome. Related FAQs are not the same FAQ. When uncertain, return false. Return only valid JSON in this exact structure: diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index 44c9f1960..d0af18311 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -13,7 +13,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService @embedding_service = Captain::Llm::EmbeddingService.new(account_id: conversation.account_id) end - def generate_and_deduplicate + def generate_suggestions return [] if no_human_interaction? generate.map { |faq| route_candidate(faq) } @@ -31,6 +31,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService embedding = embedding_service.get_embedding(candidate_text(faq)) return discard_observation(faq) if matching_record(approved_faqs_for_language, faq, embedding) + return discard_observation(faq) if matching_record(dismissed_suggestions_for_language, faq, embedding) suggestion = matching_record(open_suggestions_for_language, faq, embedding) suggestion ||= assistant.faq_suggestions.create!( @@ -66,8 +67,8 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService candidate: candidate.slice('question', 'answer'), existing: { question: existing_record.question, answer: existing_record.answer } } - prompt = Captain::Llm::ConversationFaqPromptsService.equivalence_classifier - response = instrument_llm_call(equivalence_instrumentation_params(prompt, comparison)) do + prompt = Captain::Llm::ConversationFaqPromptsService.same_faq + response = instrument_llm_call(match_instrumentation_params(prompt, comparison)) do chat .with_params(response_format: { type: 'json_object' }) .with_instructions(prompt) @@ -79,7 +80,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService JSON.parse(response_content).fetch('same_faq', false) == true rescue JSON::ParserError, RubyLLM::Error => e - Rails.logger.error "FAQ equivalence classification failed: #{e.message}" + Rails.logger.error "FAQ match failed: #{e.message}" false end @@ -114,6 +115,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService assistant.faq_suggestions.where(account_id: conversation.account_id).open.by_language(faq_language) end + def dismissed_suggestions_for_language + assistant.faq_suggestions.where(account_id: conversation.account_id).dismissed.by_language(faq_language) + end + def approved_faqs_for_language return assistant.responses.approved if faq_language == account_language @@ -153,14 +158,14 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService } end - def equivalence_instrumentation_params(prompt, comparison) + def match_instrumentation_params(prompt, comparison) { - span_name: 'llm.captain.faq_equivalence', + span_name: 'llm.captain.faq_match', model: model, temperature: temperature, account_id: conversation.account_id, conversation_id: conversation.display_id, - feature_name: 'conversation_faq_deduplication', + feature_name: 'conversation_faq_match', messages: [ { role: 'system', content: prompt }, { role: 'user', content: comparison.to_json } diff --git a/spec/enterprise/listeners/captain_listener_spec.rb b/spec/enterprise/listeners/captain_listener_spec.rb index e076a8b8f..d1fcbe95e 100644 --- a/spec/enterprise/listeners/captain_listener_spec.rb +++ b/spec/enterprise/listeners/captain_listener_spec.rb @@ -29,7 +29,7 @@ describe CaptainListener do .to receive(:new) .with(assistant, conversation) .and_return(instance_double(Captain::Llm::ContactNotesService, generate_and_update_notes: nil)) - expect(Captain::Llm::ConversationFaqService).not_to receive(:new) + expect(Captain::Llm::ConversationFaqJob).not_to receive(:perform_later) listener.conversation_resolved(event) end @@ -42,11 +42,8 @@ describe CaptainListener do assistant.save! end - it 'generates and deduplicates FAQs' do - expect(Captain::Llm::ConversationFaqService) - .to receive(:new) - .with(assistant, conversation) - .and_return(instance_double(Captain::Llm::ConversationFaqService, generate_and_deduplicate: false)) + it 'enqueues FAQ suggestion generation' do + expect(Captain::Llm::ConversationFaqJob).to receive(:perform_later).with(conversation) expect(Captain::Llm::ContactNotesService).not_to receive(:new) listener.conversation_resolved(event) diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb index 9c2021190..5c92992cf 100644 --- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb +++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do allow(mock_chat).to receive(:ask).and_return(mock_response) end - describe '#generate_and_deduplicate' do + describe '#generate_suggestions' do context 'when successful' do before do allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two) @@ -39,7 +39,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do model: Llm::Models.default_model_for('conversation_faq_generation') ).and_return(mock_chat) - described_class.new(captain_assistant, conversation).generate_and_deduplicate + described_class.new(captain_assistant, conversation).generate_suggestions end it 'uses the conversation FAQ default ahead of the legacy global installation model' do @@ -49,7 +49,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do model: Llm::Models.default_model_for('conversation_faq_generation') ).and_return(mock_chat) - described_class.new(captain_assistant, conversation).generate_and_deduplicate + described_class.new(captain_assistant, conversation).generate_suggestions end it 'keeps account conversation FAQ model overrides ahead of the feature default' do @@ -58,7 +58,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-mini').and_return(mock_chat) - described_class.new(captain_assistant, conversation).generate_and_deduplicate + described_class.new(captain_assistant, conversation).generate_suggestions end it 'resolves the feature model from the conversation account' do @@ -67,7 +67,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do account: conversation.account ).and_call_original - described_class.new(captain_assistant, conversation).generate_and_deduplicate + described_class.new(captain_assistant, conversation).generate_suggestions end it 'sends only customer and human support agent messages to the LLM' do @@ -85,7 +85,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do create(:message, conversation: conversation, account: conversation.account, inbox: conversation.inbox, message_type: :activity, content: 'Activity message') - service.generate_and_deduplicate + service.generate_suggestions expected_content = satisfy do |content| content.include?('User: Customer question') && @@ -105,7 +105,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do sender: nil, message_type: :outgoing, content: 'Human replied from the native app', content_attributes: { external_echo: true }) - service.generate_and_deduplicate + service.generate_suggestions expected_content = satisfy do |content| content.include?('User: Customer asks in a native channel') && @@ -134,19 +134,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do block.call end - service.generate_and_deduplicate + service.generate_suggestions end it 'creates suggestions instead of trusted FAQs for valid conversation content' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(captain_assistant.faq_suggestions, :count).by(2) expect(Captain::FaqObservation.count).to eq(2) expect(captain_assistant.responses.count).to be_zero end it 'saves open suggestions with one attached source each' do - service.generate_and_deduplicate + service.generate_suggestions expect( captain_assistant.faq_suggestions.pluck(:question, :answer, :status, :source_count, :language) ).to contain_exactly( @@ -163,12 +163,12 @@ RSpec.describe Captain::Llm::ConversationFaqService do let(:conversation) { create(:conversation) } it 'returns an empty array without generating FAQs' do - expect(service.generate_and_deduplicate).to eq([]) + expect(service.generate_suggestions).to eq([]) end it 'does not call the LLM API' do expect(RubyLLM).not_to receive(:chat) - service.generate_and_deduplicate + service.generate_suggestions end end @@ -177,19 +177,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account, question: 'Similar question', answer: 'Similar answer', embedding: embedding_one) end - let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } + let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } before do existing_response allow(embedding_service).to receive(:get_embedding).and_return(embedding_one) allow(mock_chat).to receive(:ask) do |input| - input.start_with?('{') ? equivalence_response : mock_response + input.start_with?('{') ? match_response : mock_response end end it 'discards candidates the LLM confirms are covered by an approved FAQ' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(Captain::FaqObservation.discarded, :count).by(2) expect(captain_assistant.faq_suggestions.count).to be_zero end @@ -212,19 +212,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do suggestion.update!(source_count: suggestion.observations.attached.count) end end - let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } + let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } before do existing_suggestion allow(embedding_service).to receive(:get_embedding).and_return(embedding_one) allow(mock_chat).to receive(:ask) do |input| - input.start_with?('{') ? equivalence_response : mock_response + input.start_with?('{') ? match_response : mock_response end end it 'attaches the observation and increments the source count' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(existing_suggestion.observations, :count).by(1) expect(existing_suggestion.reload.source_count).to eq(2) expect(captain_assistant.faq_suggestions.count).to eq(1) @@ -245,7 +245,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do it 'creates a separate suggestion in the conversation language' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(captain_assistant.faq_suggestions, :count).by(1) expect(captain_assistant.faq_suggestions.pluck(:language)).to contain_exactly('en', 'pt') @@ -267,19 +267,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do source_count: 1 ) end - let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } + let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } before do existing_suggestion allow(embedding_service).to receive(:get_embedding).and_return(embedding_one) allow(mock_chat).to receive(:ask) do |input| - input.start_with?('{') ? equivalence_response : mock_response + input.start_with?('{') ? match_response : mock_response end end it 'attaches the observation to the existing base-language suggestion' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(existing_suggestion.observations, :count).by(1) expect(existing_suggestion.reload.source_count).to eq(2) @@ -301,7 +301,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do it 'does not discard a candidate in another language' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(captain_assistant.faq_suggestions, :count).by(1) expect(Captain::FaqObservation.discarded.count).to be_zero expect(captain_assistant.faq_suggestions.last.language).to eq('pt') @@ -320,19 +320,19 @@ RSpec.describe Captain::Llm::ConversationFaqService do question: 'Como ativo o recurso?', answer: 'Ative nas configuracoes.', embedding: embedding_one) end - let(:equivalence_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } + let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) } before do existing_response allow(embedding_service).to receive(:get_embedding).and_return(embedding_one) allow(mock_chat).to receive(:ask) do |input| - input.start_with?('{') ? equivalence_response : mock_response + input.start_with?('{') ? match_response : mock_response end end it 'deduplicates against approved FAQs in the same base language' do expect do - service.generate_and_deduplicate + service.generate_suggestions end.to change(Captain::FaqObservation.discarded, :count).by(2) expect(captain_assistant.faq_suggestions.count).to be_zero end @@ -346,7 +346,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do it 'returns empty array and logs the error' do expect(Rails.logger).to receive(:error).with('LLM API Error: API Error') - expect(service.generate_and_deduplicate).to eq([]) + expect(service.generate_suggestions).to eq([]) end end @@ -361,7 +361,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do it 'handles JSON parsing errors gracefully' do expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/) - expect(service.generate_and_deduplicate).to eq([]) + expect(service.generate_suggestions).to eq([]) end end @@ -375,7 +375,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do end it 'returns empty array' do - expect(service.generate_and_deduplicate).to eq([]) + expect(service.generate_suggestions).to eq([]) end end end @@ -398,7 +398,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do .at_least(:once) .and_call_original - service.generate_and_deduplicate + service.generate_suggestions end end @@ -420,7 +420,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do .at_least(:once) .and_call_original - service.generate_and_deduplicate + service.generate_suggestions end end end