diff --git a/config/llm.yml b/config/llm.yml index 2be3d86c7..1b702ab6a 100644 --- a/config/llm.yml +++ b/config/llm.yml @@ -143,6 +143,20 @@ features: gemini-3-pro, ] default: gpt-5.2 + conversation_faq_matching: + models: + [ + gpt-4.1-mini, + gpt-5-mini, + gpt-4.1, + gpt-5.1, + gpt-5.2, + claude-haiku-4.5, + claude-sonnet-4.5, + gemini-3-flash, + gemini-3-pro, + ] + default: gpt-4.1-mini pdf_faq_generation: models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2] default: gpt-4.1-mini diff --git a/config/locales/en.yml b/config/locales/en.yml index 735d52205..4c7e74788 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -625,6 +625,7 @@ en: label_suggestion: 'Label suggestion' document_faq_generation: 'Document FAQ generation' conversation_faq_generation: 'Conversation FAQ generation' + conversation_faq_matching: 'Conversation FAQ matching' help_center_article_generation: 'Help center article generation' onboarding_content_generation: 'Onboarding content generation' help_center_query_translation: 'Help center query translation' diff --git a/enterprise/app/jobs/captain/llm/conversation_faq_job.rb b/enterprise/app/jobs/captain/llm/conversation_faq_job.rb index fa0fe45dd..9425c4b88 100644 --- a/enterprise/app/jobs/captain/llm/conversation_faq_job.rb +++ b/enterprise/app/jobs/captain/llm/conversation_faq_job.rb @@ -1,15 +1,30 @@ -class Captain::Llm::ConversationFaqJob < ApplicationJob +class Captain::Llm::ConversationFaqJob < MutexApplicationJob queue_as :low - def perform(conversation) + LOCK_TIMEOUT = 10.minutes + + retry_on_lock_conflict wait: 30.seconds, attempts: 30 + + def perform(conversation, assistant) inbox = conversation.inbox return unless conversation.resolved? return unless inbox.captain_active? - assistant = inbox.captain_assistant return if assistant.config['feature_faq'].blank? - Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_suggestions + with_lock(lock_key(assistant, conversation), LOCK_TIMEOUT) do + Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_suggestions + end + end + + private + + def lock_key(assistant, conversation) + format( + ::Redis::Alfred::CAPTAIN_CONVERSATION_FAQ_MUTEX, + assistant_id: assistant.id, + language: Captain::Llm::ConversationFaqService.language_for(conversation) + ) end end diff --git a/enterprise/app/listeners/captain_listener.rb b/enterprise/app/listeners/captain_listener.rb index 611bf7a68..cd6d8fd1f 100644 --- a/enterprise/app/listeners/captain_listener.rb +++ b/enterprise/app/listeners/captain_listener.rb @@ -8,6 +8,6 @@ class CaptainListener < BaseListener return unless conversation.inbox.captain_active? Captain::Llm::ContactNotesService.new(assistant, conversation).generate_and_update_notes if assistant.config['feature_memory'].present? - Captain::Llm::ConversationFaqJob.perform_later(conversation) if assistant.config['feature_faq'].present? + Captain::Llm::ConversationFaqJob.perform_later(conversation, assistant) if assistant.config['feature_faq'].present? end end diff --git a/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb b/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb index 155e10a06..2e1c68670 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb @@ -11,7 +11,7 @@ class Captain::Llm::ConversationFaqPromptsService - Base every FAQ strictly on information stated in the human support agent messages. Do not infer, generalize, or add external knowledge. - A human support agent must state every fact used in the FAQ answer. Customer messages cannot supply missing answer facts. - The human support agent must provide the final answer. If the agent only greets, asks clarifying questions, asks for contact details, promises to check, shares an attachment, or transfers the conversation, return: `{"faqs":[]}`. - - For each FAQ, first identify the exact human support agent message that fully answers it. If no single human agent message gives a complete public answer, remove that FAQ. + - For each FAQ, identify the human support agent message or messages that together provide a complete public answer to the same question. Combine facts only across related agent messages; never combine separate questions or unrelated topics. If those messages do not provide a complete public answer, remove that FAQ. ## Decision gate Return `{"faqs":[]}` unless every generated FAQ can pass all of these checks: diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index 889b0a151..9abbe12a1 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -4,7 +4,20 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService DISTANCE_THRESHOLD = 0.3 MATCH_LIMIT = 5 LLM_FEATURE = 'conversation_faq_generation'.freeze - FAQ_MATCH_MODEL = 'gpt-4.1-mini'.freeze + + def self.language_for(conversation) + language = conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s + normalize_language(language) + end + + def self.normalize_language(language) + language.to_s.tr('-', '_').split('_').first.downcase + end + + def self.account_language_for(account) + normalize_language(account.locale.presence || I18n.default_locale.to_s) + end + private_class_method :normalize_language def initialize(assistant, conversation) super(feature: LLM_FEATURE, account: conversation.account, fallback_model: Llm::Models.default_model_for(LLM_FEATURE)) @@ -69,20 +82,21 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService existing: { question: existing_record.question, answer: existing_record.answer } } prompt = Captain::Llm::ConversationFaqPromptsService.same_faq - response = instrument_llm_call(match_instrumentation_params(prompt, comparison)) do - chat(model: FAQ_MATCH_MODEL) + faq_match_model = Llm::FeatureRouter.resolve(feature: 'conversation_faq_matching', account: conversation.account)[:model] + response = instrument_llm_call(match_instrumentation_params(prompt, comparison, faq_match_model)) do + chat(model: faq_match_model) .with_params(response_format: { type: 'json_object' }) .with_instructions(prompt) .ask(comparison.to_json) end - response_content = sanitize_json_response(response.content) - return false if response_content.blank? + same_faq = JSON.parse(sanitize_json_response(response.content)).fetch('same_faq') + raise TypeError, 'same_faq must be a boolean' unless [true, false].include?(same_faq) - JSON.parse(response_content).fetch('same_faq', false) == true - rescue JSON::ParserError, RubyLLM::Error => e + same_faq + rescue JSON::ParserError, KeyError, TypeError, RubyLLM::Error => e Rails.logger.error "FAQ match failed: #{e.message}" - false + raise end def attach_observation(suggestion, faq) @@ -161,10 +175,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService } end - def match_instrumentation_params(prompt, comparison) + def match_instrumentation_params(prompt, comparison, faq_match_model) { span_name: 'llm.captain.faq_match', - model: FAQ_MATCH_MODEL, + model: faq_match_model, temperature: temperature, account_id: conversation.account_id, conversation_id: conversation.display_id, @@ -182,15 +196,11 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService end def faq_language - @faq_language ||= normalize_language(conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s) + @faq_language ||= self.class.language_for(conversation) end def account_language - @account_language ||= normalize_language(conversation.account.locale.presence || I18n.default_locale.to_s) - end - - def normalize_language(language) - language.to_s.tr('-', '_').split('_').first.downcase + @account_language ||= self.class.account_language_for(conversation.account) end def language_name(language) diff --git a/lib/redis/redis_keys.rb b/lib/redis/redis_keys.rb index b782270ef..c71f30f10 100644 --- a/lib/redis/redis_keys.rb +++ b/lib/redis/redis_keys.rb @@ -89,6 +89,7 @@ module Redis::RedisKeys WHATSAPP_MESSAGE_MUTEX = 'WHATSAPP_MESSAGE_CREATE_LOCK::%s::%s'.freeze CRM_PROCESS_MUTEX = 'CRM_PROCESS_MUTEX::%s'.freeze CAPTAIN_DOCUMENT_SYNC_MUTEX = 'CAPTAIN_DOCUMENT_SYNC_LOCK::%s'.freeze + CAPTAIN_CONVERSATION_FAQ_MUTEX = 'CAPTAIN_CONVERSATION_FAQ_LOCK::%s::%s'.freeze ## Auto Assignment Keys # Track conversation assignments to agents for rate limiting diff --git a/spec/enterprise/jobs/captain/llm/conversation_faq_job_spec.rb b/spec/enterprise/jobs/captain/llm/conversation_faq_job_spec.rb new file mode 100644 index 000000000..8925363e5 --- /dev/null +++ b/spec/enterprise/jobs/captain/llm/conversation_faq_job_spec.rb @@ -0,0 +1,57 @@ +require 'rails_helper' + +RSpec.describe Captain::Llm::ConversationFaqJob, type: :job do + let(:account) { create(:account) } + let(:inbox) { create(:inbox, account: account) } + let(:assistant) { create(:captain_assistant, account: account, config: { feature_faq: true }) } + let(:conversation) { create(:conversation, account: account, inbox: inbox, first_reply_created_at: Time.zone.now) } + let(:faq_service) { instance_double(Captain::Llm::ConversationFaqService, generate_suggestions: []) } + let(:lock_manager) { instance_double(Redis::LockManager, lock: true, unlock: true) } + let(:lock_key) { "CAPTAIN_CONVERSATION_FAQ_LOCK::#{assistant.id}::en" } + + before do + create(:captain_inbox, inbox: inbox, captain_assistant: assistant) + conversation.update!(status: :resolved) + allow(Redis::LockManager).to receive(:new).and_return(lock_manager) + allow(Captain::Llm::ConversationFaqService).to receive(:new).and_return(faq_service) + end + + describe '#perform' do + it 'uses the assistant captured when the job was enqueued' do + replacement_assistant = create(:captain_assistant, account: account, config: { feature_faq: true }) + inbox.captain_inbox.update!(captain_assistant: replacement_assistant) + + expect(inbox.reload.captain_assistant).to eq(replacement_assistant) + expect(Captain::Llm::ConversationFaqService).to receive(:new) + .with(assistant, conversation) + .and_return(faq_service) + expect(faq_service).to receive(:generate_suggestions) + + described_class.perform_now(conversation, assistant) + end + + it 'locks FAQ grouping for the assistant and normalized language' do + conversation.update!(additional_attributes: { conversation_language: 'pt-BR' }) + expected_key = "CAPTAIN_CONVERSATION_FAQ_LOCK::#{assistant.id}::pt" + + expect(lock_manager).to receive(:lock).with(expected_key, described_class::LOCK_TIMEOUT).and_return(true) + expect(lock_manager).to receive(:unlock).with(expected_key) + + described_class.perform_now(conversation, assistant) + end + + context 'when another job holds the grouping lock' do + before do + allow(lock_manager).to receive(:lock).with(lock_key, described_class::LOCK_TIMEOUT).and_return(false) + end + + it 'does not generate suggestions concurrently' do + expect(Captain::Llm::ConversationFaqService).not_to receive(:new) + + expect do + described_class.new.perform(conversation, assistant) + end.to raise_error(MutexApplicationJob::LockAcquisitionError) + end + end + end +end diff --git a/spec/enterprise/listeners/captain_listener_spec.rb b/spec/enterprise/listeners/captain_listener_spec.rb index d1fcbe95e..4e25c40c4 100644 --- a/spec/enterprise/listeners/captain_listener_spec.rb +++ b/spec/enterprise/listeners/captain_listener_spec.rb @@ -43,7 +43,7 @@ describe CaptainListener do end it 'enqueues FAQ suggestion generation' do - expect(Captain::Llm::ConversationFaqJob).to receive(:perform_later).with(conversation) + expect(Captain::Llm::ConversationFaqJob).to receive(:perform_later).with(conversation, assistant) expect(Captain::Llm::ContactNotesService).not_to receive(:new) listener.conversation_resolved(event) diff --git a/spec/enterprise/services/captain/llm/conversation_faq_prompts_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_prompts_service_spec.rb new file mode 100644 index 000000000..d1d5fcfb3 --- /dev/null +++ b/spec/enterprise/services/captain/llm/conversation_faq_prompts_service_spec.rb @@ -0,0 +1,13 @@ +require 'rails_helper' + +RSpec.describe Captain::Llm::ConversationFaqPromptsService do + describe '.generator' do + it 'allows a complete FAQ answer to use several related agent messages' do + prompt = described_class.generator + + expect(prompt).to include('message or messages that together provide a complete public answer') + expect(prompt).to include('Combine facts only across related agent messages') + expect(prompt).not_to include('no single human agent message') + end + end +end diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb index 5c92992cf..4a028406a 100644 --- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb +++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb @@ -193,6 +193,117 @@ RSpec.describe Captain::Llm::ConversationFaqService do end.to change(Captain::FaqObservation.discarded, :count).by(2) expect(captain_assistant.faq_suggestions.count).to be_zero end + + it 'uses the conversation FAQ matching feature model' do + expect(RubyLLM).to receive(:chat).with( + model: Llm::Models.default_model_for('conversation_faq_matching') + ).at_least(:once).and_return(mock_chat) + + service.generate_suggestions + end + + it 'uses the account model override for conversation FAQ matching' do + conversation.account.update!(captain_models: { 'conversation_faq_matching' => 'gpt-5-mini' }) + + expect(RubyLLM).to receive(:chat).with(model: 'gpt-5-mini').at_least(:once).and_return(mock_chat) + + service.generate_suggestions + end + + it 'resolves the matching feature model from the conversation account' do + allow(Llm::FeatureRouter).to receive(:resolve).and_call_original + expect(Llm::FeatureRouter).to receive(:resolve).with( + feature: 'conversation_faq_matching', + account: conversation.account + ).and_call_original + + service.generate_suggestions + end + end + + context 'when FAQ comparison cannot be completed' do + let(:existing_response) do + create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account, + question: 'Similar question', answer: 'Similar answer', embedding: embedding_one) + end + let(:comparison_response) { instance_double(RubyLLM::Message, content: comparison_response_content) } + let(:comparison_response_content) { 'invalid json' } + + before do + existing_response + allow(embedding_service).to receive(:get_embedding).and_return(embedding_one) + allow(mock_chat).to receive(:ask) do |input| + input.start_with?('{') ? comparison_response : mock_response + end + allow(Rails.logger).to receive(:error) + end + + it 'raises when the comparison response is malformed' do + expect do + service.generate_suggestions + end.to raise_error(JSON::ParserError) + expect(captain_assistant.faq_suggestions.count).to be_zero + end + + context 'when the response omits the comparison result' do + let(:comparison_response_content) { {}.to_json } + + it 'raises instead of treating the response as a non-match' do + expect do + service.generate_suggestions + end.to raise_error(KeyError) + expect(captain_assistant.faq_suggestions.count).to be_zero + end + end + + context 'when the comparison result is not a boolean' do + let(:comparison_response_content) { { same_faq: 'false' }.to_json } + + it 'raises instead of treating the response as a non-match' do + expect do + service.generate_suggestions + end.to raise_error(TypeError, 'same_faq must be a boolean') + expect(captain_assistant.faq_suggestions.count).to be_zero + end + end + + context 'when the comparison provider fails' do + before do + allow(mock_chat).to receive(:ask) do |input| + raise RubyLLM::Error.new(nil, 'API Error') if input.start_with?('{') + + mock_response + end + end + + it 'raises instead of treating the failure as a non-match' do + expect do + service.generate_suggestions + end.to raise_error(RubyLLM::Error) + expect(captain_assistant.faq_suggestions.count).to be_zero + end + end + end + + context 'when the classifier confirms a non-match' do + let(:sample_faqs) { [{ 'question' => 'How can I use the feature?', 'answer' => 'Enable it in settings.' }] } + let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: false }.to_json) } + + before do + create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account, + question: 'How do I enable the feature?', answer: 'Turn it on in settings.', + embedding: embedding_one) + allow(embedding_service).to receive(:get_embedding).and_return(embedding_one) + allow(mock_chat).to receive(:ask) do |input| + input.start_with?('{') ? match_response : mock_response + end + end + + it 'creates a new suggestion' do + expect do + service.generate_suggestions + end.to change(captain_assistant.faq_suggestions, :count).by(1) + end end context 'when an open suggestion is the same FAQ' do diff --git a/spec/lib/llm/models_spec.rb b/spec/lib/llm/models_spec.rb index 5692bee9c..815479e7c 100644 --- a/spec/lib/llm/models_spec.rb +++ b/spec/lib/llm/models_spec.rb @@ -26,9 +26,10 @@ RSpec.describe Llm::Models do end end - it 'routes document and conversation FAQ generation independently' do + it 'routes each FAQ operation independently' do expect(described_class.default_model_for('document_faq_generation')).to eq('gpt-4.1-mini') expect(described_class.default_model_for('conversation_faq_generation')).to eq('gpt-5.2') + expect(described_class.default_model_for('conversation_faq_matching')).to eq('gpt-4.1-mini') end end