diff --git a/config/llm.yml b/config/llm.yml index 2be3d86c7..1b702ab6a 100644 --- a/config/llm.yml +++ b/config/llm.yml @@ -143,6 +143,20 @@ features: gemini-3-pro, ] default: gpt-5.2 + conversation_faq_matching: + models: + [ + gpt-4.1-mini, + gpt-5-mini, + gpt-4.1, + gpt-5.1, + gpt-5.2, + claude-haiku-4.5, + claude-sonnet-4.5, + gemini-3-flash, + gemini-3-pro, + ] + default: gpt-4.1-mini pdf_faq_generation: models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2] default: gpt-4.1-mini diff --git a/config/locales/en.yml b/config/locales/en.yml index 735d52205..4c7e74788 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -625,6 +625,7 @@ en: label_suggestion: 'Label suggestion' document_faq_generation: 'Document FAQ generation' conversation_faq_generation: 'Conversation FAQ generation' + conversation_faq_matching: 'Conversation FAQ matching' help_center_article_generation: 'Help center article generation' onboarding_content_generation: 'Onboarding content generation' help_center_query_translation: 'Help center query translation' diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index e3ea9b4c8..5ace3d772 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -4,7 +4,6 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService DISTANCE_THRESHOLD = 0.3 MATCH_LIMIT = 5 LLM_FEATURE = 'conversation_faq_generation'.freeze - FAQ_MATCH_MODEL = 'gpt-4.1-mini'.freeze def self.language_for(conversation) language = conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s @@ -83,8 +82,9 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService existing: { question: existing_record.question, answer: existing_record.answer } } prompt = Captain::Llm::ConversationFaqPromptsService.same_faq - response = instrument_llm_call(match_instrumentation_params(prompt, comparison)) do - chat(model: FAQ_MATCH_MODEL) + faq_match_model = Llm::FeatureRouter.resolve(feature: 'conversation_faq_matching', account: conversation.account)[:model] + response = instrument_llm_call(match_instrumentation_params(prompt, comparison, faq_match_model)) do + chat(model: faq_match_model) .with_params(response_format: { type: 'json_object' }) .with_instructions(prompt) .ask(comparison.to_json) @@ -173,10 +173,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService } end - def match_instrumentation_params(prompt, comparison) + def match_instrumentation_params(prompt, comparison, faq_match_model) { span_name: 'llm.captain.faq_match', - model: FAQ_MATCH_MODEL, + model: faq_match_model, temperature: temperature, account_id: conversation.account_id, conversation_id: conversation.display_id, diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb index 4edf3f7a1..4a028406a 100644 --- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb +++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb @@ -193,6 +193,32 @@ RSpec.describe Captain::Llm::ConversationFaqService do end.to change(Captain::FaqObservation.discarded, :count).by(2) expect(captain_assistant.faq_suggestions.count).to be_zero end + + it 'uses the conversation FAQ matching feature model' do + expect(RubyLLM).to receive(:chat).with( + model: Llm::Models.default_model_for('conversation_faq_matching') + ).at_least(:once).and_return(mock_chat) + + service.generate_suggestions + end + + it 'uses the account model override for conversation FAQ matching' do + conversation.account.update!(captain_models: { 'conversation_faq_matching' => 'gpt-5-mini' }) + + expect(RubyLLM).to receive(:chat).with(model: 'gpt-5-mini').at_least(:once).and_return(mock_chat) + + service.generate_suggestions + end + + it 'resolves the matching feature model from the conversation account' do + allow(Llm::FeatureRouter).to receive(:resolve).and_call_original + expect(Llm::FeatureRouter).to receive(:resolve).with( + feature: 'conversation_faq_matching', + account: conversation.account + ).and_call_original + + service.generate_suggestions + end end context 'when FAQ comparison cannot be completed' do diff --git a/spec/lib/llm/models_spec.rb b/spec/lib/llm/models_spec.rb index 5692bee9c..815479e7c 100644 --- a/spec/lib/llm/models_spec.rb +++ b/spec/lib/llm/models_spec.rb @@ -26,9 +26,10 @@ RSpec.describe Llm::Models do end end - it 'routes document and conversation FAQ generation independently' do + it 'routes each FAQ operation independently' do expect(described_class.default_model_for('document_faq_generation')).to eq('gpt-4.1-mini') expect(described_class.default_model_for('conversation_faq_generation')).to eq('gpt-5.2') + expect(described_class.default_model_for('conversation_faq_matching')).to eq('gpt-4.1-mini') end end