fix(captain): route FAQ matching model

This commit is contained in:
aakashb95
2026-07-22 13:41:21 +05:30
parent c183b2aa80
commit 9f658fbfaa
5 changed files with 48 additions and 6 deletions
+14
View File
@@ -143,6 +143,20 @@ features:
gemini-3-pro,
]
default: gpt-5.2
conversation_faq_matching:
models:
[
gpt-4.1-mini,
gpt-5-mini,
gpt-4.1,
gpt-5.1,
gpt-5.2,
claude-haiku-4.5,
claude-sonnet-4.5,
gemini-3-flash,
gemini-3-pro,
]
default: gpt-4.1-mini
pdf_faq_generation:
models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2]
default: gpt-4.1-mini
+1
View File
@@ -625,6 +625,7 @@ en:
label_suggestion: 'Label suggestion'
document_faq_generation: 'Document FAQ generation'
conversation_faq_generation: 'Conversation FAQ generation'
conversation_faq_matching: 'Conversation FAQ matching'
help_center_article_generation: 'Help center article generation'
onboarding_content_generation: 'Onboarding content generation'
help_center_query_translation: 'Help center query translation'
@@ -4,7 +4,6 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
DISTANCE_THRESHOLD = 0.3
MATCH_LIMIT = 5
LLM_FEATURE = 'conversation_faq_generation'.freeze
FAQ_MATCH_MODEL = 'gpt-4.1-mini'.freeze
def self.language_for(conversation)
language = conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s
@@ -83,8 +82,9 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
existing: { question: existing_record.question, answer: existing_record.answer }
}
prompt = Captain::Llm::ConversationFaqPromptsService.same_faq
response = instrument_llm_call(match_instrumentation_params(prompt, comparison)) do
chat(model: FAQ_MATCH_MODEL)
faq_match_model = Llm::FeatureRouter.resolve(feature: 'conversation_faq_matching', account: conversation.account)[:model]
response = instrument_llm_call(match_instrumentation_params(prompt, comparison, faq_match_model)) do
chat(model: faq_match_model)
.with_params(response_format: { type: 'json_object' })
.with_instructions(prompt)
.ask(comparison.to_json)
@@ -173,10 +173,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
}
end
def match_instrumentation_params(prompt, comparison)
def match_instrumentation_params(prompt, comparison, faq_match_model)
{
span_name: 'llm.captain.faq_match',
model: FAQ_MATCH_MODEL,
model: faq_match_model,
temperature: temperature,
account_id: conversation.account_id,
conversation_id: conversation.display_id,
@@ -193,6 +193,32 @@ RSpec.describe Captain::Llm::ConversationFaqService do
end.to change(Captain::FaqObservation.discarded, :count).by(2)
expect(captain_assistant.faq_suggestions.count).to be_zero
end
it 'uses the conversation FAQ matching feature model' do
expect(RubyLLM).to receive(:chat).with(
model: Llm::Models.default_model_for('conversation_faq_matching')
).at_least(:once).and_return(mock_chat)
service.generate_suggestions
end
it 'uses the account model override for conversation FAQ matching' do
conversation.account.update!(captain_models: { 'conversation_faq_matching' => 'gpt-5-mini' })
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5-mini').at_least(:once).and_return(mock_chat)
service.generate_suggestions
end
it 'resolves the matching feature model from the conversation account' do
allow(Llm::FeatureRouter).to receive(:resolve).and_call_original
expect(Llm::FeatureRouter).to receive(:resolve).with(
feature: 'conversation_faq_matching',
account: conversation.account
).and_call_original
service.generate_suggestions
end
end
context 'when FAQ comparison cannot be completed' do
+2 -1
View File
@@ -26,9 +26,10 @@ RSpec.describe Llm::Models do
end
end
it 'routes document and conversation FAQ generation independently' do
it 'routes each FAQ operation independently' do
expect(described_class.default_model_for('document_faq_generation')).to eq('gpt-4.1-mini')
expect(described_class.default_model_for('conversation_faq_generation')).to eq('gpt-5.2')
expect(described_class.default_model_for('conversation_faq_matching')).to eq('gpt-4.1-mini')
end
end