fix(captain): route FAQ matching model
This commit is contained in:
@@ -143,6 +143,20 @@ features:
|
||||
gemini-3-pro,
|
||||
]
|
||||
default: gpt-5.2
|
||||
conversation_faq_matching:
|
||||
models:
|
||||
[
|
||||
gpt-4.1-mini,
|
||||
gpt-5-mini,
|
||||
gpt-4.1,
|
||||
gpt-5.1,
|
||||
gpt-5.2,
|
||||
claude-haiku-4.5,
|
||||
claude-sonnet-4.5,
|
||||
gemini-3-flash,
|
||||
gemini-3-pro,
|
||||
]
|
||||
default: gpt-4.1-mini
|
||||
pdf_faq_generation:
|
||||
models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2]
|
||||
default: gpt-4.1-mini
|
||||
|
||||
@@ -625,6 +625,7 @@ en:
|
||||
label_suggestion: 'Label suggestion'
|
||||
document_faq_generation: 'Document FAQ generation'
|
||||
conversation_faq_generation: 'Conversation FAQ generation'
|
||||
conversation_faq_matching: 'Conversation FAQ matching'
|
||||
help_center_article_generation: 'Help center article generation'
|
||||
onboarding_content_generation: 'Onboarding content generation'
|
||||
help_center_query_translation: 'Help center query translation'
|
||||
|
||||
@@ -4,7 +4,6 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
|
||||
DISTANCE_THRESHOLD = 0.3
|
||||
MATCH_LIMIT = 5
|
||||
LLM_FEATURE = 'conversation_faq_generation'.freeze
|
||||
FAQ_MATCH_MODEL = 'gpt-4.1-mini'.freeze
|
||||
|
||||
def self.language_for(conversation)
|
||||
language = conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s
|
||||
@@ -83,8 +82,9 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
|
||||
existing: { question: existing_record.question, answer: existing_record.answer }
|
||||
}
|
||||
prompt = Captain::Llm::ConversationFaqPromptsService.same_faq
|
||||
response = instrument_llm_call(match_instrumentation_params(prompt, comparison)) do
|
||||
chat(model: FAQ_MATCH_MODEL)
|
||||
faq_match_model = Llm::FeatureRouter.resolve(feature: 'conversation_faq_matching', account: conversation.account)[:model]
|
||||
response = instrument_llm_call(match_instrumentation_params(prompt, comparison, faq_match_model)) do
|
||||
chat(model: faq_match_model)
|
||||
.with_params(response_format: { type: 'json_object' })
|
||||
.with_instructions(prompt)
|
||||
.ask(comparison.to_json)
|
||||
@@ -173,10 +173,10 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
|
||||
}
|
||||
end
|
||||
|
||||
def match_instrumentation_params(prompt, comparison)
|
||||
def match_instrumentation_params(prompt, comparison, faq_match_model)
|
||||
{
|
||||
span_name: 'llm.captain.faq_match',
|
||||
model: FAQ_MATCH_MODEL,
|
||||
model: faq_match_model,
|
||||
temperature: temperature,
|
||||
account_id: conversation.account_id,
|
||||
conversation_id: conversation.display_id,
|
||||
|
||||
@@ -193,6 +193,32 @@ RSpec.describe Captain::Llm::ConversationFaqService do
|
||||
end.to change(Captain::FaqObservation.discarded, :count).by(2)
|
||||
expect(captain_assistant.faq_suggestions.count).to be_zero
|
||||
end
|
||||
|
||||
it 'uses the conversation FAQ matching feature model' do
|
||||
expect(RubyLLM).to receive(:chat).with(
|
||||
model: Llm::Models.default_model_for('conversation_faq_matching')
|
||||
).at_least(:once).and_return(mock_chat)
|
||||
|
||||
service.generate_suggestions
|
||||
end
|
||||
|
||||
it 'uses the account model override for conversation FAQ matching' do
|
||||
conversation.account.update!(captain_models: { 'conversation_faq_matching' => 'gpt-5-mini' })
|
||||
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5-mini').at_least(:once).and_return(mock_chat)
|
||||
|
||||
service.generate_suggestions
|
||||
end
|
||||
|
||||
it 'resolves the matching feature model from the conversation account' do
|
||||
allow(Llm::FeatureRouter).to receive(:resolve).and_call_original
|
||||
expect(Llm::FeatureRouter).to receive(:resolve).with(
|
||||
feature: 'conversation_faq_matching',
|
||||
account: conversation.account
|
||||
).and_call_original
|
||||
|
||||
service.generate_suggestions
|
||||
end
|
||||
end
|
||||
|
||||
context 'when FAQ comparison cannot be completed' do
|
||||
|
||||
@@ -26,9 +26,10 @@ RSpec.describe Llm::Models do
|
||||
end
|
||||
end
|
||||
|
||||
it 'routes document and conversation FAQ generation independently' do
|
||||
it 'routes each FAQ operation independently' do
|
||||
expect(described_class.default_model_for('document_faq_generation')).to eq('gpt-4.1-mini')
|
||||
expect(described_class.default_model_for('conversation_faq_generation')).to eq('gpt-5.2')
|
||||
expect(described_class.default_model_for('conversation_faq_matching')).to eq('gpt-4.1-mini')
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user