diff --git a/app/controllers/api/v1/accounts/captain/preferences_controller.rb b/app/controllers/api/v1/accounts/captain/preferences_controller.rb index 156c031fa..a62ec115c 100644 --- a/app/controllers/api/v1/accounts/captain/preferences_controller.rb +++ b/app/controllers/api/v1/accounts/captain/preferences_controller.rb @@ -47,17 +47,15 @@ class Api::V1::Accounts::Captain::PreferencesController < Api::V1::Accounts::Bas end def permitted_captain_models - params.require(:captain_models).permit( - :editor, :assistant, :copilot, :label_suggestion, - :audio_transcription, :help_center_search - ).to_h.stringify_keys + params.require(:captain_models).permit(*captain_feature_keys).to_h.stringify_keys end def permitted_captain_features - params.require(:captain_features).permit( - :editor, :assistant, :copilot, :label_suggestion, - :audio_transcription, :help_center_search - ).to_h.stringify_keys + params.require(:captain_features).permit(*captain_feature_keys).to_h.stringify_keys + end + + def captain_feature_keys + Llm::Models.feature_keys.map(&:to_sym) end def features_with_account_preferences diff --git a/app/models/concerns/account_settings_schema.rb b/app/models/concerns/account_settings_schema.rb index c3242fa30..755ea009e 100644 --- a/app/models/concerns/account_settings_schema.rb +++ b/app/models/concerns/account_settings_schema.rb @@ -1,6 +1,9 @@ module AccountSettingsSchema extend ActiveSupport::Concern + CAPTAIN_MODEL_PROPERTIES = Llm::Models.feature_keys.index_with { { 'type': %w[string null] } }.freeze + CAPTAIN_FEATURE_PROPERTIES = Llm::Models.feature_keys.index_with { { 'type': %w[boolean null] } }.freeze + SETTINGS_PARAMS_SCHEMA = { 'type': 'object', 'properties': @@ -19,26 +22,12 @@ module AccountSettingsSchema }, 'captain_models': { 'type': %w[object null], - 'properties': { - 'editor': { 'type': %w[string null] }, - 'assistant': { 'type': %w[string null] }, - 'copilot': { 'type': %w[string null] }, - 'label_suggestion': { 'type': %w[string null] }, - 'audio_transcription': { 'type': %w[string null] }, - 'help_center_search': { 'type': %w[string null] } - }, + 'properties': CAPTAIN_MODEL_PROPERTIES, 'additionalProperties': false }, 'captain_features': { 'type': %w[object null], - 'properties': { - 'editor': { 'type': %w[boolean null] }, - 'assistant': { 'type': %w[boolean null] }, - 'copilot': { 'type': %w[boolean null] }, - 'label_suggestion': { 'type': %w[boolean null] }, - 'audio_transcription': { 'type': %w[boolean null] }, - 'help_center_search': { 'type': %w[boolean null] } - }, + 'properties': CAPTAIN_FEATURE_PROPERTIES, 'additionalProperties': false } }, diff --git a/config/llm.yml b/config/llm.yml index 204b0a1b4..29f621c7d 100644 --- a/config/llm.yml +++ b/config/llm.yml @@ -109,6 +109,23 @@ features: models: [gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini, gemini-3-flash, claude-haiku-4.5] default: gpt-4.1-nano + document_faq_generation: + models: + [ + gpt-4.1-mini, + gpt-5-mini, + gpt-4.1, + gpt-5.1, + gpt-5.2, + claude-haiku-4.5, + claude-sonnet-4.5, + gemini-3-flash, + gemini-3-pro, + ] + default: gpt-4.1-mini + pdf_faq_generation: + models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2] + default: gpt-4.1-mini audio_transcription: models: [whisper-1] default: whisper-1 diff --git a/enterprise/app/models/concerns/agentable.rb b/enterprise/app/models/concerns/agentable.rb index 5bdb26c6c..72f876cfc 100644 --- a/enterprise/app/models/concerns/agentable.rb +++ b/enterprise/app/models/concerns/agentable.rb @@ -43,7 +43,14 @@ module Concerns::Agentable end def agent_model - InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || LlmConstants::DEFAULT_MODEL + route = Llm::FeatureRouter.resolve(feature: 'assistant', account: account) + return route[:model] if route[:source] == :account_override + + installation_model.presence || route[:model] + end + + def installation_model + InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value end def agent_response_schema diff --git a/enterprise/app/services/captain/copilot/chat_service.rb b/enterprise/app/services/captain/copilot/chat_service.rb index 473e6814b..b5b1b08f6 100644 --- a/enterprise/app/services/captain/copilot/chat_service.rb +++ b/enterprise/app/services/captain/copilot/chat_service.rb @@ -4,7 +4,7 @@ class Captain::Copilot::ChatService < Llm::BaseAiService attr_reader :assistant, :account, :user, :copilot_thread, :previous_history, :messages def initialize(assistant, config) - super() + super(feature: 'copilot', account: assistant.account) @assistant = assistant @account = assistant.account diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb index 52b6c3b5a..58b86854a 100644 --- a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb +++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb @@ -3,7 +3,7 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService include Captain::Llm::AssistantResponseInspectionHelpers def initialize(assistant:, conversation:) - super() + super(feature: 'assistant', account: conversation.account) @assistant = assistant @conversation = conversation @temperature = 0.0 diff --git a/enterprise/app/services/captain/llm/assistant_chat_service.rb b/enterprise/app/services/captain/llm/assistant_chat_service.rb index f33ae6d3e..b4e42c573 100644 --- a/enterprise/app/services/captain/llm/assistant_chat_service.rb +++ b/enterprise/app/services/captain/llm/assistant_chat_service.rb @@ -2,7 +2,7 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService include Captain::ChatHelper def initialize(assistant: nil, conversation: nil, source: nil) - super() + super(feature: 'assistant', account: assistant&.account || conversation&.account) @assistant = assistant @conversation = conversation diff --git a/enterprise/app/services/captain/llm/contact_attributes_service.rb b/enterprise/app/services/captain/llm/contact_attributes_service.rb index 79ba97769..40b7a3284 100644 --- a/enterprise/app/services/captain/llm/contact_attributes_service.rb +++ b/enterprise/app/services/captain/llm/contact_attributes_service.rb @@ -2,7 +2,7 @@ class Captain::Llm::ContactAttributesService < Llm::BaseAiService include Integrations::LlmInstrumentation def initialize(assistant, conversation) - super() + super(feature: 'assistant', account: conversation.account) @assistant = assistant @conversation = conversation @contact = conversation.contact diff --git a/enterprise/app/services/captain/llm/contact_notes_service.rb b/enterprise/app/services/captain/llm/contact_notes_service.rb index 975b1f0cd..79b83320b 100644 --- a/enterprise/app/services/captain/llm/contact_notes_service.rb +++ b/enterprise/app/services/captain/llm/contact_notes_service.rb @@ -2,7 +2,7 @@ class Captain::Llm::ContactNotesService < Llm::BaseAiService include Integrations::LlmInstrumentation def initialize(assistant, conversation) - super() + super(feature: 'assistant', account: conversation.account) @assistant = assistant @conversation = conversation @contact = conversation.contact diff --git a/enterprise/app/services/captain/llm/conversation_faq_service.rb b/enterprise/app/services/captain/llm/conversation_faq_service.rb index 31234fda7..82c838354 100644 --- a/enterprise/app/services/captain/llm/conversation_faq_service.rb +++ b/enterprise/app/services/captain/llm/conversation_faq_service.rb @@ -4,7 +4,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService DISTANCE_THRESHOLD = 0.3 def initialize(assistant, conversation) - super() + super(feature: 'document_faq_generation', account: conversation.account) @assistant = assistant @conversation = conversation @content = conversation.to_llm_text diff --git a/enterprise/app/services/captain/llm/faq_generator_service.rb b/enterprise/app/services/captain/llm/faq_generator_service.rb index b80382b3e..40f949a99 100644 --- a/enterprise/app/services/captain/llm/faq_generator_service.rb +++ b/enterprise/app/services/captain/llm/faq_generator_service.rb @@ -2,7 +2,7 @@ class Captain::Llm::FaqGeneratorService < Llm::BaseAiService include Integrations::LlmInstrumentation def initialize(document:) - super() + super(feature: 'document_faq_generation', account: document.account) @document = document @content = document.content @language = document.account.locale_english_name diff --git a/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb b/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb index b567609e8..4d842b071 100644 --- a/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb +++ b/enterprise/app/services/captain/llm/paginated_faq_generator_service.rb @@ -15,7 +15,7 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService @max_pages = options[:max_pages] # Optional limit from UI @total_pages_processed = 0 @iterations_completed = 0 - @model = LlmConstants::PDF_PROCESSING_MODEL + @model = Llm::FeatureRouter.resolve(feature: 'pdf_faq_generation', account: document.account)[:model] end def generate diff --git a/enterprise/app/services/llm/base_ai_service.rb b/enterprise/app/services/llm/base_ai_service.rb index 0df5e6a67..bec3b5cb9 100644 --- a/enterprise/app/services/llm/base_ai_service.rb +++ b/enterprise/app/services/llm/base_ai_service.rb @@ -8,7 +8,11 @@ class Llm::BaseAiService attr_reader :model, :temperature - def initialize + def initialize(feature: nil, account: nil, fallback_model: nil) + @llm_feature = feature + @llm_account = account + @fallback_model = fallback_model + Llm::Config.initialize! setup_model setup_temperature @@ -29,8 +33,24 @@ class Llm::BaseAiService end def setup_model - config_value = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value - @model = (config_value.presence || DEFAULT_MODEL) + route = feature_route + return @model = route[:model] if account_override_route?(route) + + @model = @fallback_model.presence || installation_model.presence || route&.dig(:model) || DEFAULT_MODEL + end + + def feature_route + return if @llm_feature.blank? + + Llm::FeatureRouter.resolve(feature: @llm_feature, account: @llm_account) + end + + def account_override_route?(route) + route&.dig(:source) == :account_override + end + + def installation_model + InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value end def setup_temperature diff --git a/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb b/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb index c06f3c836..82122dfa5 100644 --- a/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb +++ b/spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb @@ -84,6 +84,28 @@ RSpec.describe 'Api::V1::Accounts::Captain::Preferences', type: :request do expect(account.reload.captain_models['editor']).to eq('gpt-4.1-mini') end + it 'updates captain_models for document FAQ generation' do + put "/api/v1/accounts/#{account.id}/captain/preferences", + headers: admin.create_new_auth_token, + params: { captain_models: { document_faq_generation: 'gpt-5.2' } }, + as: :json + + expect(response).to have_http_status(:success) + expect(json_response.dig(:features, :document_faq_generation, :selected)).to eq('gpt-5.2') + expect(account.reload.captain_models['document_faq_generation']).to eq('gpt-5.2') + end + + it 'updates captain_models for PDF FAQ generation' do + put "/api/v1/accounts/#{account.id}/captain/preferences", + headers: admin.create_new_auth_token, + params: { captain_models: { pdf_faq_generation: 'gpt-5.2' } }, + as: :json + + expect(response).to have_http_status(:success) + expect(json_response.dig(:features, :pdf_faq_generation, :selected)).to eq('gpt-5.2') + expect(account.reload.captain_models['pdf_faq_generation']).to eq('gpt-5.2') + end + it 'updates captain_features' do put "/api/v1/accounts/#{account.id}/captain/preferences", headers: admin.create_new_auth_token, diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 4ce37523c..c9958a871 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -12,6 +12,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) } let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) } let(:mock_false_promise_service) { instance_double(Captain::Llm::AssistantFalsePromiseService) } + let(:assistant_model) { Llm::Models.default_model_for('assistant') } before do create(:message, conversation: conversation, content: 'Hello', message_type: :incoming) @@ -82,7 +83,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do ).and_return({ 'decision' => 'safe', 'reason' => 'safe_response', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model }) described_class.perform_now(conversation, assistant) @@ -103,12 +104,12 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do { 'decision' => 'future_work_promise', 'reason' => 'future_check_or_investigation', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model }, { 'decision' => 'safe', 'reason' => 'asks_user_to_check_or_provide_info', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model } ) @@ -143,7 +144,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do allow(mock_false_promise_service).to receive(:detect).and_return({ 'decision' => 'future_work_promise', 'reason' => 'future_check_or_investigation', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model }) described_class.perform_now(conversation, assistant) @@ -165,13 +166,13 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do { 'decision' => 'future_work_promise', 'reason' => 'future_check_or_investigation', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model }, { 'decision' => nil, 'reason' => nil, 'error' => 'verification timeout', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model } ) @@ -192,7 +193,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do allow(mock_false_promise_service).to receive(:detect).and_return({ 'decision' => 'future_work_promise', 'reason' => 'future_check_or_investigation', - 'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL + 'model' => assistant_model }) described_class.perform_now(conversation, assistant) diff --git a/spec/enterprise/models/concerns/agentable_spec.rb b/spec/enterprise/models/concerns/agentable_spec.rb index af1a617e0..996b92d45 100644 --- a/spec/enterprise/models/concerns/agentable_spec.rb +++ b/spec/enterprise/models/concerns/agentable_spec.rb @@ -7,11 +7,13 @@ RSpec.describe Concerns::Agentable do Class.new do include Concerns::Agentable + attr_reader :account attr_accessor :temperature - def initialize(name: 'Test Agent', temperature: 0.8) + def initialize(name: 'Test Agent', temperature: 0.8, account: nil) @name = name @temperature = temperature + @account = account end def self.name @@ -30,13 +32,13 @@ RSpec.describe Concerns::Agentable do end end - let(:dummy_instance) { dummy_class.new } + let(:account) { create(:account) } + let(:dummy_instance) { dummy_class.new(account: account) } let(:mock_agents_agent) { instance_double(Agents::Agent) } - let(:mock_installation_config) { instance_double(InstallationConfig, value: 'gpt-4-turbo') } before do + InstallationConfig.where(name: 'CAPTAIN_OPEN_AI_MODEL').destroy_all allow(Agents::Agent).to receive(:new).and_return(mock_agents_agent) - allow(InstallationConfig).to receive(:find_by).with(name: 'CAPTAIN_OPEN_AI_MODEL').and_return(mock_installation_config) allow(Captain::PromptRenderer).to receive(:render).and_return('rendered_template') end @@ -46,7 +48,7 @@ RSpec.describe Concerns::Agentable do name: 'Test Agent', instructions: instance_of(Proc), tools: [], - model: 'gpt-4-turbo', + model: Llm::Models.default_model_for('assistant'), temperature: 0.8, response_schema: Captain::ResponseSchema ) @@ -160,20 +162,27 @@ RSpec.describe Concerns::Agentable do end describe '#agent_model' do - it 'returns value from InstallationConfig when present' do - expect(dummy_instance.send(:agent_model)).to eq('gpt-4-turbo') + it 'returns the assistant feature default model' do + expect(dummy_instance.send(:agent_model)).to eq(Llm::Models.default_model_for('assistant')) end - it 'returns default model when config not found' do - allow(InstallationConfig).to receive(:find_by).and_return(nil) + it 'returns account override model when present' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + account.update!(captain_models: { 'assistant' => 'gpt-5.2' }) - expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1') + expect(dummy_instance.send(:agent_model)).to eq('gpt-5.2') end - it 'returns default model when config value is nil' do - allow(mock_installation_config).to receive(:value).and_return(nil) + it 'returns the installation model when account override is absent' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') - expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1') + expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1-nano') + end + + it 'returns the assistant feature default model when account is nil' do + agent = dummy_class.new(account: nil) + + expect(agent.send(:agent_model)).to eq(Llm::Models.default_model_for('assistant')) end end diff --git a/spec/enterprise/services/captain/copilot/chat_service_spec.rb b/spec/enterprise/services/captain/copilot/chat_service_spec.rb index 4903fb6a5..050923de2 100644 --- a/spec/enterprise/services/captain/copilot/chat_service_spec.rb +++ b/spec/enterprise/services/captain/copilot/chat_service_spec.rb @@ -68,6 +68,14 @@ RSpec.describe Captain::Copilot::ChatService do describe '#generate_response' do let(:service) { described_class.new(assistant, config) } + it 'uses the copilot feature model' do + account.update!(captain_models: { 'copilot' => 'gpt-5.2' }) + + expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat) + + described_class.new(assistant, config).generate_response('Hello') + end + it 'adds user input to messages when present' do expect do service.generate_response('Hello') diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb index 260e3f4f7..6138b92ee 100644 --- a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb +++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb @@ -66,15 +66,15 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do ) end - it 'uses the configured Captain model' do - create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + it 'uses the assistant feature model' do + account.update!(captain_models: { 'assistant' => 'gpt-5.2' }) - expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-nano').and_return(mock_chat) + expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat) allow(mock_chat).to receive(:ask).and_return(mock_response) result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?') - expect(result).to include('model' => 'gpt-4.1-nano') + expect(result).to include('model' => 'gpt-5.2') end context 'when the assistant has no custom instructions' do diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb index 6b2cc55c8..805e55d9e 100644 --- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb +++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb @@ -29,6 +29,16 @@ RSpec.describe Captain::Llm::AssistantChatService do end describe 'instrumentation metadata' do + it 'uses the assistant feature model' do + account.update!(captain_models: { 'assistant' => 'gpt-5.2' }) + + expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) + + service = described_class.new(assistant: assistant, conversation: conversation) + service.generate_response(message_history: [{ role: 'user', content: 'Hello' }]) + end + it 'passes channel_type to the agent session instrumentation' do service = described_class.new(assistant: assistant, conversation: conversation) diff --git a/spec/enterprise/services/captain/llm/assistant_false_promise_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_false_promise_service_spec.rb new file mode 100644 index 000000000..2011be2ed --- /dev/null +++ b/spec/enterprise/services/captain/llm/assistant_false_promise_service_spec.rb @@ -0,0 +1,61 @@ +require 'rails_helper' + +RSpec.describe Captain::Llm::AssistantFalsePromiseService do + let(:account) { create(:account) } + let(:assistant) { create(:captain_assistant, account: account) } + let(:conversation) { create(:conversation, account: account) } + let(:service) { described_class.new(assistant: assistant, conversation: conversation) } + let(:mock_chat) { instance_double(RubyLLM::Chat) } + let(:mock_response) do + instance_double( + RubyLLM::Message, + content: { 'decision' => 'safe', 'reason' => 'answer_stays_within_known_context' } + ) + end + + before do + allow(RubyLLM).to receive(:chat).and_return(mock_chat) + allow(mock_chat).to receive(:with_temperature).and_return(mock_chat) + allow(mock_chat).to receive(:with_schema).and_return(mock_chat) + allow(mock_chat).to receive(:with_instructions).and_return(mock_chat) + end + + describe '#detect' do + let(:message_history) do + [ + { role: 'user', content: 'Can you fix this later?' }, + { role: 'assistant', content: 'I can help with known troubleshooting steps.' } + ] + end + + it 'uses the detector model even when the assistant feature model is overridden' do + account.update!(captain_models: { 'assistant' => 'gpt-5-mini' }) + + expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) + + result = service.detect(message_history: message_history, assistant_response: 'Try restarting the app.') + + expect(result).to include('model' => 'gpt-5.2') + end + + it 'uses the false promise schema and detector prompt' do + expect(mock_chat).to receive(:with_schema).with(Captain::AssistantFalsePromiseSchema).and_return(mock_chat) + expect(mock_chat).to receive(:with_instructions).with( + a_string_including('future work', 'future_work_promise') + ).and_return(mock_chat) + expect(mock_chat).to receive(:ask).with( + a_string_including( + '', + 'User: Can you fix this later?', + '', + 'Try restarting the app.' + ) + ).and_return(mock_response) + + result = service.detect(message_history: message_history, assistant_response: 'Try restarting the app.') + + expect(result).to include('decision' => 'safe', 'reason' => 'answer_stays_within_known_context') + end + end +end diff --git a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb index 0ab7f37bf..004d7027b 100644 --- a/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb +++ b/spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb @@ -33,6 +33,23 @@ RSpec.describe Captain::Llm::ConversationFaqService do allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([]) end + it 'uses the document FAQ generation feature model' do + expect(RubyLLM).to receive(:chat).with( + model: Llm::Models.default_model_for('document_faq_generation') + ).and_return(mock_chat) + + described_class.new(captain_assistant, conversation).generate_and_deduplicate + end + + it 'resolves the feature model from the conversation account' do + expect(Llm::FeatureRouter).to receive(:resolve).with( + feature: 'document_faq_generation', + account: conversation.account + ).and_call_original + + described_class.new(captain_assistant, conversation).generate_and_deduplicate + end + it 'creates new FAQs for valid conversation content' do expect do service.generate_and_deduplicate diff --git a/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb b/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb index ff7138c9a..6e81d7146 100644 --- a/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb +++ b/spec/enterprise/services/captain/llm/faq_generator_service_spec.rb @@ -26,6 +26,23 @@ RSpec.describe Captain::Llm::FaqGeneratorService do describe '#generate' do context 'when successful' do + it 'uses the document FAQ generation feature model' do + expect(RubyLLM).to receive(:chat).with( + model: Llm::Models.default_model_for('document_faq_generation') + ).and_return(mock_chat) + + described_class.new(document: document).generate + end + + it 'resolves the feature model from the document account' do + expect(Llm::FeatureRouter).to receive(:resolve).with( + feature: 'document_faq_generation', + account: document.account + ).and_call_original + + described_class.new(document: document).generate + end + it 'returns parsed FAQs from the LLM response' do result = service.generate expect(result).to eq(sample_faqs) diff --git a/spec/enterprise/services/captain/llm/paginated_faq_generator_service_spec.rb b/spec/enterprise/services/captain/llm/paginated_faq_generator_service_spec.rb index ca4518435..7fc22dab9 100644 --- a/spec/enterprise/services/captain/llm/paginated_faq_generator_service_spec.rb +++ b/spec/enterprise/services/captain/llm/paginated_faq_generator_service_spec.rb @@ -16,6 +16,12 @@ RSpec.describe Captain::Llm::PaginatedFaqGeneratorService do end describe '#generate' do + it 'uses the PDF FAQ generation feature model' do + document.account.update!(captain_models: { 'pdf_faq_generation' => 'gpt-5.2' }) + + expect(service.model).to eq('gpt-5.2') + end + context 'when document lacks OpenAI file ID' do before do allow(document).to receive(:openai_file_id).and_return(nil) diff --git a/spec/enterprise/services/llm/base_ai_service_spec.rb b/spec/enterprise/services/llm/base_ai_service_spec.rb index c45fff522..f66e6bb81 100644 --- a/spec/enterprise/services/llm/base_ai_service_spec.rb +++ b/spec/enterprise/services/llm/base_ai_service_spec.rb @@ -3,10 +3,38 @@ require 'rails_helper' RSpec.describe Llm::BaseAiService do subject(:service) { described_class.new } + let(:account) { create(:account) } + before do + InstallationConfig.where(name: %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL]).destroy_all create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') end + describe '#initialize' do + it 'uses the installation model when no feature is provided' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + + expect(described_class.new.model).to eq('gpt-4.1-nano') + end + + it 'uses the account override when feature context is provided' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + account.update!(captain_models: { 'assistant' => 'gpt-5.2' }) + + expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-5.2') + end + + it 'uses the installation model when feature context has no account override' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + + expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-4.1-nano') + end + + it 'uses the feature default when feature context has no account override or installation model' do + expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-5.1') + end + end + describe '#sanitize_json_response' do it 'strips ```json fences' do input = "```json\n{\"key\": \"value\"}\n```"