feat: Route Enterprise LLM services (3/6) (#14841)
# Pull Request Template ## Description Routes Enterprise assistant, copilot, FAQ, contact memory, action-classifier, and false-promise detector LLM paths through feature-specific model resolution. `Llm::BaseAiService` now accepts feature/account context and uses `Llm::FeatureRouter` when that context is present, while retaining the installation-model fallback for unmigrated callers. This also adds a `document_faq_generation` feature default for generative FAQ/document content. Linear: https://linear.app/chatwoot/issue/CW-7425/test-new-models Depends on #14840 ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected) - [ ] This change requires a documentation update ## How Has This Been Tested? - `bundle exec rspec spec/lib/llm/models_spec.rb spec/enterprise/services/llm/base_ai_service_spec.rb spec/enterprise/services/captain/copilot/chat_service_spec.rb spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb spec/enterprise/services/captain/llm/faq_generator_service_spec.rb spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb spec/enterprise/services/captain/llm/assistant_false_promise_service_spec.rb spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb` passed with 112 examples, 0 failures. - `bundle exec rspec spec/models/concerns/captain_featurable_spec.rb spec/models/account_spec.rb spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb spec/lib/llm/feature_router_spec.rb` passed with 87 examples, 0 failures. - `bundle exec rubocop enterprise/app/services/llm/base_ai_service.rb enterprise/app/services/captain/copilot/chat_service.rb enterprise/app/services/captain/llm/assistant_chat_service.rb enterprise/app/services/captain/llm/faq_generator_service.rb enterprise/app/services/captain/llm/conversation_faq_service.rb enterprise/app/services/captain/llm/contact_notes_service.rb enterprise/app/services/captain/llm/contact_attributes_service.rb enterprise/app/services/captain/llm/assistant_action_classifier_service.rb enterprise/app/services/captain/llm/assistant_false_promise_service.rb spec/enterprise/services/llm/base_ai_service_spec.rb spec/enterprise/services/captain/copilot/chat_service_spec.rb spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb spec/enterprise/services/captain/llm/faq_generator_service_spec.rb spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb spec/enterprise/services/captain/llm/assistant_false_promise_service_spec.rb spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb` passed with no offenses. - `bundle exec ruby -e "require 'yaml'; config = YAML.load_file('config/llm.yml'); abort('missing document_faq_generation') unless config.dig('features', 'document_faq_generation'); abort('missing default') unless config.dig('features', 'document_faq_generation', 'default'); puts 'llm.yml ok'"` passed. - `git diff --check` passed. ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules
This commit is contained in:
Sony Mathew
@@ -47,17 +47,15 @@ class Api::V1::Accounts::Captain::PreferencesController < Api::V1::Accounts::Bas
|
||||
end
|
||||
|
||||
def permitted_captain_models
|
||||
params.require(:captain_models).permit(
|
||||
:editor, :assistant, :copilot, :label_suggestion,
|
||||
:audio_transcription, :help_center_search
|
||||
).to_h.stringify_keys
|
||||
params.require(:captain_models).permit(*captain_feature_keys).to_h.stringify_keys
|
||||
end
|
||||
|
||||
def permitted_captain_features
|
||||
params.require(:captain_features).permit(
|
||||
:editor, :assistant, :copilot, :label_suggestion,
|
||||
:audio_transcription, :help_center_search
|
||||
).to_h.stringify_keys
|
||||
params.require(:captain_features).permit(*captain_feature_keys).to_h.stringify_keys
|
||||
end
|
||||
|
||||
def captain_feature_keys
|
||||
Llm::Models.feature_keys.map(&:to_sym)
|
||||
end
|
||||
|
||||
def features_with_account_preferences
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
module AccountSettingsSchema
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
CAPTAIN_MODEL_PROPERTIES = Llm::Models.feature_keys.index_with { { 'type': %w[string null] } }.freeze
|
||||
CAPTAIN_FEATURE_PROPERTIES = Llm::Models.feature_keys.index_with { { 'type': %w[boolean null] } }.freeze
|
||||
|
||||
SETTINGS_PARAMS_SCHEMA = {
|
||||
'type': 'object',
|
||||
'properties':
|
||||
@@ -19,26 +22,12 @@ module AccountSettingsSchema
|
||||
},
|
||||
'captain_models': {
|
||||
'type': %w[object null],
|
||||
'properties': {
|
||||
'editor': { 'type': %w[string null] },
|
||||
'assistant': { 'type': %w[string null] },
|
||||
'copilot': { 'type': %w[string null] },
|
||||
'label_suggestion': { 'type': %w[string null] },
|
||||
'audio_transcription': { 'type': %w[string null] },
|
||||
'help_center_search': { 'type': %w[string null] }
|
||||
},
|
||||
'properties': CAPTAIN_MODEL_PROPERTIES,
|
||||
'additionalProperties': false
|
||||
},
|
||||
'captain_features': {
|
||||
'type': %w[object null],
|
||||
'properties': {
|
||||
'editor': { 'type': %w[boolean null] },
|
||||
'assistant': { 'type': %w[boolean null] },
|
||||
'copilot': { 'type': %w[boolean null] },
|
||||
'label_suggestion': { 'type': %w[boolean null] },
|
||||
'audio_transcription': { 'type': %w[boolean null] },
|
||||
'help_center_search': { 'type': %w[boolean null] }
|
||||
},
|
||||
'properties': CAPTAIN_FEATURE_PROPERTIES,
|
||||
'additionalProperties': false
|
||||
}
|
||||
},
|
||||
|
||||
@@ -109,6 +109,23 @@ features:
|
||||
models:
|
||||
[gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini, gemini-3-flash, claude-haiku-4.5]
|
||||
default: gpt-4.1-nano
|
||||
document_faq_generation:
|
||||
models:
|
||||
[
|
||||
gpt-4.1-mini,
|
||||
gpt-5-mini,
|
||||
gpt-4.1,
|
||||
gpt-5.1,
|
||||
gpt-5.2,
|
||||
claude-haiku-4.5,
|
||||
claude-sonnet-4.5,
|
||||
gemini-3-flash,
|
||||
gemini-3-pro,
|
||||
]
|
||||
default: gpt-4.1-mini
|
||||
pdf_faq_generation:
|
||||
models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2]
|
||||
default: gpt-4.1-mini
|
||||
audio_transcription:
|
||||
models: [whisper-1]
|
||||
default: whisper-1
|
||||
|
||||
@@ -43,7 +43,14 @@ module Concerns::Agentable
|
||||
end
|
||||
|
||||
def agent_model
|
||||
InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || LlmConstants::DEFAULT_MODEL
|
||||
route = Llm::FeatureRouter.resolve(feature: 'assistant', account: account)
|
||||
return route[:model] if route[:source] == :account_override
|
||||
|
||||
installation_model.presence || route[:model]
|
||||
end
|
||||
|
||||
def installation_model
|
||||
InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value
|
||||
end
|
||||
|
||||
def agent_response_schema
|
||||
|
||||
@@ -4,7 +4,7 @@ class Captain::Copilot::ChatService < Llm::BaseAiService
|
||||
attr_reader :assistant, :account, :user, :copilot_thread, :previous_history, :messages
|
||||
|
||||
def initialize(assistant, config)
|
||||
super()
|
||||
super(feature: 'copilot', account: assistant.account)
|
||||
|
||||
@assistant = assistant
|
||||
@account = assistant.account
|
||||
|
||||
@@ -3,7 +3,7 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
include Captain::Llm::AssistantResponseInspectionHelpers
|
||||
|
||||
def initialize(assistant:, conversation:)
|
||||
super()
|
||||
super(feature: 'assistant', account: conversation.account)
|
||||
@assistant = assistant
|
||||
@conversation = conversation
|
||||
@temperature = 0.0
|
||||
|
||||
@@ -2,7 +2,7 @@ class Captain::Llm::AssistantChatService < Llm::BaseAiService
|
||||
include Captain::ChatHelper
|
||||
|
||||
def initialize(assistant: nil, conversation: nil, source: nil)
|
||||
super()
|
||||
super(feature: 'assistant', account: assistant&.account || conversation&.account)
|
||||
|
||||
@assistant = assistant
|
||||
@conversation = conversation
|
||||
|
||||
@@ -2,7 +2,7 @@ class Captain::Llm::ContactAttributesService < Llm::BaseAiService
|
||||
include Integrations::LlmInstrumentation
|
||||
|
||||
def initialize(assistant, conversation)
|
||||
super()
|
||||
super(feature: 'assistant', account: conversation.account)
|
||||
@assistant = assistant
|
||||
@conversation = conversation
|
||||
@contact = conversation.contact
|
||||
|
||||
@@ -2,7 +2,7 @@ class Captain::Llm::ContactNotesService < Llm::BaseAiService
|
||||
include Integrations::LlmInstrumentation
|
||||
|
||||
def initialize(assistant, conversation)
|
||||
super()
|
||||
super(feature: 'assistant', account: conversation.account)
|
||||
@assistant = assistant
|
||||
@conversation = conversation
|
||||
@contact = conversation.contact
|
||||
|
||||
@@ -4,7 +4,7 @@ class Captain::Llm::ConversationFaqService < Llm::BaseAiService
|
||||
DISTANCE_THRESHOLD = 0.3
|
||||
|
||||
def initialize(assistant, conversation)
|
||||
super()
|
||||
super(feature: 'document_faq_generation', account: conversation.account)
|
||||
@assistant = assistant
|
||||
@conversation = conversation
|
||||
@content = conversation.to_llm_text
|
||||
|
||||
@@ -2,7 +2,7 @@ class Captain::Llm::FaqGeneratorService < Llm::BaseAiService
|
||||
include Integrations::LlmInstrumentation
|
||||
|
||||
def initialize(document:)
|
||||
super()
|
||||
super(feature: 'document_faq_generation', account: document.account)
|
||||
@document = document
|
||||
@content = document.content
|
||||
@language = document.account.locale_english_name
|
||||
|
||||
@@ -15,7 +15,7 @@ class Captain::Llm::PaginatedFaqGeneratorService < Llm::LegacyBaseOpenAiService
|
||||
@max_pages = options[:max_pages] # Optional limit from UI
|
||||
@total_pages_processed = 0
|
||||
@iterations_completed = 0
|
||||
@model = LlmConstants::PDF_PROCESSING_MODEL
|
||||
@model = Llm::FeatureRouter.resolve(feature: 'pdf_faq_generation', account: document.account)[:model]
|
||||
end
|
||||
|
||||
def generate
|
||||
|
||||
@@ -8,7 +8,11 @@ class Llm::BaseAiService
|
||||
|
||||
attr_reader :model, :temperature
|
||||
|
||||
def initialize
|
||||
def initialize(feature: nil, account: nil, fallback_model: nil)
|
||||
@llm_feature = feature
|
||||
@llm_account = account
|
||||
@fallback_model = fallback_model
|
||||
|
||||
Llm::Config.initialize!
|
||||
setup_model
|
||||
setup_temperature
|
||||
@@ -29,8 +33,24 @@ class Llm::BaseAiService
|
||||
end
|
||||
|
||||
def setup_model
|
||||
config_value = InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value
|
||||
@model = (config_value.presence || DEFAULT_MODEL)
|
||||
route = feature_route
|
||||
return @model = route[:model] if account_override_route?(route)
|
||||
|
||||
@model = @fallback_model.presence || installation_model.presence || route&.dig(:model) || DEFAULT_MODEL
|
||||
end
|
||||
|
||||
def feature_route
|
||||
return if @llm_feature.blank?
|
||||
|
||||
Llm::FeatureRouter.resolve(feature: @llm_feature, account: @llm_account)
|
||||
end
|
||||
|
||||
def account_override_route?(route)
|
||||
route&.dig(:source) == :account_override
|
||||
end
|
||||
|
||||
def installation_model
|
||||
InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value
|
||||
end
|
||||
|
||||
def setup_temperature
|
||||
|
||||
@@ -84,6 +84,28 @@ RSpec.describe 'Api::V1::Accounts::Captain::Preferences', type: :request do
|
||||
expect(account.reload.captain_models['editor']).to eq('gpt-4.1-mini')
|
||||
end
|
||||
|
||||
it 'updates captain_models for document FAQ generation' do
|
||||
put "/api/v1/accounts/#{account.id}/captain/preferences",
|
||||
headers: admin.create_new_auth_token,
|
||||
params: { captain_models: { document_faq_generation: 'gpt-5.2' } },
|
||||
as: :json
|
||||
|
||||
expect(response).to have_http_status(:success)
|
||||
expect(json_response.dig(:features, :document_faq_generation, :selected)).to eq('gpt-5.2')
|
||||
expect(account.reload.captain_models['document_faq_generation']).to eq('gpt-5.2')
|
||||
end
|
||||
|
||||
it 'updates captain_models for PDF FAQ generation' do
|
||||
put "/api/v1/accounts/#{account.id}/captain/preferences",
|
||||
headers: admin.create_new_auth_token,
|
||||
params: { captain_models: { pdf_faq_generation: 'gpt-5.2' } },
|
||||
as: :json
|
||||
|
||||
expect(response).to have_http_status(:success)
|
||||
expect(json_response.dig(:features, :pdf_faq_generation, :selected)).to eq('gpt-5.2')
|
||||
expect(account.reload.captain_models['pdf_faq_generation']).to eq('gpt-5.2')
|
||||
end
|
||||
|
||||
it 'updates captain_features' do
|
||||
put "/api/v1/accounts/#{account.id}/captain/preferences",
|
||||
headers: admin.create_new_auth_token,
|
||||
|
||||
@@ -12,6 +12,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
|
||||
let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
|
||||
let(:mock_false_promise_service) { instance_double(Captain::Llm::AssistantFalsePromiseService) }
|
||||
let(:assistant_model) { Llm::Models.default_model_for('assistant') }
|
||||
|
||||
before do
|
||||
create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
|
||||
@@ -82,7 +83,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
).and_return({
|
||||
'decision' => 'safe',
|
||||
'reason' => 'safe_response',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
})
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
@@ -103,12 +104,12 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
{
|
||||
'decision' => 'future_work_promise',
|
||||
'reason' => 'future_check_or_investigation',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
},
|
||||
{
|
||||
'decision' => 'safe',
|
||||
'reason' => 'asks_user_to_check_or_provide_info',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
}
|
||||
)
|
||||
|
||||
@@ -143,7 +144,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
allow(mock_false_promise_service).to receive(:detect).and_return({
|
||||
'decision' => 'future_work_promise',
|
||||
'reason' => 'future_check_or_investigation',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
})
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
@@ -165,13 +166,13 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
{
|
||||
'decision' => 'future_work_promise',
|
||||
'reason' => 'future_check_or_investigation',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
},
|
||||
{
|
||||
'decision' => nil,
|
||||
'reason' => nil,
|
||||
'error' => 'verification timeout',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
}
|
||||
)
|
||||
|
||||
@@ -192,7 +193,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
allow(mock_false_promise_service).to receive(:detect).and_return({
|
||||
'decision' => 'future_work_promise',
|
||||
'reason' => 'future_check_or_investigation',
|
||||
'model' => Captain::Llm::AssistantFalsePromiseService::DETECTOR_MODEL
|
||||
'model' => assistant_model
|
||||
})
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
@@ -7,11 +7,13 @@ RSpec.describe Concerns::Agentable do
|
||||
Class.new do
|
||||
include Concerns::Agentable
|
||||
|
||||
attr_reader :account
|
||||
attr_accessor :temperature
|
||||
|
||||
def initialize(name: 'Test Agent', temperature: 0.8)
|
||||
def initialize(name: 'Test Agent', temperature: 0.8, account: nil)
|
||||
@name = name
|
||||
@temperature = temperature
|
||||
@account = account
|
||||
end
|
||||
|
||||
def self.name
|
||||
@@ -30,13 +32,13 @@ RSpec.describe Concerns::Agentable do
|
||||
end
|
||||
end
|
||||
|
||||
let(:dummy_instance) { dummy_class.new }
|
||||
let(:account) { create(:account) }
|
||||
let(:dummy_instance) { dummy_class.new(account: account) }
|
||||
let(:mock_agents_agent) { instance_double(Agents::Agent) }
|
||||
let(:mock_installation_config) { instance_double(InstallationConfig, value: 'gpt-4-turbo') }
|
||||
|
||||
before do
|
||||
InstallationConfig.where(name: 'CAPTAIN_OPEN_AI_MODEL').destroy_all
|
||||
allow(Agents::Agent).to receive(:new).and_return(mock_agents_agent)
|
||||
allow(InstallationConfig).to receive(:find_by).with(name: 'CAPTAIN_OPEN_AI_MODEL').and_return(mock_installation_config)
|
||||
allow(Captain::PromptRenderer).to receive(:render).and_return('rendered_template')
|
||||
end
|
||||
|
||||
@@ -46,7 +48,7 @@ RSpec.describe Concerns::Agentable do
|
||||
name: 'Test Agent',
|
||||
instructions: instance_of(Proc),
|
||||
tools: [],
|
||||
model: 'gpt-4-turbo',
|
||||
model: Llm::Models.default_model_for('assistant'),
|
||||
temperature: 0.8,
|
||||
response_schema: Captain::ResponseSchema
|
||||
)
|
||||
@@ -160,20 +162,27 @@ RSpec.describe Concerns::Agentable do
|
||||
end
|
||||
|
||||
describe '#agent_model' do
|
||||
it 'returns value from InstallationConfig when present' do
|
||||
expect(dummy_instance.send(:agent_model)).to eq('gpt-4-turbo')
|
||||
it 'returns the assistant feature default model' do
|
||||
expect(dummy_instance.send(:agent_model)).to eq(Llm::Models.default_model_for('assistant'))
|
||||
end
|
||||
|
||||
it 'returns default model when config not found' do
|
||||
allow(InstallationConfig).to receive(:find_by).and_return(nil)
|
||||
it 'returns account override model when present' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
account.update!(captain_models: { 'assistant' => 'gpt-5.2' })
|
||||
|
||||
expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1')
|
||||
expect(dummy_instance.send(:agent_model)).to eq('gpt-5.2')
|
||||
end
|
||||
|
||||
it 'returns default model when config value is nil' do
|
||||
allow(mock_installation_config).to receive(:value).and_return(nil)
|
||||
it 'returns the installation model when account override is absent' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
|
||||
expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1')
|
||||
expect(dummy_instance.send(:agent_model)).to eq('gpt-4.1-nano')
|
||||
end
|
||||
|
||||
it 'returns the assistant feature default model when account is nil' do
|
||||
agent = dummy_class.new(account: nil)
|
||||
|
||||
expect(agent.send(:agent_model)).to eq(Llm::Models.default_model_for('assistant'))
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -68,6 +68,14 @@ RSpec.describe Captain::Copilot::ChatService do
|
||||
describe '#generate_response' do
|
||||
let(:service) { described_class.new(assistant, config) }
|
||||
|
||||
it 'uses the copilot feature model' do
|
||||
account.update!(captain_models: { 'copilot' => 'gpt-5.2' })
|
||||
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat)
|
||||
|
||||
described_class.new(assistant, config).generate_response('Hello')
|
||||
end
|
||||
|
||||
it 'adds user input to messages when present' do
|
||||
expect do
|
||||
service.generate_response('Hello')
|
||||
|
||||
@@ -66,15 +66,15 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
|
||||
)
|
||||
end
|
||||
|
||||
it 'uses the configured Captain model' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
it 'uses the assistant feature model' do
|
||||
account.update!(captain_models: { 'assistant' => 'gpt-5.2' })
|
||||
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-nano').and_return(mock_chat)
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
|
||||
|
||||
expect(result).to include('model' => 'gpt-4.1-nano')
|
||||
expect(result).to include('model' => 'gpt-5.2')
|
||||
end
|
||||
|
||||
context 'when the assistant has no custom instructions' do
|
||||
|
||||
@@ -29,6 +29,16 @@ RSpec.describe Captain::Llm::AssistantChatService do
|
||||
end
|
||||
|
||||
describe 'instrumentation metadata' do
|
||||
it 'uses the assistant feature model' do
|
||||
account.update!(captain_models: { 'assistant' => 'gpt-5.2' })
|
||||
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
service = described_class.new(assistant: assistant, conversation: conversation)
|
||||
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
||||
end
|
||||
|
||||
it 'passes channel_type to the agent session instrumentation' do
|
||||
service = described_class.new(assistant: assistant, conversation: conversation)
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::Llm::AssistantFalsePromiseService do
|
||||
let(:account) { create(:account) }
|
||||
let(:assistant) { create(:captain_assistant, account: account) }
|
||||
let(:conversation) { create(:conversation, account: account) }
|
||||
let(:service) { described_class.new(assistant: assistant, conversation: conversation) }
|
||||
let(:mock_chat) { instance_double(RubyLLM::Chat) }
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: { 'decision' => 'safe', 'reason' => 'answer_stays_within_known_context' }
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
|
||||
end
|
||||
|
||||
describe '#detect' do
|
||||
let(:message_history) do
|
||||
[
|
||||
{ role: 'user', content: 'Can you fix this later?' },
|
||||
{ role: 'assistant', content: 'I can help with known troubleshooting steps.' }
|
||||
]
|
||||
end
|
||||
|
||||
it 'uses the detector model even when the assistant feature model is overridden' do
|
||||
account.update!(captain_models: { 'assistant' => 'gpt-5-mini' })
|
||||
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
result = service.detect(message_history: message_history, assistant_response: 'Try restarting the app.')
|
||||
|
||||
expect(result).to include('model' => 'gpt-5.2')
|
||||
end
|
||||
|
||||
it 'uses the false promise schema and detector prompt' do
|
||||
expect(mock_chat).to receive(:with_schema).with(Captain::AssistantFalsePromiseSchema).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:with_instructions).with(
|
||||
a_string_including('future work', 'future_work_promise')
|
||||
).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:ask).with(
|
||||
a_string_including(
|
||||
'<conversation_context>',
|
||||
'User: Can you fix this later?',
|
||||
'<assistant_response_to_check>',
|
||||
'Try restarting the app.'
|
||||
)
|
||||
).and_return(mock_response)
|
||||
|
||||
result = service.detect(message_history: message_history, assistant_response: 'Try restarting the app.')
|
||||
|
||||
expect(result).to include('decision' => 'safe', 'reason' => 'answer_stays_within_known_context')
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -33,6 +33,23 @@ RSpec.describe Captain::Llm::ConversationFaqService do
|
||||
allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
|
||||
end
|
||||
|
||||
it 'uses the document FAQ generation feature model' do
|
||||
expect(RubyLLM).to receive(:chat).with(
|
||||
model: Llm::Models.default_model_for('document_faq_generation')
|
||||
).and_return(mock_chat)
|
||||
|
||||
described_class.new(captain_assistant, conversation).generate_and_deduplicate
|
||||
end
|
||||
|
||||
it 'resolves the feature model from the conversation account' do
|
||||
expect(Llm::FeatureRouter).to receive(:resolve).with(
|
||||
feature: 'document_faq_generation',
|
||||
account: conversation.account
|
||||
).and_call_original
|
||||
|
||||
described_class.new(captain_assistant, conversation).generate_and_deduplicate
|
||||
end
|
||||
|
||||
it 'creates new FAQs for valid conversation content' do
|
||||
expect do
|
||||
service.generate_and_deduplicate
|
||||
|
||||
@@ -26,6 +26,23 @@ RSpec.describe Captain::Llm::FaqGeneratorService do
|
||||
|
||||
describe '#generate' do
|
||||
context 'when successful' do
|
||||
it 'uses the document FAQ generation feature model' do
|
||||
expect(RubyLLM).to receive(:chat).with(
|
||||
model: Llm::Models.default_model_for('document_faq_generation')
|
||||
).and_return(mock_chat)
|
||||
|
||||
described_class.new(document: document).generate
|
||||
end
|
||||
|
||||
it 'resolves the feature model from the document account' do
|
||||
expect(Llm::FeatureRouter).to receive(:resolve).with(
|
||||
feature: 'document_faq_generation',
|
||||
account: document.account
|
||||
).and_call_original
|
||||
|
||||
described_class.new(document: document).generate
|
||||
end
|
||||
|
||||
it 'returns parsed FAQs from the LLM response' do
|
||||
result = service.generate
|
||||
expect(result).to eq(sample_faqs)
|
||||
|
||||
@@ -16,6 +16,12 @@ RSpec.describe Captain::Llm::PaginatedFaqGeneratorService do
|
||||
end
|
||||
|
||||
describe '#generate' do
|
||||
it 'uses the PDF FAQ generation feature model' do
|
||||
document.account.update!(captain_models: { 'pdf_faq_generation' => 'gpt-5.2' })
|
||||
|
||||
expect(service.model).to eq('gpt-5.2')
|
||||
end
|
||||
|
||||
context 'when document lacks OpenAI file ID' do
|
||||
before do
|
||||
allow(document).to receive(:openai_file_id).and_return(nil)
|
||||
|
||||
@@ -3,10 +3,38 @@ require 'rails_helper'
|
||||
RSpec.describe Llm::BaseAiService do
|
||||
subject(:service) { described_class.new }
|
||||
|
||||
let(:account) { create(:account) }
|
||||
|
||||
before do
|
||||
InstallationConfig.where(name: %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL]).destroy_all
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
|
||||
end
|
||||
|
||||
describe '#initialize' do
|
||||
it 'uses the installation model when no feature is provided' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
|
||||
expect(described_class.new.model).to eq('gpt-4.1-nano')
|
||||
end
|
||||
|
||||
it 'uses the account override when feature context is provided' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
account.update!(captain_models: { 'assistant' => 'gpt-5.2' })
|
||||
|
||||
expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-5.2')
|
||||
end
|
||||
|
||||
it 'uses the installation model when feature context has no account override' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
|
||||
expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-4.1-nano')
|
||||
end
|
||||
|
||||
it 'uses the feature default when feature context has no account override or installation model' do
|
||||
expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-5.1')
|
||||
end
|
||||
end
|
||||
|
||||
describe '#sanitize_json_response' do
|
||||
it 'strips ```json fences' do
|
||||
input = "```json\n{\"key\": \"value\"}\n```"
|
||||
|
||||
Reference in New Issue
Block a user