From 4e26c5b4bb46d88eb436e25126b9d8eb6a6dd725 Mon Sep 17 00:00:00 2001 From: Sony Mathew Date: Thu, 25 Jun 2026 17:37:45 +0530 Subject: [PATCH] feat: Route system LLM jobs (4/6) (#14843) ## Description Routes the remaining system-only and legacy-sensitive LLM jobs through feature-level model configuration, while preserving system credential usage and usage-accounting behavior. This adds dedicated defaults for help center article generation, onboarding content generation, query translation, transcription, and search embeddings so these flows can be configured per account without falling back to installation-wide model settings. Fixes https://linear.app/chatwoot/issue/CW-7425/test-new-models ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected) - [ ] This change requires a documentation update ## How Has This Been Tested? Verified the feature routing defaults and account overrides for the touched Captain/system LLM paths, including the legacy OpenAI transcription and paginated FAQ services. - `eval "$(rbenv init -)" && bundle exec rspec spec/lib/captain/base_task_service_spec.rb spec/lib/llm/models_spec.rb spec/models/concerns/captain_featurable_spec.rb spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb spec/enterprise/services/captain/llm/paginated_faq_generator_service_spec.rb spec/enterprise/services/captain/llm/pdf_processing_service_spec.rb spec/enterprise/services/messages/audio_transcription_service_spec.rb spec/enterprise/services/onboarding/help_center_article_builder_spec.rb spec/enterprise/services/captain/onboarding/website_analyzer_service_spec.rb` - `eval "$(rbenv init -)" && bundle exec rubocop app/controllers/api/v1/accounts/captain/preferences_controller.rb app/models/concerns/account_settings_schema.rb lib/captain/base_task_service.rb enterprise/app/services/captain/llm/article_translation_service.rb enterprise/app/services/captain/llm/article_writer_service.rb enterprise/app/services/captain/llm/embedding_service.rb enterprise/app/services/captain/llm/help_center_curation_service.rb enterprise/app/services/captain/llm/paginated_faq_generator_service.rb enterprise/app/services/captain/llm/translate_query_service.rb enterprise/app/services/captain/llm/widget_tagline_service.rb enterprise/app/services/captain/onboarding/website_analyzer_service.rb enterprise/app/services/messages/audio_transcription_service.rb spec/enterprise/services/messages/audio_transcription_service_spec.rb spec/lib/captain/base_task_service_spec.rb` - `ruby -e "require 'yaml'; config = YAML.load_file('config/llm.yml'); %w[document_faq_generation help_center_article_generation onboarding_content_generation help_center_query_translation audio_transcription help_center_search].each { |feature| abort(%(missing #{feature})) unless config.dig('features', feature) }; abort('wrong article default') unless config.dig('features', 'help_center_article_generation', 'default') == 'gpt-5.2'; puts 'llm.yml ok'"` - `git diff --check` ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- config/llm.yml | 21 ++++++++++ .../llm/article_translation_service.rb | 6 +-- .../captain/llm/article_writer_service.rb | 6 +-- .../services/captain/llm/embedding_service.rb | 2 +- .../llm/help_center_curation_service.rb | 2 +- .../captain/llm/translate_query_service.rb | 4 +- .../captain/llm/widget_tagline_service.rb | 6 +-- .../onboarding/website_analyzer_service.rb | 2 +- .../messages/audio_transcription_service.rb | 11 ++++-- lib/captain/base_task_service.rb | 5 ++- .../llm/article_translation_service_spec.rb | 1 + .../captain/llm/embedding_service_spec.rb | 38 +++++++++++++++++++ .../audio_transcription_service_spec.rb | 27 ++++++++++++- spec/lib/captain/base_task_service_spec.rb | 21 ++++++++++ 14 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 spec/enterprise/services/captain/llm/embedding_service_spec.rb diff --git a/config/llm.yml b/config/llm.yml index 29f621c7d..83cd9355f 100644 --- a/config/llm.yml +++ b/config/llm.yml @@ -126,6 +126,27 @@ features: pdf_faq_generation: models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2] default: gpt-4.1-mini + help_center_article_generation: + models: + [ + gpt-4.1-mini, + gpt-5-mini, + gpt-4.1, + gpt-5.1, + gpt-5.2, + claude-haiku-4.5, + claude-sonnet-4.5, + gemini-3-flash, + gemini-3-pro, + ] + default: gpt-5.2 + onboarding_content_generation: + models: + [gpt-4.1, gpt-4.1-mini, gpt-5-mini, gpt-5.1, gpt-5.2] + default: gpt-4.1 + help_center_query_translation: + models: [gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini] + default: gpt-4.1-nano audio_transcription: models: [whisper-1] default: whisper-1 diff --git a/enterprise/app/services/captain/llm/article_translation_service.rb b/enterprise/app/services/captain/llm/article_translation_service.rb index 5db26088e..fab0f934c 100644 --- a/enterprise/app/services/captain/llm/article_translation_service.rb +++ b/enterprise/app/services/captain/llm/article_translation_service.rb @@ -6,7 +6,7 @@ class Captain::Llm::ArticleTranslationService < Captain::BaseTaskService def perform raise ArgumentError, "Invalid type: #{type}" unless TYPES.include?(type) - response = make_api_call(model: translation_model, messages: messages) + response = make_api_call(feature: 'help_center_article_generation', messages: messages) return response if response[:error] response.merge(message: response[:message].strip) @@ -33,10 +33,6 @@ class Captain::Llm::ArticleTranslationService < Captain::BaseTaskService @llm_credential ||= system_llm_credential end - def translation_model - @translation_model ||= InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || GPT_MODEL - end - def title_system_prompt <<~SYSTEM_PROMPT_MESSAGE You are a professional translator. diff --git a/enterprise/app/services/captain/llm/article_writer_service.rb b/enterprise/app/services/captain/llm/article_writer_service.rb index b94027248..73b49b0ed 100644 --- a/enterprise/app/services/captain/llm/article_writer_service.rb +++ b/enterprise/app/services/captain/llm/article_writer_service.rb @@ -6,7 +6,7 @@ class Captain::Llm::ArticleWriterService < Captain::BaseTaskService pattr_initialize [:account!, :source_pages!, { hint_title: nil }] def perform - response = make_api_call(model: writer_model, messages: messages, schema: RESPONSE_SCHEMA) + response = make_api_call(feature: 'help_center_article_generation', messages: messages, schema: RESPONSE_SCHEMA) return response if response[:error] response.merge(message: extract_payload(response[:message])) @@ -92,10 +92,6 @@ class Captain::Llm::ArticleWriterService < Captain::BaseTaskService false end - def writer_model - 'gpt-5.2' - end - def build_follow_up_context? false end diff --git a/enterprise/app/services/captain/llm/embedding_service.rb b/enterprise/app/services/captain/llm/embedding_service.rb index 2fac54594..c78c70f23 100644 --- a/enterprise/app/services/captain/llm/embedding_service.rb +++ b/enterprise/app/services/captain/llm/embedding_service.rb @@ -6,7 +6,7 @@ class Captain::Llm::EmbeddingService def initialize(account_id: nil) Llm::Config.initialize! @account_id = account_id - @embedding_model = InstallationConfig.find_by(name: 'CAPTAIN_EMBEDDING_MODEL')&.value.presence || LlmConstants::DEFAULT_EMBEDDING_MODEL + @embedding_model = self.class.embedding_model end def self.embedding_model diff --git a/enterprise/app/services/captain/llm/help_center_curation_service.rb b/enterprise/app/services/captain/llm/help_center_curation_service.rb index 1f8b8acb2..37056dc25 100644 --- a/enterprise/app/services/captain/llm/help_center_curation_service.rb +++ b/enterprise/app/services/captain/llm/help_center_curation_service.rb @@ -9,7 +9,7 @@ class Captain::Llm::HelpCenterCurationService < Captain::BaseTaskService pattr_initialize [:account!, :links!] def perform - response = make_api_call(model: CURATION_MODEL, messages: messages, schema: RESPONSE_SCHEMA) + response = make_api_call(feature: 'onboarding_content_generation', model: CURATION_MODEL, messages: messages, schema: RESPONSE_SCHEMA) return response if response[:error] response.merge(message: extract_payload(response[:message])) diff --git a/enterprise/app/services/captain/llm/translate_query_service.rb b/enterprise/app/services/captain/llm/translate_query_service.rb index 3e05244d3..12fb841fa 100644 --- a/enterprise/app/services/captain/llm/translate_query_service.rb +++ b/enterprise/app/services/captain/llm/translate_query_service.rb @@ -1,6 +1,4 @@ class Captain::Llm::TranslateQueryService < Captain::BaseTaskService - MODEL = 'gpt-4.1-nano'.freeze - pattr_initialize [:account!] def translate(query, target_language:) @@ -11,7 +9,7 @@ class Captain::Llm::TranslateQueryService < Captain::BaseTaskService { role: 'user', content: query } ] - response = make_api_call(model: MODEL, messages: messages) + response = make_api_call(feature: 'help_center_query_translation', messages: messages) return query if response[:error] response[:message].strip diff --git a/enterprise/app/services/captain/llm/widget_tagline_service.rb b/enterprise/app/services/captain/llm/widget_tagline_service.rb index 230c54165..155b10396 100644 --- a/enterprise/app/services/captain/llm/widget_tagline_service.rb +++ b/enterprise/app/services/captain/llm/widget_tagline_service.rb @@ -4,7 +4,7 @@ class Captain::Llm::WidgetTaglineService < Captain::BaseTaskService pattr_initialize [:account!] def perform - response = make_api_call(model: tagline_model, messages: messages, schema: RESPONSE_SCHEMA) + response = make_api_call(feature: 'onboarding_content_generation', messages: messages, schema: RESPONSE_SCHEMA) return response if response[:error] response.merge(message: extract_tagline(response[:message])) @@ -68,10 +68,6 @@ class Captain::Llm::WidgetTaglineService < Captain::BaseTaskService false end - def tagline_model - @tagline_model ||= InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || GPT_MODEL - end - def build_follow_up_context? false end diff --git a/enterprise/app/services/captain/onboarding/website_analyzer_service.rb b/enterprise/app/services/captain/onboarding/website_analyzer_service.rb index 799b9de93..fb6bab33b 100644 --- a/enterprise/app/services/captain/onboarding/website_analyzer_service.rb +++ b/enterprise/app/services/captain/onboarding/website_analyzer_service.rb @@ -4,7 +4,7 @@ class Captain::Onboarding::WebsiteAnalyzerService < Llm::BaseAiService MAX_CONTENT_LENGTH = 8000 def initialize(website_url) - super() + super(feature: 'onboarding_content_generation') @website_url = normalize_url(website_url) @website_content = nil @favicon_url = nil diff --git a/enterprise/app/services/messages/audio_transcription_service.rb b/enterprise/app/services/messages/audio_transcription_service.rb index 748bf1efa..93ed867f2 100644 --- a/enterprise/app/services/messages/audio_transcription_service.rb +++ b/enterprise/app/services/messages/audio_transcription_service.rb @@ -1,7 +1,7 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService include Integrations::LlmInstrumentation - TRANSCRIPTION_MODEL = 'gpt-4o-mini-transcribe'.freeze + TRANSCRIPTION_MODEL = 'whisper-1'.freeze # OpenAI's transcription endpoint hard limit is 25 MB *decimal* (25_000_000), not # binary (25.megabytes = 26_214_400) — using the binary form leaks the 25.0–26.2 MB # range to the API as 413s. Long audio (~70+ min Opus) keeps the attachment but skips @@ -15,6 +15,7 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService @attachment = attachment @message = attachment.message @account = message.account + @transcription_model = Llm::FeatureRouter.resolve(feature: 'audio_transcription', account: account)[:model] end def perform @@ -81,7 +82,7 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService # behaviour across OpenAI transcription models. response = @client.audio.transcribe( parameters: { - model: TRANSCRIPTION_MODEL, + model: transcription_model, file: file, temperature: 0.0 } @@ -98,7 +99,7 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService def instrumentation_params(file_path) { span_name: 'llm.messages.audio_transcription', - model: TRANSCRIPTION_MODEL, + model: transcription_model, account_id: account&.id, feature_name: 'audio_transcription', file_path: file_path @@ -127,4 +128,8 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService 'x-mp3' => 'mp3' }.fetch(subtype, subtype) end + + def transcription_model + @transcription_model || TRANSCRIPTION_MODEL + end end diff --git a/lib/captain/base_task_service.rb b/lib/captain/base_task_service.rb index d098b24af..cfeb4e427 100644 --- a/lib/captain/base_task_service.rb +++ b/lib/captain/base_task_service.rb @@ -59,7 +59,10 @@ class Captain::BaseTaskService def resolved_model(model:, feature:) return model if feature.blank? - Llm::FeatureRouter.resolve(feature: feature, account: account)[:model] + route = Llm::FeatureRouter.resolve(feature: feature, account: account) + return model if model.present? && route[:source] == :default + + route[:model] end def execute_ruby_llm_request(model:, messages:, schema: nil, tools: []) diff --git a/spec/enterprise/services/captain/llm/article_translation_service_spec.rb b/spec/enterprise/services/captain/llm/article_translation_service_spec.rb index 1c0d83b65..0661a906a 100644 --- a/spec/enterprise/services/captain/llm/article_translation_service_spec.rb +++ b/spec/enterprise/services/captain/llm/article_translation_service_spec.rb @@ -17,6 +17,7 @@ RSpec.describe Captain::Llm::ArticleTranslationService do it 'returns the stripped translated title' do expect(service).to receive(:make_api_call) do |args| + expect(args[:feature]).to eq('help_center_article_generation') expect(args[:messages][0][:content]).to include('professional translator') expect(args[:messages][0][:content]).to include(target_language) expect(args[:messages][1][:content]).to eq('Getting Started') diff --git a/spec/enterprise/services/captain/llm/embedding_service_spec.rb b/spec/enterprise/services/captain/llm/embedding_service_spec.rb new file mode 100644 index 000000000..206ca147d --- /dev/null +++ b/spec/enterprise/services/captain/llm/embedding_service_spec.rb @@ -0,0 +1,38 @@ +require 'rails_helper' + +RSpec.describe Captain::Llm::EmbeddingService, type: :service do + def configure_embedding_model(value) + InstallationConfig.find_or_initialize_by(name: 'CAPTAIN_EMBEDDING_MODEL').tap do |config| + config.value = value + config.locked = false + config.save! + end + end + + describe '.embedding_model' do + it 'uses the installation embedding model when configured' do + configure_embedding_model('custom-embedding-model') + + expect(described_class.embedding_model).to eq('custom-embedding-model') + end + + it 'falls back to the default embedding model when the installation value is blank' do + configure_embedding_model('') + + expect(described_class.embedding_model).to eq(LlmConstants::DEFAULT_EMBEDDING_MODEL) + end + end + + describe '#get_embedding' do + let(:account) { create(:account) } + let(:embedding_response) { double('embedding_response', vectors: [0.1, 0.2]) } # rubocop:disable RSpec/VerifiedDoubles + + it 'sends the installation embedding model to RubyLLM' do + configure_embedding_model('custom-embedding-model') + + expect(RubyLLM).to receive(:embed).with('search text', model: 'custom-embedding-model').and_return(embedding_response) + + expect(described_class.new(account_id: account.id).get_embedding('search text')).to eq([0.1, 0.2]) + end + end +end diff --git a/spec/enterprise/services/messages/audio_transcription_service_spec.rb b/spec/enterprise/services/messages/audio_transcription_service_spec.rb index 32752c2b2..ecc574501 100644 --- a/spec/enterprise/services/messages/audio_transcription_service_spec.rb +++ b/spec/enterprise/services/messages/audio_transcription_service_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' RSpec.describe Messages::AudioTranscriptionService, type: :service do let(:account) { create(:account, audio_transcriptions: true) } let(:conversation) { create(:conversation, account: account) } - let(:message) { create(:message, conversation: conversation) } + let(:message) { create(:message, account: account, conversation: conversation) } let(:attachment) { message.attachments.create!(account: account, file_type: :audio) } before do @@ -101,4 +101,29 @@ RSpec.describe Messages::AudioTranscriptionService, type: :service do FileUtils.rm_f(temp_file_path) if temp_file_path.present? end end + + describe '#transcribe_audio' do + let(:service) { described_class.new(attachment) } + let(:audio_api) { double('audio_api') } # rubocop:disable RSpec/VerifiedDoubles + let(:audio_file_path) { Rails.root.join('tmp/audio_transcription_service_spec.mp3').to_s } + + before do + File.binwrite(audio_file_path, 'audio') + allow(service).to receive(:fetch_audio_file).and_return(audio_file_path) + allow(service).to receive(:update_transcription) + allow(service.client).to receive(:audio).and_return(audio_api) + end + + after do + FileUtils.rm_f(audio_file_path) + end + + it 'uses the audio transcription feature model' do + expect(audio_api).to receive(:transcribe).with( + parameters: hash_including(model: 'whisper-1', temperature: 0.0) + ).and_return({ 'text' => 'Audio transcript' }) + + expect(service.send(:transcribe_audio)).to eq('Audio transcript') + end + end end diff --git a/spec/lib/captain/base_task_service_spec.rb b/spec/lib/captain/base_task_service_spec.rb index 7b67ab94a..b24a5c49c 100644 --- a/spec/lib/captain/base_task_service_spec.rb +++ b/spec/lib/captain/base_task_service_spec.rb @@ -21,6 +21,7 @@ RSpec.describe Captain::BaseTaskService do let(:service) { test_service_class.new(account: account, conversation_display_id: conversation.display_id) } before do + InstallationConfig.where(name: 'CAPTAIN_OPEN_AI_API_KEY').destroy_all create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') # Stub captain enabled check to allow OSS specs to test base functionality # without enterprise module interference @@ -178,6 +179,26 @@ RSpec.describe Captain::BaseTaskService do service.send(:make_api_call, feature: 'editor', messages: messages) end + it 'uses the supplied model as a feature fallback when there is no account override' do + expect(mock_context).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat) + + service.send(:make_api_call, feature: 'document_faq_generation', model: 'gpt-5.2', messages: messages) + end + + it 'uses the help center article generation feature default' do + expect(mock_context).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat) + + service.send(:make_api_call, feature: 'help_center_article_generation', messages: messages) + end + + it 'prefers account overrides over supplied feature fallback models' do + account.update!(captain_models: { 'help_center_article_generation' => 'gpt-4.1' }) + + expect(mock_context).to receive(:chat).with(model: 'gpt-4.1').and_return(mock_chat) + + service.send(:make_api_call, feature: 'help_center_article_generation', model: 'gpt-5.2', messages: messages) + end + it 'returns formatted response with tokens' do result = service.send(:make_api_call, model: model, messages: messages)