From cf134deb373b705b6a85567970187fc477deed66 Mon Sep 17 00:00:00 2001 From: Sony Mathew Date: Thu, 25 Jun 2026 21:35:09 +0530 Subject: [PATCH] fix: Preserve Captain LLM defaults (#14858) # Pull Request Template ## Description Adjusts the Captain LLM feature defaults after feature routing so the defaults stay intentional and avoid unintended high-cost model upgrades. Assistant, copilot, and onboarding content generation now default to `gpt-4.1`; audio transcription keeps `gpt-4o-mini-transcribe` as the default while exposing `whisper-1` as an available account override option. Related: https://linear.app/chatwoot/issue/CW-7425/test-new-models ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? - `ruby -ryaml -e 'yaml = YAML.load_file("config/llm.yml"); features = yaml.fetch("features"); models = yaml.fetch("models"); features.each { |name, cfg| missing = Array(cfg["models"]) - models.keys; raise "#{name}: missing #{missing.join(",")}" if missing.any?; raise "#{name}: default not in models" unless Array(cfg["models"]).include?(cfg["default"]) }'` - `node -e "JSON.parse(require('fs').readFileSync('app/javascript/dashboard/i18n/locale/en/settings.json', 'utf8'))"` - `pnpm exec prettier --check app/javascript/dashboard/i18n/locale/en/settings.json` - `bundle exec rspec spec/lib/llm/feature_router_spec.rb spec/enterprise/services/llm/base_ai_service_spec.rb spec/enterprise/models/concerns/agentable_spec.rb spec/enterprise/services/messages/audio_transcription_service_spec.rb spec/models/concerns/captain_featurable_spec.rb spec/models/account_spec.rb spec/controllers/api/v1/accounts/captain/preferences_controller_spec.rb spec/controllers/super_admin/accounts_controller_spec.rb` - `bundle exec rspec spec/enterprise/services/captain/llm/article_translation_service_spec.rb spec/enterprise/services/messages/audio_transcription_service_spec.rb spec/enterprise/services/llm/base_ai_service_spec.rb` - `RUBOCOP_CACHE_ROOT=/private/tmp/rubocop_cache bundle exec rubocop enterprise/app/services/captain/llm/article_translation_service.rb enterprise/app/services/messages/audio_transcription_service.rb spec/enterprise/services/captain/llm/article_translation_service_spec.rb spec/enterprise/services/llm/base_ai_service_spec.rb spec/enterprise/services/messages/audio_transcription_service_spec.rb` - `git diff --check` ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- .../dashboard/i18n/locale/en/settings.json | 4 +++- config/llm.yml | 16 +++++++++++----- .../captain/llm/article_translation_service.rb | 6 +++++- .../messages/audio_transcription_service.rb | 7 +------ .../llm/article_translation_service_spec.rb | 15 +++++++++++++++ .../services/llm/base_ai_service_spec.rb | 2 +- .../messages/audio_transcription_service_spec.rb | 2 +- 7 files changed, 37 insertions(+), 15 deletions(-) diff --git a/app/javascript/dashboard/i18n/locale/en/settings.json b/app/javascript/dashboard/i18n/locale/en/settings.json index 0a87b4cbe..640b9c506 100644 --- a/app/javascript/dashboard/i18n/locale/en/settings.json +++ b/app/javascript/dashboard/i18n/locale/en/settings.json @@ -440,7 +440,9 @@ "DESCRIPTION": "Enable or disable AI-powered features.", "AUDIO_TRANSCRIPTION": { "TITLE": "Audio Transcription", - "DESCRIPTION": "Automatically convert voice messages and call recordings into searchable text transcripts." + "DESCRIPTION": "Automatically convert voice messages and call recordings into searchable text transcripts.", + "MODEL_TITLE": "Audio Transcription Model", + "MODEL_DESCRIPTION": "Select the AI model to use for converting audio messages into text transcripts" }, "HELP_CENTER_SEARCH": { "TITLE": "Help Center Search Indexing", diff --git a/config/llm.yml b/config/llm.yml index 83cd9355f..b54a2cbb6 100644 --- a/config/llm.yml +++ b/config/llm.yml @@ -59,6 +59,10 @@ models: provider: openai display_name: 'Whisper' credit_multiplier: 1 + gpt-4o-mini-transcribe: + provider: openai + display_name: 'GPT-4o Mini Transcribe' + credit_multiplier: 1 text-embedding-3-small: provider: openai display_name: 'Text Embedding 3 Small' @@ -82,6 +86,7 @@ features: assistant: models: [ + gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, @@ -91,10 +96,11 @@ features: gemini-3-flash, gemini-3-pro, ] - default: gpt-5.1 + default: gpt-4.1 copilot: models: [ + gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, @@ -104,11 +110,11 @@ features: gemini-3-flash, gemini-3-pro, ] - default: gpt-5.1 + default: gpt-4.1 label_suggestion: models: [gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini, gemini-3-flash, claude-haiku-4.5] - default: gpt-4.1-nano + default: gpt-4.1-mini document_faq_generation: models: [ @@ -148,8 +154,8 @@ features: models: [gpt-4.1-nano, gpt-4.1-mini, gpt-5-mini] default: gpt-4.1-nano audio_transcription: - models: [whisper-1] - default: whisper-1 + models: [gpt-4o-mini-transcribe, whisper-1] + default: gpt-4o-mini-transcribe help_center_search: models: [text-embedding-3-small] default: text-embedding-3-small diff --git a/enterprise/app/services/captain/llm/article_translation_service.rb b/enterprise/app/services/captain/llm/article_translation_service.rb index fab0f934c..e086bdbac 100644 --- a/enterprise/app/services/captain/llm/article_translation_service.rb +++ b/enterprise/app/services/captain/llm/article_translation_service.rb @@ -6,7 +6,7 @@ class Captain::Llm::ArticleTranslationService < Captain::BaseTaskService def perform raise ArgumentError, "Invalid type: #{type}" unless TYPES.include?(type) - response = make_api_call(feature: 'help_center_article_generation', messages: messages) + response = make_api_call(feature: 'help_center_article_generation', model: translation_model, messages: messages) return response if response[:error] response.merge(message: response[:message].strip) @@ -33,6 +33,10 @@ class Captain::Llm::ArticleTranslationService < Captain::BaseTaskService @llm_credential ||= system_llm_credential end + def translation_model + @translation_model ||= InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value.presence || GPT_MODEL + end + def title_system_prompt <<~SYSTEM_PROMPT_MESSAGE You are a professional translator. diff --git a/enterprise/app/services/messages/audio_transcription_service.rb b/enterprise/app/services/messages/audio_transcription_service.rb index 93ed867f2..ccda0368c 100644 --- a/enterprise/app/services/messages/audio_transcription_service.rb +++ b/enterprise/app/services/messages/audio_transcription_service.rb @@ -1,14 +1,13 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService include Integrations::LlmInstrumentation - TRANSCRIPTION_MODEL = 'whisper-1'.freeze # OpenAI's transcription endpoint hard limit is 25 MB *decimal* (25_000_000), not # binary (25.megabytes = 26_214_400) — using the binary form leaks the 25.0–26.2 MB # range to the API as 413s. Long audio (~70+ min Opus) keeps the attachment but skips # transcription. TRANSCRIPTION_BYTE_LIMIT = 25_000_000 - attr_reader :attachment, :message, :account + attr_reader :attachment, :message, :account, :transcription_model def initialize(attachment) super() @@ -128,8 +127,4 @@ class Messages::AudioTranscriptionService< Llm::LegacyBaseOpenAiService 'x-mp3' => 'mp3' }.fetch(subtype, subtype) end - - def transcription_model - @transcription_model || TRANSCRIPTION_MODEL - end end diff --git a/spec/enterprise/services/captain/llm/article_translation_service_spec.rb b/spec/enterprise/services/captain/llm/article_translation_service_spec.rb index 0661a906a..c31846f68 100644 --- a/spec/enterprise/services/captain/llm/article_translation_service_spec.rb +++ b/spec/enterprise/services/captain/llm/article_translation_service_spec.rb @@ -5,6 +5,7 @@ RSpec.describe Captain::Llm::ArticleTranslationService do let(:target_language) { 'Spanish' } before do + InstallationConfig.where(name: %w[CAPTAIN_OPEN_AI_API_KEY CAPTAIN_OPEN_AI_MODEL]).destroy_all create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') allow(account).to receive(:feature_enabled?).and_call_original allow(account).to receive(:feature_enabled?).with('captain_tasks').and_return(true) @@ -18,6 +19,7 @@ RSpec.describe Captain::Llm::ArticleTranslationService do it 'returns the stripped translated title' do expect(service).to receive(:make_api_call) do |args| expect(args[:feature]).to eq('help_center_article_generation') + expect(args[:model]).to eq(Llm::Config::DEFAULT_MODEL) expect(args[:messages][0][:content]).to include('professional translator') expect(args[:messages][0][:content]).to include(target_language) expect(args[:messages][1][:content]).to eq('Getting Started') @@ -26,6 +28,19 @@ RSpec.describe Captain::Llm::ArticleTranslationService do expect(service.perform).to include(message: 'Primeros pasos') end + + it 'uses the installation model when no account override is configured' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + + expect(service).to receive(:make_api_call).with( + hash_including( + feature: 'help_center_article_generation', + model: 'gpt-4.1-nano' + ) + ).and_return(message: 'Primeros pasos') + + expect(service.perform).to include(message: 'Primeros pasos') + end end describe '#perform with type: :content' do diff --git a/spec/enterprise/services/llm/base_ai_service_spec.rb b/spec/enterprise/services/llm/base_ai_service_spec.rb index f66e6bb81..f18752485 100644 --- a/spec/enterprise/services/llm/base_ai_service_spec.rb +++ b/spec/enterprise/services/llm/base_ai_service_spec.rb @@ -31,7 +31,7 @@ RSpec.describe Llm::BaseAiService do end it 'uses the feature default when feature context has no account override or installation model' do - expect(described_class.new(feature: 'assistant', account: account).model).to eq('gpt-5.1') + expect(described_class.new(feature: 'assistant', account: account).model).to eq(Llm::Models.default_model_for('assistant')) end end diff --git a/spec/enterprise/services/messages/audio_transcription_service_spec.rb b/spec/enterprise/services/messages/audio_transcription_service_spec.rb index ecc574501..265ce6c33 100644 --- a/spec/enterprise/services/messages/audio_transcription_service_spec.rb +++ b/spec/enterprise/services/messages/audio_transcription_service_spec.rb @@ -120,7 +120,7 @@ RSpec.describe Messages::AudioTranscriptionService, type: :service do it 'uses the audio transcription feature model' do expect(audio_api).to receive(:transcribe).with( - parameters: hash_including(model: 'whisper-1', temperature: 0.0) + parameters: hash_including(model: 'gpt-4o-mini-transcribe', temperature: 0.0) ).and_return({ 'text' => 'Audio transcript' }) expect(service.send(:transcribe_audio)).to eq('Audio transcript')