diff --git a/config/features.yml b/config/features.yml index f16199932..92d004f81 100644 --- a/config/features.yml +++ b/config/features.yml @@ -144,10 +144,11 @@ - name: chatwoot_v4 display_name: Chatwoot V4 enabled: true -- name: report_v4 - display_name: Report V4 +- name: captain_v1_action_classifier + display_name: Captain V1 Action Classifier enabled: false - deprecated: true + premium: true + chatwoot_internal: true - name: contact_chatwoot_support_team display_name: Contact Chatwoot Support Team enabled: true diff --git a/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb new file mode 100644 index 000000000..8f7056ba7 --- /dev/null +++ b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb @@ -0,0 +1,15 @@ +class RepurposeReportV4FlagForCaptainV1ActionClassifier < ActiveRecord::Migration[7.1] + def up + Account.feature_captain_v1_action_classifier.find_each(batch_size: 100) do |account| + account.disable_features(:captain_v1_action_classifier) + account.save!(validate: false) + end + + config = InstallationConfig.find_by(name: 'ACCOUNT_LEVEL_FEATURE_DEFAULTS') + return if config&.value.blank? + + config.value = config.value.reject { |feature| feature['name'] == 'report_v4' } + config.save! + GlobalConfig.clear_cache + end +end diff --git a/db/schema.rb b/db/schema.rb index 4ff250ada..9ce734ba7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_04_28_120000) do +ActiveRecord::Schema[7.1].define(version: 2026_04_30_114500) do # These extensions should be enabled to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" diff --git a/enterprise/app/helpers/captain/chat_response_helper.rb b/enterprise/app/helpers/captain/chat_response_helper.rb index bfb8adc11..e8ac1044c 100644 --- a/enterprise/app/helpers/captain/chat_response_helper.rb +++ b/enterprise/app/helpers/captain/chat_response_helper.rb @@ -35,7 +35,10 @@ module Captain::ChatResponseHelper def credit_used_for_response?(parsed_response) response = parsed_response['response'] - response.present? && response != 'conversation_handoff' + + # The classifier can still decide to hand off after this trace is written. + # Actual response usage is charged later in ResponseBuilderJob, so billing stays correct. + response.present? && response != 'conversation_handoff' && parsed_response['action'] != 'handoff' end def captain_v1_assistant? diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index 5e7c5b3c2..5050f11b2 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -1,4 +1,6 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob + include Captain::Conversation::V1ActionClassifier + MAX_MESSAGE_LENGTH = 10_000 retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds @@ -31,9 +33,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob delegate :account, :inbox, to: :@conversation def generate_and_process_response + message_history = collect_previous_messages @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response( - message_history: collect_previous_messages + message_history: message_history ) + classify_v1_response_action(message_history) if conversation_pending? process_response end @@ -102,6 +106,14 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def v1_handoff_requested? + legacy_v1_handoff_token? || classifier_v1_handoff_requested? + end + + def classifier_v1_handoff_requested? + @response['action'] == 'handoff' + end + + def legacy_v1_handoff_token? @response['response'] == 'conversation_handoff' end @@ -111,8 +123,13 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob def process_v1_handoff I18n.with_locale(@assistant.account.locale) do + Rails.logger.info( + "[CAPTAIN][ResponseBuilderJob] V1 handoff requested for account=#{account.id} conversation=#{@conversation.display_id} " \ + "source=#{@response&.dig('action_source') || 'legacy'} reason=#{@response&.dig('action_reason')}" + ) create_handoff_message @conversation.bot_handoff! + report_v1_handoff_not_executed if conversation_pending? send_out_of_office_message_if_applicable end end @@ -166,6 +183,9 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob def handle_error(error) log_error(error) + @response ||= {} + @response['action_source'] ||= 'error' + @response['action_reason'] ||= error_action_reason(error) process_v1_handoff if conversation_pending? true end @@ -174,10 +194,23 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob ChatwootExceptionTracker.new(error, account: account).capture_exception end + def error_action_reason(error) + error.class.name.underscore.tr('/', '_') + end + def captain_v2_enabled? account.feature_enabled?('captain_integration_v2') end + def report_v1_handoff_not_executed + error = StandardError.new("Captain V1 handoff requested but conversation #{@conversation.display_id} is still pending") + ChatwootExceptionTracker.new(error, account: account).capture_exception + Rails.logger.error( + "[CAPTAIN][ResponseBuilderJob] V1 handoff requested but not executed for account=#{account.id} " \ + "conversation=#{@conversation.display_id}" + ) + end + def conversation_pending? status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) } status == 'pending' || status == Conversation.statuses[:pending] diff --git a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb new file mode 100644 index 000000000..9d2010b79 --- /dev/null +++ b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb @@ -0,0 +1,57 @@ +module Captain::Conversation::V1ActionClassifier + private + + def v1_action_classifier_enabled? + account.feature_enabled?('captain_v1_action_classifier') + end + + def classify_v1_response_action(message_history) + return unless v1_action_classifier_enabled? + return if legacy_v1_handoff_token? + + classification = Captain::Llm::AssistantActionClassifierService.new( + assistant: @assistant, + conversation: @conversation + ).classify(message_history: message_history, assistant_response: @response['response']) + + apply_v1_action_classification(classification) + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: account).capture_exception + Rails.logger.warn( + "[CAPTAIN][ResponseBuilderJob] V1 action classifier failed for account=#{account.id} " \ + "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}" + ) + end + + def apply_v1_action_classification(classification) + action = classification['action'] + return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action) + + @response.merge!( + 'action' => action, + 'action_reason' => classification['action_reason'], + 'action_source' => 'classifier', + 'action_classifier_model' => classification['model'] + ) + + log_v1_action_classification(action, classification) + end + + def log_v1_action_classification(action, classification) + Rails.logger.info( + "[CAPTAIN][ResponseBuilderJob] V1 action classifier account=#{account.id} conversation=#{@conversation.display_id} " \ + "action=#{action} reason=#{classification['action_reason']} model=#{classification['model']}" + ) + end + + def valid_v1_action_classification?(action) + Captain::AssistantActionSchema::ACTIONS.include?(action) + end + + def log_invalid_v1_action_classification(classification) + Rails.logger.warn( + '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \ + "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}" + ) + end +end diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb new file mode 100644 index 000000000..7c0f1e91e --- /dev/null +++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb @@ -0,0 +1,148 @@ +class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService + include Integrations::LlmInstrumentation + + MAX_CONTEXT_MESSAGES = 10 + + def initialize(assistant:, conversation:) + super() + @assistant = assistant + @conversation = conversation + @temperature = 0.0 + end + + def classify(message_history:, assistant_response:) + user_prompt = classification_user_prompt( + message_history: message_history, + assistant_response: assistant_response + ) + + response = instrument_llm_call(instrumentation_params(user_prompt)) do + chat(model: @model, temperature: @temperature) + .with_schema(Captain::AssistantActionSchema) + .with_instructions(system_prompt) + .ask(user_prompt) + end + + parsed = parse_response(response.content) + normalize_response(parsed, response.content) + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception + Rails.logger.warn( + "[CAPTAIN][AssistantActionClassifier] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}" + ) + { 'action' => nil, 'action_reason' => nil, 'error' => e.message, 'model' => @model } + end + + private + + def classification_user_prompt(message_history:, assistant_response:) + <<~PROMPT + + #{@assistant.config['instructions']} + + + + #{format_conversation_context(message_history)} + + + + #{assistant_response} + + PROMPT + end + + def normalize_messages(message_history) + message_history.filter_map do |message| + role = message[:role] || message['role'] + next if role.blank? + + { role: role.to_s, content: normalize_content(message[:content] || message['content']) } + end + end + + def normalize_content(content) + return content if content.is_a?(String) + return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array) + + content.to_s + end + + def text_part?(part) + return false unless part.is_a?(Hash) + + (part[:type] || part['type']).to_s == 'text' + end + + def format_conversation_context(messages) + normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message| + content = message[:content].to_s.strip + next if content.blank? + + "#{role_label(message[:role])}: #{content}" + end.join("\n") + end + + def role_label(role) + return 'User' if role == 'user' + return 'Assistant' if role == 'assistant' + + role.to_s.titleize + end + + def parse_response(content) + return content if content.is_a?(Hash) + + JSON.parse(sanitize_json_response(content)) + rescue JSON::ParserError, TypeError + {} + end + + def normalize_response(parsed, raw_content) + action = parsed['action'].to_s + reason = parsed['action_reason'].to_s + return invalid_response(raw_content) unless Captain::AssistantActionSchema::ACTIONS.include?(action) + + { + 'action' => action, + 'action_reason' => reason.presence, + 'raw_response' => raw_content, + 'model' => @model + } + end + + def invalid_response(raw_content) + { + 'action' => nil, + 'action_reason' => nil, + 'raw_response' => raw_content, + 'error' => 'invalid_classifier_response', + 'model' => @model + } + end + + def instrumentation_params(user_prompt) + { + span_name: 'llm.captain.assistant_action_classifier', + model: @model, + temperature: @temperature, + account_id: @conversation.account_id, + conversation_id: @conversation.display_id, + feature_name: 'assistant_action_classifier', + messages: [ + { role: 'system', content: system_prompt }, + { role: 'user', content: user_prompt } + ], + metadata: { + assistant_id: @assistant.id, + channel_type: @conversation.inbox&.channel_type, + source: 'v1_response_builder' + } + } + end + + def system_prompt + Captain::Llm::SystemPromptsService.assistant_action_classifier( + has_custom_instructions: @assistant.config['instructions'].present? + ) + end +end diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb index eb8c334f4..9520330f6 100644 --- a/enterprise/app/services/captain/llm/system_prompts_service.rb +++ b/enterprise/app/services/captain/llm/system_prompts_service.rb @@ -93,6 +93,50 @@ class Captain::Llm::SystemPromptsService SYSTEM_PROMPT_MESSAGE end + def assistant_action_classifier(has_custom_instructions: false) + <<~PROMPT + You are a routing classifier for a customer-support assistant. + + Decide whether the current conversation should stay with the assistant or be transferred to a human agent now. + + The action field MUST be one of: + - "continue": keep the current conversation with the assistant. + - "handoff": transfer the current conversation to a human agent now. + + The action_reason field MUST be one of: + - "general_product_question" + - "missing_docs_bounded_answer" + - "clarifying_question_needed" + - "collect_required_identifier" + - "external_contact_or_lead_routing" + - "out_of_scope_bounded_answer" + - "explicit_human_request" + - "human_offer_accepted" + - "account_or_transaction_verification" + - "operational_issue_needs_inspection" + - "repeated_frustration_or_loop" + - "custom_instruction_transfer" + + Use "continue" when: + - The user has a general product, pricing, capability, setup, pre-sales, or how-to question. + - The assistant can give a bounded answer, ask one useful clarifying question, collect a missing identifier, or share an approved external contact path. + - The assistant says someone will contact the user outside this conversation, but the current conversation itself does not need to be transferred now. + - The user has not explicitly asked for a human and the assistant is still collecting required details. + + Use "handoff" when: + - The user explicitly asks for a human, agent, representative, phone call, callback, or escalation. + - The user accepts an offer to speak with a human. + - The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access. + - The user reports the same unresolved bug or operational issue after trying the assistant's suggested step, repeating the action, checking again, or otherwise making more than one reasonable attempt. + - The user is repeatedly frustrated, distrustful, or stuck in a loop. + - The assistant response itself says the current conversation will be transferred to a human agent now. + + #{assistant_action_classifier_custom_instructions_policy if has_custom_instructions} + + Return only the structured fields requested by the response schema. + PROMPT + end + # rubocop:disable Metrics/MethodLength def copilot_response_generator(product_name, available_tools, config = {}) citation_guidelines = if config['feature_citation'] @@ -208,7 +252,9 @@ class Captain::Llm::SystemPromptsService - Do not share anything outside of the context provided. - Add the reasoning why you arrived at the answer - Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format. - #{config['instructions'] || ''} + + #{build_custom_instructions_section(config['instructions'])} + ```json { reasoning: '', @@ -322,6 +368,17 @@ class Captain::Llm::SystemPromptsService TOOLS end + def assistant_action_classifier_custom_instructions_policy + <<~POLICY + Account custom instructions are provided inside tags. + These are instructions configured by the account administrator, not the current end user's message. + Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team. + If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above. + Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff. + Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria. + POLICY + end + def build_contact_context(contact) return '' if contact.nil? @@ -331,6 +388,18 @@ class Captain::Llm::SystemPromptsService "[Contact Information]\n#{lines.join("\n")}\n\n" end + def build_custom_instructions_section(instructions) + return '' if instructions.blank? + + <<~CUSTOM_INSTRUCTIONS + [Account Custom Instructions] + These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context. + + #{instructions} + + CUSTOM_INSTRUCTIONS + end + def contact_basic_lines(contact) [ (["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?), diff --git a/enterprise/lib/captain/assistant_action_schema.rb b/enterprise/lib/captain/assistant_action_schema.rb new file mode 100644 index 000000000..e8a8de435 --- /dev/null +++ b/enterprise/lib/captain/assistant_action_schema.rb @@ -0,0 +1,20 @@ +class Captain::AssistantActionSchema < RubyLLM::Schema + ACTIONS = %w[continue handoff].freeze + REASONS = %w[ + general_product_question + missing_docs_bounded_answer + clarifying_question_needed + collect_required_identifier + external_contact_or_lead_routing + out_of_scope_bounded_answer + explicit_human_request + human_offer_accepted + account_or_transaction_verification + operational_issue_needs_inspection + repeated_frustration_or_loop + custom_instruction_transfer + ].freeze + + string :action, enum: ACTIONS, description: 'Whether to keep the conversation with the assistant or transfer it to a human agent' + string :action_reason, enum: REASONS, description: 'The reason for the selected routing action' +end diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 548b84992..8fac81d60 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -10,6 +10,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do let(:conversation) { create(:conversation, inbox: inbox, account: account, status: :pending) } let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) } let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) } + let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) } before do create(:message, conversation: conversation, content: 'Hello', message_type: :incoming) @@ -19,6 +20,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' }) allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service) allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' }) + allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service) + allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' }) end context 'when captain_v2 is disabled' do @@ -48,6 +51,107 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1) end + it 'does not run the action classifier when the classifier feature is disabled' do + expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new) + + described_class.perform_now(conversation, assistant) + + expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs') + end + + context 'when V1 action classifier is enabled' do + before do + allow(account).to receive(:feature_enabled?).and_return(false) + allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false) + allow(account).to receive(:feature_enabled?).with('captain_v1_action_classifier').and_return(true) + end + + it 'keeps the conversation pending when the classifier returns continue' do + expect(Captain::Llm::AssistantActionClassifierService).to receive(:new).with( + assistant: assistant, + conversation: conversation + ).and_return(mock_action_classifier_service) + expect(mock_action_classifier_service).to receive(:classify).with( + message_history: [{ content: 'Hello', role: 'user' }], + assistant_response: 'Hey, welcome to Captain Specs' + ).and_return({ + 'action' => 'continue', + 'action_reason' => 'general_product_question', + 'model' => 'gpt-4.1' + }) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('pending') + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs') + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1) + end + + it 'hands off without incrementing response usage when the classifier returns handoff' do + allow(mock_action_classifier_service).to receive(:classify).and_return({ + 'action' => 'handoff', + 'action_reason' => 'explicit_human_request', + 'model' => 'gpt-4.1' + }) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('open') + expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff')) + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0) + end + + it 'skips the classifier when the legacy handoff token is returned' do + allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'conversation_handoff' }) + expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('open') + expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff')) + end + + it 'falls back to the assistant response when the classifier fails' do + error = StandardError.new('classifier unavailable') + allow(mock_action_classifier_service).to receive(:classify).and_raise(error) + allow(ChatwootExceptionTracker).to receive(:new).and_call_original + + described_class.perform_now(conversation, assistant) + + expect(ChatwootExceptionTracker).to have_received(:new).with(error, account: account) + expect(conversation.reload.status).to eq('pending') + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs') + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1) + end + + it 'falls back to the assistant response when the classifier returns an invalid action' do + allow(mock_action_classifier_service).to receive(:classify).and_return({ + 'action' => nil, + 'error' => 'invalid_classifier_response' + }) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('pending') + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs') + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1) + end + + it 'skips the classifier when the conversation is no longer pending after response generation' do + allow(mock_llm_chat_service).to receive(:generate_response) do + conversation.open! + { 'response' => 'Hey, welcome to Captain Specs' } + end + + expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new) + + described_class.perform_now(conversation, assistant) + + expect(conversation.messages.outgoing.count).to eq(0) + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0) + end + end + it 'does not send a response when the conversation is no longer pending' do conversation.open! @@ -292,9 +396,11 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do it 'handles API errors and triggers handoff' do allow(mock_llm_chat_service).to receive(:generate_response) .and_raise(Faraday::BadRequestError, 'Bad request to image service') + allow(Rails.logger).to receive(:info).and_call_original described_class.perform_now(conversation, assistant) expect(conversation.reload.status).to eq('open') + expect(Rails.logger).to have_received(:info).with(include('source=error reason=faraday_bad_request_error')) end it 'succeeds when no error occurs' do diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb new file mode 100644 index 000000000..260e3f4f7 --- /dev/null +++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb @@ -0,0 +1,95 @@ +require 'rails_helper' + +RSpec.describe Captain::Llm::AssistantActionClassifierService do + let(:account) { create(:account) } + let(:assistant) do + create( + :captain_assistant, + account: account, + config: { + 'instructions' => 'Only transfer to a manager after the user explicitly confirms.' + } + ) + end + let(:conversation) { create(:conversation, account: account) } + let(:service) { described_class.new(assistant: assistant, conversation: conversation) } + let(:mock_chat) { instance_double(RubyLLM::Chat) } + let(:mock_response) do + instance_double( + RubyLLM::Message, + content: { 'action' => 'handoff', 'action_reason' => 'human_offer_accepted' } + ) + end + + before do + allow(RubyLLM).to receive(:chat).and_return(mock_chat) + allow(mock_chat).to receive(:with_temperature).and_return(mock_chat) + allow(mock_chat).to receive(:with_schema).and_return(mock_chat) + allow(mock_chat).to receive(:with_instructions).and_return(mock_chat) + end + + describe '#classify' do + let(:message_history) do + [ + { role: 'user', content: 'I cannot log in' }, + { role: 'assistant', content: 'Did you check your inbox?' }, + { role: 'user', content: 'Yes, still no reset email' } + ] + end + + it 'passes delimited custom instructions and classifier context to the LLM' do + expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat) + expect(mock_chat).to receive(:with_instructions).with( + a_string_including('Account custom instructions are provided inside tags.') + ).and_return(mock_chat) + expect(mock_chat).to receive(:ask) do |prompt| + expect(prompt).to include( + '', + 'Only transfer to a manager after the user explicitly confirms.', + '', + 'User: I cannot log in', + 'Assistant: Did you check your inbox?', + 'User: Yes, still no reset email', + '', + 'Would you like to talk to support?' + ) + expect(prompt).not_to include('"role"', '"content"', '') + + mock_response + end + + result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?') + + expect(result).to include( + 'action' => 'handoff', + 'action_reason' => 'human_offer_accepted' + ) + end + + it 'uses the configured Captain model' do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano') + + expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-nano').and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) + + result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?') + + expect(result).to include('model' => 'gpt-4.1-nano') + end + + context 'when the assistant has no custom instructions' do + before do + assistant.update!(config: assistant.config.except('instructions')) + end + + it 'does not add custom-instruction policy to the system prompt' do + expect(mock_chat).to receive(:with_instructions).with( + satisfy { |prompt| prompt.exclude?('Account custom instructions are provided') } + ).and_return(mock_chat) + allow(mock_chat).to receive(:ask).and_return(mock_response) + + service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?') + end + end + end +end diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb index c43eb08bd..9d233943e 100644 --- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb +++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb @@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do end end end + + describe 'account custom instructions in system prompt' do + before do + assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff')) + end + + it 'adds custom instructions in a separate delimited section' do + allow(mock_chat).to receive(:ask).and_return(mock_response) + + expect(mock_chat).to receive(:with_instructions).with( + a_string_including( + '', + 'if user enters 1112234 suggest handoff', + '' + ) + ) do |instructions| + expect(instructions).not_to include('') + expect(instructions.index('')).to be < instructions.index('```json') + mock_chat + end + + service = described_class.new(assistant: assistant, conversation: conversation) + service.generate_response(message_history: [{ role: 'user', content: 'Hello' }]) + end + end end