diff --git a/config/features.yml b/config/features.yml
index f16199932..92d004f81 100644
--- a/config/features.yml
+++ b/config/features.yml
@@ -144,10 +144,11 @@
- name: chatwoot_v4
display_name: Chatwoot V4
enabled: true
-- name: report_v4
- display_name: Report V4
+- name: captain_v1_action_classifier
+ display_name: Captain V1 Action Classifier
enabled: false
- deprecated: true
+ premium: true
+ chatwoot_internal: true
- name: contact_chatwoot_support_team
display_name: Contact Chatwoot Support Team
enabled: true
diff --git a/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb
new file mode 100644
index 000000000..8f7056ba7
--- /dev/null
+++ b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb
@@ -0,0 +1,15 @@
+class RepurposeReportV4FlagForCaptainV1ActionClassifier < ActiveRecord::Migration[7.1]
+ def up
+ Account.feature_captain_v1_action_classifier.find_each(batch_size: 100) do |account|
+ account.disable_features(:captain_v1_action_classifier)
+ account.save!(validate: false)
+ end
+
+ config = InstallationConfig.find_by(name: 'ACCOUNT_LEVEL_FEATURE_DEFAULTS')
+ return if config&.value.blank?
+
+ config.value = config.value.reject { |feature| feature['name'] == 'report_v4' }
+ config.save!
+ GlobalConfig.clear_cache
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 4ff250ada..9ce734ba7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.1].define(version: 2026_04_28_120000) do
+ActiveRecord::Schema[7.1].define(version: 2026_04_30_114500) do
# These extensions should be enabled to support this database
enable_extension "pg_stat_statements"
enable_extension "pg_trgm"
diff --git a/enterprise/app/helpers/captain/chat_response_helper.rb b/enterprise/app/helpers/captain/chat_response_helper.rb
index bfb8adc11..e8ac1044c 100644
--- a/enterprise/app/helpers/captain/chat_response_helper.rb
+++ b/enterprise/app/helpers/captain/chat_response_helper.rb
@@ -35,7 +35,10 @@ module Captain::ChatResponseHelper
def credit_used_for_response?(parsed_response)
response = parsed_response['response']
- response.present? && response != 'conversation_handoff'
+
+ # The classifier can still decide to hand off after this trace is written.
+ # Actual response usage is charged later in ResponseBuilderJob, so billing stays correct.
+ response.present? && response != 'conversation_handoff' && parsed_response['action'] != 'handoff'
end
def captain_v1_assistant?
diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
index 5e7c5b3c2..5050f11b2 100644
--- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb
+++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
@@ -1,4 +1,6 @@
class Captain::Conversation::ResponseBuilderJob < ApplicationJob
+ include Captain::Conversation::V1ActionClassifier
+
MAX_MESSAGE_LENGTH = 10_000
retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds
@@ -31,9 +33,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
delegate :account, :inbox, to: :@conversation
def generate_and_process_response
+ message_history = collect_previous_messages
@response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
- message_history: collect_previous_messages
+ message_history: message_history
)
+ classify_v1_response_action(message_history) if conversation_pending?
process_response
end
@@ -102,6 +106,14 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
end
def v1_handoff_requested?
+ legacy_v1_handoff_token? || classifier_v1_handoff_requested?
+ end
+
+ def classifier_v1_handoff_requested?
+ @response['action'] == 'handoff'
+ end
+
+ def legacy_v1_handoff_token?
@response['response'] == 'conversation_handoff'
end
@@ -111,8 +123,13 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
def process_v1_handoff
I18n.with_locale(@assistant.account.locale) do
+ Rails.logger.info(
+ "[CAPTAIN][ResponseBuilderJob] V1 handoff requested for account=#{account.id} conversation=#{@conversation.display_id} " \
+ "source=#{@response&.dig('action_source') || 'legacy'} reason=#{@response&.dig('action_reason')}"
+ )
create_handoff_message
@conversation.bot_handoff!
+ report_v1_handoff_not_executed if conversation_pending?
send_out_of_office_message_if_applicable
end
end
@@ -166,6 +183,9 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
def handle_error(error)
log_error(error)
+ @response ||= {}
+ @response['action_source'] ||= 'error'
+ @response['action_reason'] ||= error_action_reason(error)
process_v1_handoff if conversation_pending?
true
end
@@ -174,10 +194,23 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
ChatwootExceptionTracker.new(error, account: account).capture_exception
end
+ def error_action_reason(error)
+ error.class.name.underscore.tr('/', '_')
+ end
+
def captain_v2_enabled?
account.feature_enabled?('captain_integration_v2')
end
+ def report_v1_handoff_not_executed
+ error = StandardError.new("Captain V1 handoff requested but conversation #{@conversation.display_id} is still pending")
+ ChatwootExceptionTracker.new(error, account: account).capture_exception
+ Rails.logger.error(
+ "[CAPTAIN][ResponseBuilderJob] V1 handoff requested but not executed for account=#{account.id} " \
+ "conversation=#{@conversation.display_id}"
+ )
+ end
+
def conversation_pending?
status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) }
status == 'pending' || status == Conversation.statuses[:pending]
diff --git a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
new file mode 100644
index 000000000..9d2010b79
--- /dev/null
+++ b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
@@ -0,0 +1,57 @@
+module Captain::Conversation::V1ActionClassifier
+ private
+
+ def v1_action_classifier_enabled?
+ account.feature_enabled?('captain_v1_action_classifier')
+ end
+
+ def classify_v1_response_action(message_history)
+ return unless v1_action_classifier_enabled?
+ return if legacy_v1_handoff_token?
+
+ classification = Captain::Llm::AssistantActionClassifierService.new(
+ assistant: @assistant,
+ conversation: @conversation
+ ).classify(message_history: message_history, assistant_response: @response['response'])
+
+ apply_v1_action_classification(classification)
+ rescue StandardError => e
+ ChatwootExceptionTracker.new(e, account: account).capture_exception
+ Rails.logger.warn(
+ "[CAPTAIN][ResponseBuilderJob] V1 action classifier failed for account=#{account.id} " \
+ "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
+ )
+ end
+
+ def apply_v1_action_classification(classification)
+ action = classification['action']
+ return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action)
+
+ @response.merge!(
+ 'action' => action,
+ 'action_reason' => classification['action_reason'],
+ 'action_source' => 'classifier',
+ 'action_classifier_model' => classification['model']
+ )
+
+ log_v1_action_classification(action, classification)
+ end
+
+ def log_v1_action_classification(action, classification)
+ Rails.logger.info(
+ "[CAPTAIN][ResponseBuilderJob] V1 action classifier account=#{account.id} conversation=#{@conversation.display_id} " \
+ "action=#{action} reason=#{classification['action_reason']} model=#{classification['model']}"
+ )
+ end
+
+ def valid_v1_action_classification?(action)
+ Captain::AssistantActionSchema::ACTIONS.include?(action)
+ end
+
+ def log_invalid_v1_action_classification(classification)
+ Rails.logger.warn(
+ '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \
+ "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
+ )
+ end
+end
diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
new file mode 100644
index 000000000..7c0f1e91e
--- /dev/null
+++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
@@ -0,0 +1,148 @@
+class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
+ include Integrations::LlmInstrumentation
+
+ MAX_CONTEXT_MESSAGES = 10
+
+ def initialize(assistant:, conversation:)
+ super()
+ @assistant = assistant
+ @conversation = conversation
+ @temperature = 0.0
+ end
+
+ def classify(message_history:, assistant_response:)
+ user_prompt = classification_user_prompt(
+ message_history: message_history,
+ assistant_response: assistant_response
+ )
+
+ response = instrument_llm_call(instrumentation_params(user_prompt)) do
+ chat(model: @model, temperature: @temperature)
+ .with_schema(Captain::AssistantActionSchema)
+ .with_instructions(system_prompt)
+ .ask(user_prompt)
+ end
+
+ parsed = parse_response(response.content)
+ normalize_response(parsed, response.content)
+ rescue StandardError => e
+ ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
+ Rails.logger.warn(
+ "[CAPTAIN][AssistantActionClassifier] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
+ )
+ { 'action' => nil, 'action_reason' => nil, 'error' => e.message, 'model' => @model }
+ end
+
+ private
+
+ def classification_user_prompt(message_history:, assistant_response:)
+ <<~PROMPT
+
+ #{@assistant.config['instructions']}
+
+
+
+ #{format_conversation_context(message_history)}
+
+
+
+ #{assistant_response}
+
+ PROMPT
+ end
+
+ def normalize_messages(message_history)
+ message_history.filter_map do |message|
+ role = message[:role] || message['role']
+ next if role.blank?
+
+ { role: role.to_s, content: normalize_content(message[:content] || message['content']) }
+ end
+ end
+
+ def normalize_content(content)
+ return content if content.is_a?(String)
+ return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
+
+ content.to_s
+ end
+
+ def text_part?(part)
+ return false unless part.is_a?(Hash)
+
+ (part[:type] || part['type']).to_s == 'text'
+ end
+
+ def format_conversation_context(messages)
+ normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+ content = message[:content].to_s.strip
+ next if content.blank?
+
+ "#{role_label(message[:role])}: #{content}"
+ end.join("\n")
+ end
+
+ def role_label(role)
+ return 'User' if role == 'user'
+ return 'Assistant' if role == 'assistant'
+
+ role.to_s.titleize
+ end
+
+ def parse_response(content)
+ return content if content.is_a?(Hash)
+
+ JSON.parse(sanitize_json_response(content))
+ rescue JSON::ParserError, TypeError
+ {}
+ end
+
+ def normalize_response(parsed, raw_content)
+ action = parsed['action'].to_s
+ reason = parsed['action_reason'].to_s
+ return invalid_response(raw_content) unless Captain::AssistantActionSchema::ACTIONS.include?(action)
+
+ {
+ 'action' => action,
+ 'action_reason' => reason.presence,
+ 'raw_response' => raw_content,
+ 'model' => @model
+ }
+ end
+
+ def invalid_response(raw_content)
+ {
+ 'action' => nil,
+ 'action_reason' => nil,
+ 'raw_response' => raw_content,
+ 'error' => 'invalid_classifier_response',
+ 'model' => @model
+ }
+ end
+
+ def instrumentation_params(user_prompt)
+ {
+ span_name: 'llm.captain.assistant_action_classifier',
+ model: @model,
+ temperature: @temperature,
+ account_id: @conversation.account_id,
+ conversation_id: @conversation.display_id,
+ feature_name: 'assistant_action_classifier',
+ messages: [
+ { role: 'system', content: system_prompt },
+ { role: 'user', content: user_prompt }
+ ],
+ metadata: {
+ assistant_id: @assistant.id,
+ channel_type: @conversation.inbox&.channel_type,
+ source: 'v1_response_builder'
+ }
+ }
+ end
+
+ def system_prompt
+ Captain::Llm::SystemPromptsService.assistant_action_classifier(
+ has_custom_instructions: @assistant.config['instructions'].present?
+ )
+ end
+end
diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb
index eb8c334f4..9520330f6 100644
--- a/enterprise/app/services/captain/llm/system_prompts_service.rb
+++ b/enterprise/app/services/captain/llm/system_prompts_service.rb
@@ -93,6 +93,50 @@ class Captain::Llm::SystemPromptsService
SYSTEM_PROMPT_MESSAGE
end
+ def assistant_action_classifier(has_custom_instructions: false)
+ <<~PROMPT
+ You are a routing classifier for a customer-support assistant.
+
+ Decide whether the current conversation should stay with the assistant or be transferred to a human agent now.
+
+ The action field MUST be one of:
+ - "continue": keep the current conversation with the assistant.
+ - "handoff": transfer the current conversation to a human agent now.
+
+ The action_reason field MUST be one of:
+ - "general_product_question"
+ - "missing_docs_bounded_answer"
+ - "clarifying_question_needed"
+ - "collect_required_identifier"
+ - "external_contact_or_lead_routing"
+ - "out_of_scope_bounded_answer"
+ - "explicit_human_request"
+ - "human_offer_accepted"
+ - "account_or_transaction_verification"
+ - "operational_issue_needs_inspection"
+ - "repeated_frustration_or_loop"
+ - "custom_instruction_transfer"
+
+ Use "continue" when:
+ - The user has a general product, pricing, capability, setup, pre-sales, or how-to question.
+ - The assistant can give a bounded answer, ask one useful clarifying question, collect a missing identifier, or share an approved external contact path.
+ - The assistant says someone will contact the user outside this conversation, but the current conversation itself does not need to be transferred now.
+ - The user has not explicitly asked for a human and the assistant is still collecting required details.
+
+ Use "handoff" when:
+ - The user explicitly asks for a human, agent, representative, phone call, callback, or escalation.
+ - The user accepts an offer to speak with a human.
+ - The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access.
+ - The user reports the same unresolved bug or operational issue after trying the assistant's suggested step, repeating the action, checking again, or otherwise making more than one reasonable attempt.
+ - The user is repeatedly frustrated, distrustful, or stuck in a loop.
+ - The assistant response itself says the current conversation will be transferred to a human agent now.
+
+ #{assistant_action_classifier_custom_instructions_policy if has_custom_instructions}
+
+ Return only the structured fields requested by the response schema.
+ PROMPT
+ end
+
# rubocop:disable Metrics/MethodLength
def copilot_response_generator(product_name, available_tools, config = {})
citation_guidelines = if config['feature_citation']
@@ -208,7 +252,9 @@ class Captain::Llm::SystemPromptsService
- Do not share anything outside of the context provided.
- Add the reasoning why you arrived at the answer
- Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format.
- #{config['instructions'] || ''}
+
+ #{build_custom_instructions_section(config['instructions'])}
+
```json
{
reasoning: '',
@@ -322,6 +368,17 @@ class Captain::Llm::SystemPromptsService
TOOLS
end
+ def assistant_action_classifier_custom_instructions_policy
+ <<~POLICY
+ Account custom instructions are provided inside tags.
+ These are instructions configured by the account administrator, not the current end user's message.
+ Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
+ If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
+ Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
+ Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
+ POLICY
+ end
+
def build_contact_context(contact)
return '' if contact.nil?
@@ -331,6 +388,18 @@ class Captain::Llm::SystemPromptsService
"[Contact Information]\n#{lines.join("\n")}\n\n"
end
+ def build_custom_instructions_section(instructions)
+ return '' if instructions.blank?
+
+ <<~CUSTOM_INSTRUCTIONS
+ [Account Custom Instructions]
+ These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context.
+
+ #{instructions}
+
+ CUSTOM_INSTRUCTIONS
+ end
+
def contact_basic_lines(contact)
[
(["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?),
diff --git a/enterprise/lib/captain/assistant_action_schema.rb b/enterprise/lib/captain/assistant_action_schema.rb
new file mode 100644
index 000000000..e8a8de435
--- /dev/null
+++ b/enterprise/lib/captain/assistant_action_schema.rb
@@ -0,0 +1,20 @@
+class Captain::AssistantActionSchema < RubyLLM::Schema
+ ACTIONS = %w[continue handoff].freeze
+ REASONS = %w[
+ general_product_question
+ missing_docs_bounded_answer
+ clarifying_question_needed
+ collect_required_identifier
+ external_contact_or_lead_routing
+ out_of_scope_bounded_answer
+ explicit_human_request
+ human_offer_accepted
+ account_or_transaction_verification
+ operational_issue_needs_inspection
+ repeated_frustration_or_loop
+ custom_instruction_transfer
+ ].freeze
+
+ string :action, enum: ACTIONS, description: 'Whether to keep the conversation with the assistant or transfer it to a human agent'
+ string :action_reason, enum: REASONS, description: 'The reason for the selected routing action'
+end
diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
index 548b84992..8fac81d60 100644
--- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
+++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
@@ -10,6 +10,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
let(:conversation) { create(:conversation, inbox: inbox, account: account, status: :pending) }
let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
+ let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
before do
create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -19,6 +20,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
+ allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
+ allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' })
end
context 'when captain_v2 is disabled' do
@@ -48,6 +51,107 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
end
+ it 'does not run the action classifier when the classifier feature is disabled' do
+ expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
+ end
+
+ context 'when V1 action classifier is enabled' do
+ before do
+ allow(account).to receive(:feature_enabled?).and_return(false)
+ allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
+ allow(account).to receive(:feature_enabled?).with('captain_v1_action_classifier').and_return(true)
+ end
+
+ it 'keeps the conversation pending when the classifier returns continue' do
+ expect(Captain::Llm::AssistantActionClassifierService).to receive(:new).with(
+ assistant: assistant,
+ conversation: conversation
+ ).and_return(mock_action_classifier_service)
+ expect(mock_action_classifier_service).to receive(:classify).with(
+ message_history: [{ content: 'Hello', role: 'user' }],
+ assistant_response: 'Hey, welcome to Captain Specs'
+ ).and_return({
+ 'action' => 'continue',
+ 'action_reason' => 'general_product_question',
+ 'model' => 'gpt-4.1'
+ })
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.reload.status).to eq('pending')
+ expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+ expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+ end
+
+ it 'hands off without incrementing response usage when the classifier returns handoff' do
+ allow(mock_action_classifier_service).to receive(:classify).and_return({
+ 'action' => 'handoff',
+ 'action_reason' => 'explicit_human_request',
+ 'model' => 'gpt-4.1'
+ })
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.reload.status).to eq('open')
+ expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
+ expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
+ end
+
+ it 'skips the classifier when the legacy handoff token is returned' do
+ allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'conversation_handoff' })
+ expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.reload.status).to eq('open')
+ expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
+ end
+
+ it 'falls back to the assistant response when the classifier fails' do
+ error = StandardError.new('classifier unavailable')
+ allow(mock_action_classifier_service).to receive(:classify).and_raise(error)
+ allow(ChatwootExceptionTracker).to receive(:new).and_call_original
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(ChatwootExceptionTracker).to have_received(:new).with(error, account: account)
+ expect(conversation.reload.status).to eq('pending')
+ expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+ expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+ end
+
+ it 'falls back to the assistant response when the classifier returns an invalid action' do
+ allow(mock_action_classifier_service).to receive(:classify).and_return({
+ 'action' => nil,
+ 'error' => 'invalid_classifier_response'
+ })
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.reload.status).to eq('pending')
+ expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+ expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+ end
+
+ it 'skips the classifier when the conversation is no longer pending after response generation' do
+ allow(mock_llm_chat_service).to receive(:generate_response) do
+ conversation.open!
+ { 'response' => 'Hey, welcome to Captain Specs' }
+ end
+
+ expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+ described_class.perform_now(conversation, assistant)
+
+ expect(conversation.messages.outgoing.count).to eq(0)
+ expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
+ end
+ end
+
it 'does not send a response when the conversation is no longer pending' do
conversation.open!
@@ -292,9 +396,11 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
it 'handles API errors and triggers handoff' do
allow(mock_llm_chat_service).to receive(:generate_response)
.and_raise(Faraday::BadRequestError, 'Bad request to image service')
+ allow(Rails.logger).to receive(:info).and_call_original
described_class.perform_now(conversation, assistant)
expect(conversation.reload.status).to eq('open')
+ expect(Rails.logger).to have_received(:info).with(include('source=error reason=faraday_bad_request_error'))
end
it 'succeeds when no error occurs' do
diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
new file mode 100644
index 000000000..260e3f4f7
--- /dev/null
+++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
@@ -0,0 +1,95 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Llm::AssistantActionClassifierService do
+ let(:account) { create(:account) }
+ let(:assistant) do
+ create(
+ :captain_assistant,
+ account: account,
+ config: {
+ 'instructions' => 'Only transfer to a manager after the user explicitly confirms.'
+ }
+ )
+ end
+ let(:conversation) { create(:conversation, account: account) }
+ let(:service) { described_class.new(assistant: assistant, conversation: conversation) }
+ let(:mock_chat) { instance_double(RubyLLM::Chat) }
+ let(:mock_response) do
+ instance_double(
+ RubyLLM::Message,
+ content: { 'action' => 'handoff', 'action_reason' => 'human_offer_accepted' }
+ )
+ end
+
+ before do
+ allow(RubyLLM).to receive(:chat).and_return(mock_chat)
+ allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
+ allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
+ allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
+ end
+
+ describe '#classify' do
+ let(:message_history) do
+ [
+ { role: 'user', content: 'I cannot log in' },
+ { role: 'assistant', content: 'Did you check your inbox?' },
+ { role: 'user', content: 'Yes, still no reset email' }
+ ]
+ end
+
+ it 'passes delimited custom instructions and classifier context to the LLM' do
+ expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat)
+ expect(mock_chat).to receive(:with_instructions).with(
+ a_string_including('Account custom instructions are provided inside tags.')
+ ).and_return(mock_chat)
+ expect(mock_chat).to receive(:ask) do |prompt|
+ expect(prompt).to include(
+ '',
+ 'Only transfer to a manager after the user explicitly confirms.',
+ '',
+ 'User: I cannot log in',
+ 'Assistant: Did you check your inbox?',
+ 'User: Yes, still no reset email',
+ '',
+ 'Would you like to talk to support?'
+ )
+ expect(prompt).not_to include('"role"', '"content"', '')
+
+ mock_response
+ end
+
+ result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+
+ expect(result).to include(
+ 'action' => 'handoff',
+ 'action_reason' => 'human_offer_accepted'
+ )
+ end
+
+ it 'uses the configured Captain model' do
+ create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
+
+ expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-nano').and_return(mock_chat)
+ allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+ result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+
+ expect(result).to include('model' => 'gpt-4.1-nano')
+ end
+
+ context 'when the assistant has no custom instructions' do
+ before do
+ assistant.update!(config: assistant.config.except('instructions'))
+ end
+
+ it 'does not add custom-instruction policy to the system prompt' do
+ expect(mock_chat).to receive(:with_instructions).with(
+ satisfy { |prompt| prompt.exclude?('Account custom instructions are provided') }
+ ).and_return(mock_chat)
+ allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+ service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+ end
+ end
+ end
+end
diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
index c43eb08bd..9d233943e 100644
--- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
+++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
@@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do
end
end
end
+
+ describe 'account custom instructions in system prompt' do
+ before do
+ assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff'))
+ end
+
+ it 'adds custom instructions in a separate delimited section' do
+ allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+ expect(mock_chat).to receive(:with_instructions).with(
+ a_string_including(
+ '',
+ 'if user enters 1112234 suggest handoff',
+ ''
+ )
+ ) do |instructions|
+ expect(instructions).not_to include('')
+ expect(instructions.index('')).to be < instructions.index('```json')
+ mock_chat
+ end
+
+ service = described_class.new(assistant: assistant, conversation: conversation)
+ service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
+ end
+ end
end