From 70f799ab355c1200dc6b7dea4f3039b0db477b08 Mon Sep 17 00:00:00 2001
From: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com>
Date: Tue, 5 May 2026 14:34:31 +0530
Subject: [PATCH] fix(captain): add v1 handoff classifier [AI-137] (#14337)

# Pull Request Template

## Description

Captain (v1) makes false promises by saying it will handoff but doesn't.
This happens due to an exact string match comparison and the prompt
gives the model a lot of responsibilities:
- identity
- what to respond
- obey custom instructions
- decide on tool calls

This PR decouples responsibility, the core prompt responds, and an
additional llm call evaluates if handoff was needed or not after that
message.

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)

## How Has This Been Tested?

Please describe the tests that you ran to verify your changes. Provide
instructions so we can reproduce. Please also list any relevant details
for your test configuration.

Locally


## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have commented on my code, particularly in hard-to-understand
areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream
modules
---
 config/features.yml                           |   7 +-
 ...4_flag_for_captain_v1_action_classifier.rb |  15 ++
 db/schema.rb                                  |   2 +-
 .../helpers/captain/chat_response_helper.rb   |   5 +-
 .../conversation/response_builder_job.rb      |  35 ++++-
 .../conversation/v1_action_classifier.rb      |  57 +++++++
 .../assistant_action_classifier_service.rb    | 148 ++++++++++++++++++
 .../captain/llm/system_prompts_service.rb     |  71 ++++++++-
 .../lib/captain/assistant_action_schema.rb    |  20 +++
 .../conversation/response_builder_job_spec.rb | 106 +++++++++++++
 ...ssistant_action_classifier_service_spec.rb |  95 +++++++++++
 .../llm/assistant_chat_service_spec.rb        |  25 +++
 12 files changed, 579 insertions(+), 7 deletions(-)
 create mode 100644 db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb
 create mode 100644 enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
 create mode 100644 enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
 create mode 100644 enterprise/lib/captain/assistant_action_schema.rb
 create mode 100644 spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb

diff --git a/config/features.yml b/config/features.yml
index f16199932..92d004f81 100644
--- a/config/features.yml
+++ b/config/features.yml
@@ -144,10 +144,11 @@
 - name: chatwoot_v4
   display_name: Chatwoot V4
   enabled: true
-- name: report_v4
-  display_name: Report V4
+- name: captain_v1_action_classifier
+  display_name: Captain V1 Action Classifier
   enabled: false
-  deprecated: true
+  premium: true
+  chatwoot_internal: true
 - name: contact_chatwoot_support_team
   display_name: Contact Chatwoot Support Team
   enabled: true
diff --git a/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb
new file mode 100644
index 000000000..8f7056ba7
--- /dev/null
+++ b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb
@@ -0,0 +1,15 @@
+class RepurposeReportV4FlagForCaptainV1ActionClassifier < ActiveRecord::Migration[7.1]
+  def up
+    Account.feature_captain_v1_action_classifier.find_each(batch_size: 100) do |account|
+      account.disable_features(:captain_v1_action_classifier)
+      account.save!(validate: false)
+    end
+
+    config = InstallationConfig.find_by(name: 'ACCOUNT_LEVEL_FEATURE_DEFAULTS')
+    return if config&.value.blank?
+
+    config.value = config.value.reject { |feature| feature['name'] == 'report_v4' }
+    config.save!
+    GlobalConfig.clear_cache
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 4ff250ada..9ce734ba7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2026_04_28_120000) do
+ActiveRecord::Schema[7.1].define(version: 2026_04_30_114500) do
   # These extensions should be enabled to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"
diff --git a/enterprise/app/helpers/captain/chat_response_helper.rb b/enterprise/app/helpers/captain/chat_response_helper.rb
index bfb8adc11..e8ac1044c 100644
--- a/enterprise/app/helpers/captain/chat_response_helper.rb
+++ b/enterprise/app/helpers/captain/chat_response_helper.rb
@@ -35,7 +35,10 @@ module Captain::ChatResponseHelper
 
   def credit_used_for_response?(parsed_response)
     response = parsed_response['response']
-    response.present? && response != 'conversation_handoff'
+
+    # The classifier can still decide to hand off after this trace is written.
+    # Actual response usage is charged later in ResponseBuilderJob, so billing stays correct.
+    response.present? && response != 'conversation_handoff' && parsed_response['action'] != 'handoff'
   end
 
   def captain_v1_assistant?
diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
index 5e7c5b3c2..5050f11b2 100644
--- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb
+++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
@@ -1,4 +1,6 @@
 class Captain::Conversation::ResponseBuilderJob < ApplicationJob
+  include Captain::Conversation::V1ActionClassifier
+
   MAX_MESSAGE_LENGTH = 10_000
   retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
   retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds
@@ -31,9 +33,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   delegate :account, :inbox, to: :@conversation
 
   def generate_and_process_response
+    message_history = collect_previous_messages
     @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
-      message_history: collect_previous_messages
+      message_history: message_history
     )
+    classify_v1_response_action(message_history) if conversation_pending?
     process_response
   end
 
@@ -102,6 +106,14 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   end
 
   def v1_handoff_requested?
+    legacy_v1_handoff_token? || classifier_v1_handoff_requested?
+  end
+
+  def classifier_v1_handoff_requested?
+    @response['action'] == 'handoff'
+  end
+
+  def legacy_v1_handoff_token?
     @response['response'] == 'conversation_handoff'
   end
 
@@ -111,8 +123,13 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
   def process_v1_handoff
     I18n.with_locale(@assistant.account.locale) do
+      Rails.logger.info(
+        "[CAPTAIN][ResponseBuilderJob] V1 handoff requested for account=#{account.id} conversation=#{@conversation.display_id} " \
+        "source=#{@response&.dig('action_source') || 'legacy'} reason=#{@response&.dig('action_reason')}"
+      )
       create_handoff_message
       @conversation.bot_handoff!
+      report_v1_handoff_not_executed if conversation_pending?
       send_out_of_office_message_if_applicable
     end
   end
@@ -166,6 +183,9 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
   def handle_error(error)
     log_error(error)
+    @response ||= {}
+    @response['action_source'] ||= 'error'
+    @response['action_reason'] ||= error_action_reason(error)
     process_v1_handoff if conversation_pending?
     true
   end
@@ -174,10 +194,23 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
     ChatwootExceptionTracker.new(error, account: account).capture_exception
   end
 
+  def error_action_reason(error)
+    error.class.name.underscore.tr('/', '_')
+  end
+
   def captain_v2_enabled?
     account.feature_enabled?('captain_integration_v2')
   end
 
+  def report_v1_handoff_not_executed
+    error = StandardError.new("Captain V1 handoff requested but conversation #{@conversation.display_id} is still pending")
+    ChatwootExceptionTracker.new(error, account: account).capture_exception
+    Rails.logger.error(
+      "[CAPTAIN][ResponseBuilderJob] V1 handoff requested but not executed for account=#{account.id} " \
+      "conversation=#{@conversation.display_id}"
+    )
+  end
+
   def conversation_pending?
     status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) }
     status == 'pending' || status == Conversation.statuses[:pending]
diff --git a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
new file mode 100644
index 000000000..9d2010b79
--- /dev/null
+++ b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
@@ -0,0 +1,57 @@
+module Captain::Conversation::V1ActionClassifier
+  private
+
+  def v1_action_classifier_enabled?
+    account.feature_enabled?('captain_v1_action_classifier')
+  end
+
+  def classify_v1_response_action(message_history)
+    return unless v1_action_classifier_enabled?
+    return if legacy_v1_handoff_token?
+
+    classification = Captain::Llm::AssistantActionClassifierService.new(
+      assistant: @assistant,
+      conversation: @conversation
+    ).classify(message_history: message_history, assistant_response: @response['response'])
+
+    apply_v1_action_classification(classification)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][ResponseBuilderJob] V1 action classifier failed for account=#{account.id} " \
+      "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+  end
+
+  def apply_v1_action_classification(classification)
+    action = classification['action']
+    return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action)
+
+    @response.merge!(
+      'action' => action,
+      'action_reason' => classification['action_reason'],
+      'action_source' => 'classifier',
+      'action_classifier_model' => classification['model']
+    )
+
+    log_v1_action_classification(action, classification)
+  end
+
+  def log_v1_action_classification(action, classification)
+    Rails.logger.info(
+      "[CAPTAIN][ResponseBuilderJob] V1 action classifier account=#{account.id} conversation=#{@conversation.display_id} " \
+      "action=#{action} reason=#{classification['action_reason']} model=#{classification['model']}"
+    )
+  end
+
+  def valid_v1_action_classification?(action)
+    Captain::AssistantActionSchema::ACTIONS.include?(action)
+  end
+
+  def log_invalid_v1_action_classification(classification)
+    Rails.logger.warn(
+      '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \
+      "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
+    )
+  end
+end
diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
new file mode 100644
index 000000000..7c0f1e91e
--- /dev/null
+++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
@@ -0,0 +1,148 @@
+class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
+  include Integrations::LlmInstrumentation
+
+  MAX_CONTEXT_MESSAGES = 10
+
+  def initialize(assistant:, conversation:)
+    super()
+    @assistant = assistant
+    @conversation = conversation
+    @temperature = 0.0
+  end
+
+  def classify(message_history:, assistant_response:)
+    user_prompt = classification_user_prompt(
+      message_history: message_history,
+      assistant_response: assistant_response
+    )
+
+    response = instrument_llm_call(instrumentation_params(user_prompt)) do
+      chat(model: @model, temperature: @temperature)
+        .with_schema(Captain::AssistantActionSchema)
+        .with_instructions(system_prompt)
+        .ask(user_prompt)
+    end
+
+    parsed = parse_response(response.content)
+    normalize_response(parsed, response.content)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][AssistantActionClassifier] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+    { 'action' => nil, 'action_reason' => nil, 'error' => e.message, 'model' => @model }
+  end
+
+  private
+
+  def classification_user_prompt(message_history:, assistant_response:)
+    <<~PROMPT
+      <account_custom_instructions>
+      #{@assistant.config['instructions']}
+      </account_custom_instructions>
+
+      <conversation_context>
+      #{format_conversation_context(message_history)}
+      </conversation_context>
+
+      <assistant_response_to_classify>
+      #{assistant_response}
+      </assistant_response_to_classify>
+    PROMPT
+  end
+
+  def normalize_messages(message_history)
+    message_history.filter_map do |message|
+      role = message[:role] || message['role']
+      next if role.blank?
+
+      { role: role.to_s, content: normalize_content(message[:content] || message['content']) }
+    end
+  end
+
+  def normalize_content(content)
+    return content if content.is_a?(String)
+    return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
+
+    content.to_s
+  end
+
+  def text_part?(part)
+    return false unless part.is_a?(Hash)
+
+    (part[:type] || part['type']).to_s == 'text'
+  end
+
+  def format_conversation_context(messages)
+    normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+      content = message[:content].to_s.strip
+      next if content.blank?
+
+      "#{role_label(message[:role])}: #{content}"
+    end.join("\n")
+  end
+
+  def role_label(role)
+    return 'User' if role == 'user'
+    return 'Assistant' if role == 'assistant'
+
+    role.to_s.titleize
+  end
+
+  def parse_response(content)
+    return content if content.is_a?(Hash)
+
+    JSON.parse(sanitize_json_response(content))
+  rescue JSON::ParserError, TypeError
+    {}
+  end
+
+  def normalize_response(parsed, raw_content)
+    action = parsed['action'].to_s
+    reason = parsed['action_reason'].to_s
+    return invalid_response(raw_content) unless Captain::AssistantActionSchema::ACTIONS.include?(action)
+
+    {
+      'action' => action,
+      'action_reason' => reason.presence,
+      'raw_response' => raw_content,
+      'model' => @model
+    }
+  end
+
+  def invalid_response(raw_content)
+    {
+      'action' => nil,
+      'action_reason' => nil,
+      'raw_response' => raw_content,
+      'error' => 'invalid_classifier_response',
+      'model' => @model
+    }
+  end
+
+  def instrumentation_params(user_prompt)
+    {
+      span_name: 'llm.captain.assistant_action_classifier',
+      model: @model,
+      temperature: @temperature,
+      account_id: @conversation.account_id,
+      conversation_id: @conversation.display_id,
+      feature_name: 'assistant_action_classifier',
+      messages: [
+        { role: 'system', content: system_prompt },
+        { role: 'user', content: user_prompt }
+      ],
+      metadata: {
+        assistant_id: @assistant.id,
+        channel_type: @conversation.inbox&.channel_type,
+        source: 'v1_response_builder'
+      }
+    }
+  end
+
+  def system_prompt
+    Captain::Llm::SystemPromptsService.assistant_action_classifier(
+      has_custom_instructions: @assistant.config['instructions'].present?
+    )
+  end
+end
diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb
index eb8c334f4..9520330f6 100644
--- a/enterprise/app/services/captain/llm/system_prompts_service.rb
+++ b/enterprise/app/services/captain/llm/system_prompts_service.rb
@@ -93,6 +93,50 @@ class Captain::Llm::SystemPromptsService
       SYSTEM_PROMPT_MESSAGE
     end
 
+    def assistant_action_classifier(has_custom_instructions: false)
+      <<~PROMPT
+        You are a routing classifier for a customer-support assistant.
+
+        Decide whether the current conversation should stay with the assistant or be transferred to a human agent now.
+
+        The action field MUST be one of:
+        - "continue": keep the current conversation with the assistant.
+        - "handoff": transfer the current conversation to a human agent now.
+
+        The action_reason field MUST be one of:
+        - "general_product_question"
+        - "missing_docs_bounded_answer"
+        - "clarifying_question_needed"
+        - "collect_required_identifier"
+        - "external_contact_or_lead_routing"
+        - "out_of_scope_bounded_answer"
+        - "explicit_human_request"
+        - "human_offer_accepted"
+        - "account_or_transaction_verification"
+        - "operational_issue_needs_inspection"
+        - "repeated_frustration_or_loop"
+        - "custom_instruction_transfer"
+
+        Use "continue" when:
+        - The user has a general product, pricing, capability, setup, pre-sales, or how-to question.
+        - The assistant can give a bounded answer, ask one useful clarifying question, collect a missing identifier, or share an approved external contact path.
+        - The assistant says someone will contact the user outside this conversation, but the current conversation itself does not need to be transferred now.
+        - The user has not explicitly asked for a human and the assistant is still collecting required details.
+
+        Use "handoff" when:
+        - The user explicitly asks for a human, agent, representative, phone call, callback, or escalation.
+        - The user accepts an offer to speak with a human.
+        - The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access.
+        - The user reports the same unresolved bug or operational issue after trying the assistant's suggested step, repeating the action, checking again, or otherwise making more than one reasonable attempt.
+        - The user is repeatedly frustrated, distrustful, or stuck in a loop.
+        - The assistant response itself says the current conversation will be transferred to a human agent now.
+
+        #{assistant_action_classifier_custom_instructions_policy if has_custom_instructions}
+
+        Return only the structured fields requested by the response schema.
+      PROMPT
+    end
+
     # rubocop:disable Metrics/MethodLength
     def copilot_response_generator(product_name, available_tools, config = {})
       citation_guidelines = if config['feature_citation']
@@ -208,7 +252,9 @@ class Captain::Llm::SystemPromptsService
         - Do not share anything outside of the context provided.
         - Add the reasoning why you arrived at the answer
         - Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format.
-        #{config['instructions'] || ''}
+
+        #{build_custom_instructions_section(config['instructions'])}
+
         ```json
         {
           reasoning: '',
@@ -322,6 +368,17 @@ class Captain::Llm::SystemPromptsService
       TOOLS
     end
 
+    def assistant_action_classifier_custom_instructions_policy
+      <<~POLICY
+        Account custom instructions are provided inside <account_custom_instructions> tags.
+        These are instructions configured by the account administrator, not the current end user's message.
+        Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
+        If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
+        Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
+        Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
+      POLICY
+    end
+
     def build_contact_context(contact)
       return '' if contact.nil?
 
@@ -331,6 +388,18 @@ class Captain::Llm::SystemPromptsService
       "[Contact Information]\n#{lines.join("\n")}\n\n"
     end
 
+    def build_custom_instructions_section(instructions)
+      return '' if instructions.blank?
+
+      <<~CUSTOM_INSTRUCTIONS
+        [Account Custom Instructions]
+        These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context.
+        <account_custom_instructions>
+        #{instructions}
+        </account_custom_instructions>
+      CUSTOM_INSTRUCTIONS
+    end
+
     def contact_basic_lines(contact)
       [
         (["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?),
diff --git a/enterprise/lib/captain/assistant_action_schema.rb b/enterprise/lib/captain/assistant_action_schema.rb
new file mode 100644
index 000000000..e8a8de435
--- /dev/null
+++ b/enterprise/lib/captain/assistant_action_schema.rb
@@ -0,0 +1,20 @@
+class Captain::AssistantActionSchema < RubyLLM::Schema
+  ACTIONS = %w[continue handoff].freeze
+  REASONS = %w[
+    general_product_question
+    missing_docs_bounded_answer
+    clarifying_question_needed
+    collect_required_identifier
+    external_contact_or_lead_routing
+    out_of_scope_bounded_answer
+    explicit_human_request
+    human_offer_accepted
+    account_or_transaction_verification
+    operational_issue_needs_inspection
+    repeated_frustration_or_loop
+    custom_instruction_transfer
+  ].freeze
+
+  string :action, enum: ACTIONS, description: 'Whether to keep the conversation with the assistant or transfer it to a human agent'
+  string :action_reason, enum: REASONS, description: 'The reason for the selected routing action'
+end
diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
index 548b84992..8fac81d60 100644
--- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
+++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb
@@ -10,6 +10,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
     let(:conversation) { create(:conversation, inbox: inbox, account: account, status: :pending) }
     let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
     let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
+    let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
 
     before do
       create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -19,6 +20,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
       allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
       allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
+      allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
+      allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' })
     end
 
     context 'when captain_v2 is disabled' do
@@ -48,6 +51,107 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
         expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
       end
 
+      it 'does not run the action classifier when the classifier feature is disabled' do
+        expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+        described_class.perform_now(conversation, assistant)
+
+        expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
+      end
+
+      context 'when V1 action classifier is enabled' do
+        before do
+          allow(account).to receive(:feature_enabled?).and_return(false)
+          allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
+          allow(account).to receive(:feature_enabled?).with('captain_v1_action_classifier').and_return(true)
+        end
+
+        it 'keeps the conversation pending when the classifier returns continue' do
+          expect(Captain::Llm::AssistantActionClassifierService).to receive(:new).with(
+            assistant: assistant,
+            conversation: conversation
+          ).and_return(mock_action_classifier_service)
+          expect(mock_action_classifier_service).to receive(:classify).with(
+            message_history: [{ content: 'Hello', role: 'user' }],
+            assistant_response: 'Hey, welcome to Captain Specs'
+          ).and_return({
+                         'action' => 'continue',
+                         'action_reason' => 'general_product_question',
+                         'model' => 'gpt-4.1'
+                       })
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('pending')
+          expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'hands off without incrementing response usage when the classifier returns handoff' do
+          allow(mock_action_classifier_service).to receive(:classify).and_return({
+                                                                                   'action' => 'handoff',
+                                                                                   'action_reason' => 'explicit_human_request',
+                                                                                   'model' => 'gpt-4.1'
+                                                                                 })
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('open')
+          expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
+        end
+
+        it 'skips the classifier when the legacy handoff token is returned' do
+          allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'conversation_handoff' })
+          expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('open')
+          expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
+        end
+
+        it 'falls back to the assistant response when the classifier fails' do
+          error = StandardError.new('classifier unavailable')
+          allow(mock_action_classifier_service).to receive(:classify).and_raise(error)
+          allow(ChatwootExceptionTracker).to receive(:new).and_call_original
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(ChatwootExceptionTracker).to have_received(:new).with(error, account: account)
+          expect(conversation.reload.status).to eq('pending')
+          expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'falls back to the assistant response when the classifier returns an invalid action' do
+          allow(mock_action_classifier_service).to receive(:classify).and_return({
+                                                                                   'action' => nil,
+                                                                                   'error' => 'invalid_classifier_response'
+                                                                                 })
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('pending')
+          expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'skips the classifier when the conversation is no longer pending after response generation' do
+          allow(mock_llm_chat_service).to receive(:generate_response) do
+            conversation.open!
+            { 'response' => 'Hey, welcome to Captain Specs' }
+          end
+
+          expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.count).to eq(0)
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
+        end
+      end
+
       it 'does not send a response when the conversation is no longer pending' do
         conversation.open!
 
@@ -292,9 +396,11 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       it 'handles API errors and triggers handoff' do
         allow(mock_llm_chat_service).to receive(:generate_response)
           .and_raise(Faraday::BadRequestError, 'Bad request to image service')
+        allow(Rails.logger).to receive(:info).and_call_original
 
         described_class.perform_now(conversation, assistant)
         expect(conversation.reload.status).to eq('open')
+        expect(Rails.logger).to have_received(:info).with(include('source=error reason=faraday_bad_request_error'))
       end
 
       it 'succeeds when no error occurs' do
diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
new file mode 100644
index 000000000..260e3f4f7
--- /dev/null
+++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
@@ -0,0 +1,95 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Llm::AssistantActionClassifierService do
+  let(:account) { create(:account) }
+  let(:assistant) do
+    create(
+      :captain_assistant,
+      account: account,
+      config: {
+        'instructions' => 'Only transfer to a manager after the user explicitly confirms.'
+      }
+    )
+  end
+  let(:conversation) { create(:conversation, account: account) }
+  let(:service) { described_class.new(assistant: assistant, conversation: conversation) }
+  let(:mock_chat) { instance_double(RubyLLM::Chat) }
+  let(:mock_response) do
+    instance_double(
+      RubyLLM::Message,
+      content: { 'action' => 'handoff', 'action_reason' => 'human_offer_accepted' }
+    )
+  end
+
+  before do
+    allow(RubyLLM).to receive(:chat).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
+  end
+
+  describe '#classify' do
+    let(:message_history) do
+      [
+        { role: 'user', content: 'I cannot log in' },
+        { role: 'assistant', content: 'Did you check your inbox?' },
+        { role: 'user', content: 'Yes, still no reset email' }
+      ]
+    end
+
+    it 'passes delimited custom instructions and classifier context to the LLM' do
+      expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat)
+      expect(mock_chat).to receive(:with_instructions).with(
+        a_string_including('Account custom instructions are provided inside <account_custom_instructions> tags.')
+      ).and_return(mock_chat)
+      expect(mock_chat).to receive(:ask) do |prompt|
+        expect(prompt).to include(
+          '<account_custom_instructions>',
+          'Only transfer to a manager after the user explicitly confirms.',
+          '<conversation_context>',
+          'User: I cannot log in',
+          'Assistant: Did you check your inbox?',
+          'User: Yes, still no reset email',
+          '<assistant_response_to_classify>',
+          'Would you like to talk to support?'
+        )
+        expect(prompt).not_to include('"role"', '"content"', '<current_user_message>')
+
+        mock_response
+      end
+
+      result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+
+      expect(result).to include(
+        'action' => 'handoff',
+        'action_reason' => 'human_offer_accepted'
+      )
+    end
+
+    it 'uses the configured Captain model' do
+      create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
+
+      expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-nano').and_return(mock_chat)
+      allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+      result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+
+      expect(result).to include('model' => 'gpt-4.1-nano')
+    end
+
+    context 'when the assistant has no custom instructions' do
+      before do
+        assistant.update!(config: assistant.config.except('instructions'))
+      end
+
+      it 'does not add custom-instruction policy to the system prompt' do
+        expect(mock_chat).to receive(:with_instructions).with(
+          satisfy { |prompt| prompt.exclude?('Account custom instructions are provided') }
+        ).and_return(mock_chat)
+        allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+        service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+      end
+    end
+  end
+end
diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
index c43eb08bd..9d233943e 100644
--- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
+++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
@@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do
       end
     end
   end
+
+  describe 'account custom instructions in system prompt' do
+    before do
+      assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff'))
+    end
+
+    it 'adds custom instructions in a separate delimited section' do
+      allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+      expect(mock_chat).to receive(:with_instructions).with(
+        a_string_including(
+          '<account_custom_instructions>',
+          'if user enters 1112234 suggest handoff',
+          '</account_custom_instructions>'
+        )
+      ) do |instructions|
+        expect(instructions).not_to include('<custom-instructions>')
+        expect(instructions.index('<account_custom_instructions>')).to be < instructions.index('```json')
+        mock_chat
+      end
+
+      service = described_class.new(assistant: assistant, conversation: conversation)
+      service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
+    end
+  end
 end