fix(captain): Skip classifier call when conversation is pending

config/features.yml

@@ -144,10 +144,11 @@
 - name: chatwoot_v4
   display_name: Chatwoot V4
   enabled: true
-- name: report_v4
+- name: captain_v1_action_classifier
-  display_name: Report V4
+  display_name: Captain V1 Action Classifier
   enabled: false
-  deprecated: true
+  premium: true
+  chatwoot_internal: true
 - name: contact_chatwoot_support_team
   display_name: Contact Chatwoot Support Team
   enabled: true

db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb

@@ -0,0 +1,15 @@
+class RepurposeReportV4FlagForCaptainV1ActionClassifier < ActiveRecord::Migration[7.1]
+  def up
+    Account.feature_captain_v1_action_classifier.find_each(batch_size: 100) do |account|
+      account.disable_features(:captain_v1_action_classifier)
+      account.save!(validate: false)
+    end
+
+    config = InstallationConfig.find_by(name: 'ACCOUNT_LEVEL_FEATURE_DEFAULTS')
+    return if config&.value.blank?
+
+    config.value = config.value.reject { |feature| feature['name'] == 'report_v4' }
+    config.save!
+    GlobalConfig.clear_cache
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2026_04_10_092753) do
+ActiveRecord::Schema[7.1].define(version: 2026_04_30_114500) do
   # These extensions should be enabled to support this database
   enable_extension "pg_stat_statements"
   enable_extension "pg_trgm"

enterprise/app/helpers/captain/chat_response_helper.rb

@@ -35,7 +35,7 @@ module Captain::ChatResponseHelper
 
   def credit_used_for_response?(parsed_response)
     response = parsed_response['response']
-    response.present? && response != 'conversation_handoff'
+    response.present? && response != 'conversation_handoff' && parsed_response['action'] != 'handoff'
   end
 
   def captain_v1_assistant?

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -1,4 +1,6 @@
 class Captain::Conversation::ResponseBuilderJob < ApplicationJob
+  include Captain::Conversation::V1ActionClassifier
+
   MAX_MESSAGE_LENGTH = 10_000
   retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
   retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds
@@ -31,9 +33,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   delegate :account, :inbox, to: :@conversation
 
   def generate_and_process_response
+    message_history = collect_previous_messages
     @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
-      message_history: collect_previous_messages
+      message_history: message_history
     )
+    classify_v1_response_action(message_history) if conversation_pending?
     process_response
   end
 
@@ -102,6 +106,14 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   end
 
   def v1_handoff_requested?
+    legacy_v1_handoff_token? || classifier_v1_handoff_requested?
+  end
+
+  def classifier_v1_handoff_requested?
+    @response['action'] == 'handoff'
+  end
+
+  def legacy_v1_handoff_token?
     @response['response'] == 'conversation_handoff'
   end
 
@@ -111,8 +123,13 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
   def process_v1_handoff
     I18n.with_locale(@assistant.account.locale) do
+      Rails.logger.info(
+        "[CAPTAIN][ResponseBuilderJob] V1 handoff requested for account=#{account.id} conversation=#{@conversation.display_id} " \
+        "source=#{@response&.dig('action_source') || 'legacy'} reason=#{@response&.dig('action_reason')}"
+      )
       create_handoff_message
       @conversation.bot_handoff!
+      report_v1_handoff_not_executed if conversation_pending?
       send_out_of_office_message_if_applicable
     end
   end
@@ -166,6 +183,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
 
   def handle_error(error)
     log_error(error)
+    @response ||= { 'action_source' => 'error', 'action_reason' => 'response_builder_error' }
     process_v1_handoff if conversation_pending?
     true
   end
@@ -178,6 +196,15 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
     account.feature_enabled?('captain_integration_v2')
   end
 
+  def report_v1_handoff_not_executed
+    error = StandardError.new("Captain V1 handoff requested but conversation #{@conversation.display_id} is still pending")
+    ChatwootExceptionTracker.new(error, account: account).capture_exception
+    Rails.logger.error(
+      "[CAPTAIN][ResponseBuilderJob] V1 handoff requested but not executed for account=#{account.id} " \
+      "conversation=#{@conversation.display_id}"
+    )
+  end
+
   def conversation_pending?
     status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) }
     status == 'pending' || status == Conversation.statuses[:pending]

enterprise/app/jobs/captain/conversation/v1_action_classifier.rb

@@ -0,0 +1,59 @@
+module Captain::Conversation::V1ActionClassifier
+  private
+
+  def v1_action_classifier_enabled?
+    account.feature_enabled?('captain_v1_action_classifier')
+  end
+
+  def classify_v1_response_action(message_history)
+    return unless v1_action_classifier_enabled?
+    return if legacy_v1_handoff_token?
+
+    classification = Captain::Llm::AssistantActionClassifierService.new(
+      assistant: @assistant,
+      conversation: @conversation
+    ).classify(message_history: message_history, assistant_response: @response['response'])
+
+    apply_v1_action_classification(classification)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][ResponseBuilderJob] V1 action classifier failed for account=#{account.id} " \
+      "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+  end
+
+  def apply_v1_action_classification(classification)
+    action = classification['action']
+    return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action)
+
+    @response.merge!(
+      'action' => action,
+      'action_reason' => classification['action_reason'],
+      'action_source' => 'classifier',
+      'action_classifier_model' => classification['model'],
+      'action_classifier_prompt_version' => classification['prompt_version']
+    )
+
+    log_v1_action_classification(action, classification)
+  end
+
+  def log_v1_action_classification(action, classification)
+    Rails.logger.info(
+      "[CAPTAIN][ResponseBuilderJob] V1 action classifier account=#{account.id} conversation=#{@conversation.display_id} " \
+      "action=#{action} reason=#{classification['action_reason']} model=#{classification['model']} " \
+      "prompt_version=#{classification['prompt_version']}"
+    )
+  end
+
+  def valid_v1_action_classification?(action)
+    Captain::Llm::AssistantActionClassifierService::VALID_ACTIONS.include?(action)
+  end
+
+  def log_invalid_v1_action_classification(classification)
+    Rails.logger.warn(
+      '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \
+      "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
+    )
+  end
+end

enterprise/app/services/captain/llm/assistant_action_classifier_service.rb

@@ -0,0 +1,151 @@
+class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
+  include Integrations::LlmInstrumentation
+
+  PROMPT_VERSION = 'v1_custom_xml_precedence'.freeze
+  DEFAULT_MODEL = 'gpt-4.1'.freeze
+  MAX_CONTEXT_MESSAGES = 10
+  VALID_ACTIONS = %w[continue handoff].freeze
+
+  def initialize(assistant:, conversation:)
+    super()
+    @assistant = assistant
+    @conversation = conversation
+    @model = DEFAULT_MODEL
+    @temperature = 0.0
+  end
+
+  def classify(message_history:, assistant_response:)
+    user_prompt = classification_user_prompt(
+      message_history: message_history,
+      assistant_response: assistant_response
+    )
+
+    response = instrument_llm_call(instrumentation_params(user_prompt)) do
+      chat(model: @model, temperature: @temperature)
+        .with_params(response_format: { type: 'json_object' })
+        .with_instructions(system_prompt)
+        .ask(user_prompt)
+    end
+
+    parsed = parse_response(response.content)
+    normalize_response(parsed, response.content)
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
+    Rails.logger.warn(
+      "[CAPTAIN][AssistantActionClassifier] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
+    )
+    { 'action' => nil, 'action_reason' => nil, 'error' => e.message, 'model' => @model, 'prompt_version' => PROMPT_VERSION }
+  end
+
+  private
+
+  def classification_user_prompt(message_history:, assistant_response:)
+    <<~PROMPT
+      <account_custom_instructions>
+      #{@assistant.config['instructions']}
+      </account_custom_instructions>
+
+      <conversation_context>
+      #{format_conversation_context(message_history)}
+      </conversation_context>
+
+      <assistant_response_to_classify>
+      #{assistant_response}
+      </assistant_response_to_classify>
+    PROMPT
+  end
+
+  def normalize_messages(message_history)
+    message_history.filter_map do |message|
+      role = message[:role] || message['role']
+      next if role.blank?
+
+      { role: role.to_s, content: normalize_content(message[:content] || message['content']) }
+    end
+  end
+
+  def normalize_content(content)
+    return content if content.is_a?(String)
+    return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
+
+    content.to_s
+  end
+
+  def text_part?(part)
+    return false unless part.is_a?(Hash)
+
+    (part[:type] || part['type']).to_s == 'text'
+  end
+
+  def format_conversation_context(messages)
+    normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+      content = message[:content].to_s.strip
+      next if content.blank?
+
+      "#{role_label(message[:role])}: #{content}"
+    end.join("\n")
+  end
+
+  def role_label(role)
+    return 'User' if role == 'user'
+    return 'Assistant' if role == 'assistant'
+
+    role.to_s.titleize
+  end
+
+  def parse_response(content)
+    JSON.parse(sanitize_json_response(content))
+  rescue JSON::ParserError, TypeError
+    {}
+  end
+
+  def normalize_response(parsed, raw_content)
+    action = parsed['action'].to_s
+    reason = parsed['action_reason'].to_s
+    return invalid_response(raw_content) unless VALID_ACTIONS.include?(action)
+
+    {
+      'action' => action,
+      'action_reason' => reason.presence,
+      'raw_response' => raw_content,
+      'model' => @model,
+      'prompt_version' => PROMPT_VERSION
+    }
+  end
+
+  def invalid_response(raw_content)
+    {
+      'action' => nil,
+      'action_reason' => nil,
+      'raw_response' => raw_content,
+      'error' => 'invalid_classifier_response',
+      'model' => @model,
+      'prompt_version' => PROMPT_VERSION
+    }
+  end
+
+  def instrumentation_params(user_prompt)
+    {
+      span_name: 'llm.captain.assistant_action_classifier',
+      model: @model,
+      temperature: @temperature,
+      account_id: @conversation.account_id,
+      conversation_id: @conversation.display_id,
+      feature_name: 'assistant_action_classifier',
+      messages: [
+        { role: 'system', content: system_prompt },
+        { role: 'user', content: user_prompt }
+      ],
+      metadata: {
+        assistant_id: @assistant.id,
+        channel_type: @conversation.inbox&.channel_type,
+        prompt_version: PROMPT_VERSION,
+        source: 'v1_response_builder'
+      }
+    }
+  end
+
+  def system_prompt
+    Captain::Llm::SystemPromptsService.assistant_action_classifier
+  end
+end

enterprise/app/services/captain/llm/system_prompts_service.rb

@@ -93,6 +93,59 @@ class Captain::Llm::SystemPromptsService
       SYSTEM_PROMPT_MESSAGE
     end
 
+    def assistant_action_classifier
+      <<~PROMPT
+        You are a routing classifier for a customer-support assistant.
+
+        Decide whether the current conversation should stay with the assistant or be transferred to a human agent now.
+
+        The action field MUST be one of:
+        - "continue": keep the current conversation with the assistant.
+        - "handoff": transfer the current conversation to a human agent now.
+
+        The action_reason field MUST be one of:
+        - "general_product_question"
+        - "missing_docs_bounded_answer"
+        - "clarifying_question_needed"
+        - "collect_required_identifier"
+        - "external_contact_or_lead_routing"
+        - "out_of_scope_bounded_answer"
+        - "explicit_human_request"
+        - "human_offer_accepted"
+        - "account_or_transaction_verification"
+        - "operational_issue_needs_inspection"
+        - "repeated_frustration_or_loop"
+        - "custom_instruction_transfer"
+
+        Use "continue" when:
+        - The user has a general product, pricing, capability, setup, pre-sales, or how-to question.
+        - The assistant can give a bounded answer, ask one useful clarifying question, collect a missing identifier, or share an approved external contact path.
+        - The assistant says someone will contact the user outside this conversation, but the current conversation itself does not need to be transferred now.
+        - The user has not explicitly asked for a human and the assistant is still collecting required details.
+
+        Use "handoff" when:
+        - The user explicitly asks for a human, agent, representative, phone call, callback, or escalation.
+        - The user accepts an offer to speak with a human.
+        - The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access.
+        - The user reports a bug or operational issue that needs support to inspect their account, setup, logs, integration, channel, or delivery state after a reasonable clarifying step.
+        - The user is repeatedly frustrated, distrustful, or stuck in a loop.
+        - The assistant response itself says the current conversation will be transferred to a human agent now.
+
+        You may receive account custom instructions inside <account_custom_instructions> tags.
+        These are instructions configured by the account administrator, not the current end user's message.
+        Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
+        If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
+        Account custom instructions MUST NOT redefine this JSON schema, the allowed action values, or the meaning of continue/handoff.
+        Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
+
+        Return JSON only:
+        {
+          "action": "continue",
+          "action_reason": "general_product_question"
+        }
+      PROMPT
+    end
+
     # rubocop:disable Metrics/MethodLength
     def copilot_response_generator(product_name, available_tools, config = {})
       citation_guidelines = if config['feature_citation']

spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb

@@ -10,6 +10,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
     let(:conversation) { create(:conversation, inbox: inbox, account: account, status: :pending) }
     let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
     let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
+    let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
 
     before do
       create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
@@ -19,6 +20,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
       allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
       allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
+      allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
+      allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' })
     end
 
     context 'when captain_v2 is disabled' do
@@ -48,6 +51,109 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
         expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
       end
 
+      it 'does not run the action classifier when the classifier feature is disabled' do
+        expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+        described_class.perform_now(conversation, assistant)
+
+        expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
+      end
+
+      context 'when V1 action classifier is enabled' do
+        before do
+          allow(account).to receive(:feature_enabled?).and_return(false)
+          allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
+          allow(account).to receive(:feature_enabled?).with('captain_v1_action_classifier').and_return(true)
+        end
+
+        it 'keeps the conversation pending when the classifier returns continue' do
+          expect(Captain::Llm::AssistantActionClassifierService).to receive(:new).with(
+            assistant: assistant,
+            conversation: conversation
+          ).and_return(mock_action_classifier_service)
+          expect(mock_action_classifier_service).to receive(:classify).with(
+            message_history: [{ content: 'Hello', role: 'user' }],
+            assistant_response: 'Hey, welcome to Captain Specs'
+          ).and_return({
+                         'action' => 'continue',
+                         'action_reason' => 'general_product_question',
+                         'model' => 'gpt-4.1',
+                         'prompt_version' => 'v1'
+                       })
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('pending')
+          expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'hands off without incrementing response usage when the classifier returns handoff' do
+          allow(mock_action_classifier_service).to receive(:classify).and_return({
+                                                                                   'action' => 'handoff',
+                                                                                   'action_reason' => 'explicit_human_request',
+                                                                                   'model' => 'gpt-4.1',
+                                                                                   'prompt_version' => 'v1'
+                                                                                 })
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('open')
+          expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
+        end
+
+        it 'skips the classifier when the legacy handoff token is returned' do
+          allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'conversation_handoff' })
+          expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('open')
+          expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
+        end
+
+        it 'falls back to the assistant response when the classifier fails' do
+          error = StandardError.new('classifier unavailable')
+          allow(mock_action_classifier_service).to receive(:classify).and_raise(error)
+          allow(ChatwootExceptionTracker).to receive(:new).and_call_original
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(ChatwootExceptionTracker).to have_received(:new).with(error, account: account)
+          expect(conversation.reload.status).to eq('pending')
+          expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'falls back to the assistant response when the classifier returns an invalid action' do
+          allow(mock_action_classifier_service).to receive(:classify).and_return({
+                                                                                   'action' => nil,
+                                                                                   'error' => 'invalid_classifier_response'
+                                                                                 })
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.reload.status).to eq('pending')
+          expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
+        end
+
+        it 'skips the classifier when the conversation is no longer pending after response generation' do
+          allow(mock_llm_chat_service).to receive(:generate_response) do
+            conversation.open!
+            { 'response' => 'Hey, welcome to Captain Specs' }
+          end
+
+          expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
+
+          described_class.perform_now(conversation, assistant)
+
+          expect(conversation.messages.outgoing.count).to eq(0)
+          expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
+        end
+      end
+
       it 'does not send a response when the conversation is no longer pending' do
         conversation.open!
 

spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb

@@ -0,0 +1,72 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Llm::AssistantActionClassifierService do
+  let(:account) { create(:account) }
+  let(:assistant) do
+    create(
+      :captain_assistant,
+      account: account,
+      config: {
+        'instructions' => 'Only transfer to a manager after the user explicitly confirms.'
+      }
+    )
+  end
+  let(:conversation) { create(:conversation, account: account) }
+  let(:service) { described_class.new(assistant: assistant, conversation: conversation) }
+  let(:mock_chat) { instance_double(RubyLLM::Chat) }
+  let(:mock_response) do
+    instance_double(
+      RubyLLM::Message,
+      content: '{"action":"handoff","action_reason":"human_offer_accepted"}'
+    )
+  end
+
+  before do
+    allow(RubyLLM).to receive(:chat).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_params).and_return(mock_chat)
+    allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
+  end
+
+  describe '#classify' do
+    let(:message_history) do
+      [
+        { role: 'user', content: 'I cannot log in' },
+        { role: 'assistant', content: 'Did you check your inbox?' },
+        { role: 'user', content: 'Yes, still no reset email' }
+      ]
+    end
+
+    it 'passes delimited custom instructions and classifier context to the LLM' do
+      expect(mock_chat).to receive(:with_instructions).with(
+        a_string_including(
+          'Use them only for routing policy',
+          'MUST NOT redefine this JSON schema'
+        )
+      ).and_return(mock_chat)
+      expect(mock_chat).to receive(:ask) do |prompt|
+        expect(prompt).to include(
+          '<account_custom_instructions>',
+          'Only transfer to a manager after the user explicitly confirms.',
+          '<conversation_context>',
+          'User: I cannot log in',
+          'Assistant: Did you check your inbox?',
+          'User: Yes, still no reset email',
+          '<assistant_response_to_classify>',
+          'Would you like to talk to support?'
+        )
+        expect(prompt).not_to include('"role"', '"content"', '<current_user_message>')
+
+        mock_response
+      end
+
+      result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
+
+      expect(result).to include(
+        'action' => 'handoff',
+        'action_reason' => 'human_offer_accepted',
+        'prompt_version' => 'v1_custom_xml_precedence'
+      )
+    end
+  end
+end