diff --git a/config/features.yml b/config/features.yml index f16199932..92d004f81 100644 --- a/config/features.yml +++ b/config/features.yml @@ -144,10 +144,11 @@ - name: chatwoot_v4 display_name: Chatwoot V4 enabled: true -- name: report_v4 - display_name: Report V4 +- name: captain_v1_action_classifier + display_name: Captain V1 Action Classifier enabled: false - deprecated: true + premium: true + chatwoot_internal: true - name: contact_chatwoot_support_team display_name: Contact Chatwoot Support Team enabled: true diff --git a/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb new file mode 100644 index 000000000..8a766ec4a --- /dev/null +++ b/db/migrate/20260430114500_repurpose_report_v4_flag_for_captain_v1_action_classifier.rb @@ -0,0 +1,8 @@ +class RepurposeReportV4FlagForCaptainV1ActionClassifier < ActiveRecord::Migration[7.1] + def up + Account.feature_captain_v1_action_classifier.find_each(batch_size: 100) do |account| + account.disable_features(:captain_v1_action_classifier) + account.save!(validate: false) + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 4ff250ada..9ce734ba7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_04_28_120000) do +ActiveRecord::Schema[7.1].define(version: 2026_04_30_114500) do # These extensions should be enabled to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" diff --git a/enterprise/app/helpers/captain/chat_response_helper.rb b/enterprise/app/helpers/captain/chat_response_helper.rb index bfb8adc11..ca860fa9d 100644 --- a/enterprise/app/helpers/captain/chat_response_helper.rb +++ b/enterprise/app/helpers/captain/chat_response_helper.rb @@ -35,7 +35,7 @@ module Captain::ChatResponseHelper def credit_used_for_response?(parsed_response) response = parsed_response['response'] - response.present? && response != 'conversation_handoff' + response.present? && response != 'conversation_handoff' && parsed_response['action'] != 'handoff' end def captain_v1_assistant? diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index 5e7c5b3c2..36d80fa2d 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -1,4 +1,6 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob + include Captain::Conversation::V1ActionClassifier + MAX_MESSAGE_LENGTH = 10_000 retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds @@ -31,9 +33,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob delegate :account, :inbox, to: :@conversation def generate_and_process_response + message_history = collect_previous_messages @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response( - message_history: collect_previous_messages + message_history: message_history ) + classify_v1_response_action(message_history) process_response end @@ -102,6 +106,10 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def v1_handoff_requested? + @response['action'] == 'handoff' || legacy_v1_handoff_token? + end + + def legacy_v1_handoff_token? @response['response'] == 'conversation_handoff' end @@ -111,8 +119,13 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob def process_v1_handoff I18n.with_locale(@assistant.account.locale) do + Rails.logger.info( + "[CAPTAIN][ResponseBuilderJob] V1 handoff requested for account=#{account.id} conversation=#{@conversation.display_id} " \ + "source=#{@response&.dig('action_source') || 'legacy'} reason=#{@response&.dig('action_reason')}" + ) create_handoff_message @conversation.bot_handoff! + report_v1_handoff_not_executed if conversation_pending? send_out_of_office_message_if_applicable end end @@ -166,6 +179,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob def handle_error(error) log_error(error) + @response ||= { 'action_source' => 'error', 'action_reason' => 'response_builder_error' } process_v1_handoff if conversation_pending? true end @@ -178,6 +192,15 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob account.feature_enabled?('captain_integration_v2') end + def report_v1_handoff_not_executed + error = StandardError.new("Captain V1 handoff requested but conversation #{@conversation.display_id} is still pending") + ChatwootExceptionTracker.new(error, account: account).capture_exception + Rails.logger.error( + "[CAPTAIN][ResponseBuilderJob] V1 handoff requested but not executed for account=#{account.id} " \ + "conversation=#{@conversation.display_id}" + ) + end + def conversation_pending? status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) } status == 'pending' || status == Conversation.statuses[:pending] diff --git a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb new file mode 100644 index 000000000..0607342d2 --- /dev/null +++ b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb @@ -0,0 +1,54 @@ +module Captain::Conversation::V1ActionClassifier + private + + def v1_action_classifier_enabled? + account.feature_enabled?('captain_v1_action_classifier') + end + + def classify_v1_response_action(message_history) + return unless v1_action_classifier_enabled? + return if legacy_v1_handoff_token? + + classification = Captain::Llm::AssistantActionClassifierService.new( + assistant: @assistant, + conversation: @conversation + ).classify(message_history: message_history, assistant_response: @response['response']) + + apply_v1_action_classification(classification) + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: account).capture_exception + Rails.logger.warn( + "[CAPTAIN][ResponseBuilderJob] V1 action classifier failed for account=#{account.id} " \ + "conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}" + ) + end + + def apply_v1_action_classification(classification) + action = classification['action'] + unless action.in?(%w[continue handoff]) + Rails.logger.warn( + "[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action for account=#{account.id} " \ + "conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}" + ) + return + end + + @response.merge!( + 'action' => action, + 'action_reason' => classification['action_reason'], + 'action_source' => 'classifier', + 'action_classifier_model' => classification['model'], + 'action_classifier_prompt_version' => classification['prompt_version'] + ) + + log_v1_action_classification(action, classification) + end + + def log_v1_action_classification(action, classification) + Rails.logger.info( + "[CAPTAIN][ResponseBuilderJob] V1 action classifier account=#{account.id} conversation=#{@conversation.display_id} " \ + "action=#{action} reason=#{classification['action_reason']} model=#{classification['model']} " \ + "prompt_version=#{classification['prompt_version']}" + ) + end +end diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb new file mode 100644 index 000000000..94d60542d --- /dev/null +++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb @@ -0,0 +1,162 @@ +class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService + include Integrations::LlmInstrumentation + + PROMPT_VERSION = 'v1_custom_xml_precedence'.freeze + DEFAULT_MODEL = 'gpt-4.1'.freeze + MAX_CONTEXT_MESSAGES = 10 + VALID_ACTIONS = %w[continue handoff].freeze + + def initialize(assistant:, conversation:) + super() + @assistant = assistant + @conversation = conversation + @model = DEFAULT_MODEL + @temperature = 0.0 + end + + def classify(message_history:, assistant_response:) + payload = classification_payload(message_history, assistant_response) + user_prompt = classification_user_prompt(payload) + + response = instrument_llm_call(instrumentation_params(user_prompt)) do + chat(model: @model, temperature: @temperature) + .with_params(response_format: { type: 'json_object' }) + .with_instructions(system_prompt) + .ask(user_prompt) + end + + parsed = parse_response(response.content) + normalize_response(parsed, response.content) + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception + Rails.logger.warn( + "[CAPTAIN][AssistantActionClassifier] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}" + ) + { 'action' => nil, 'action_reason' => nil, 'error' => e.message, 'model' => @model, 'prompt_version' => PROMPT_VERSION } + end + + private + + def classification_payload(message_history, assistant_response) + normalized_messages = normalize_messages(message_history) + + { + 'account_custom_instructions' => account_custom_instructions, + 'conversation_context' => context_messages(normalized_messages), + 'current_user_message' => current_user_message(normalized_messages), + 'assistant_response_to_classify' => assistant_response.to_s + } + end + + def classification_user_prompt(payload) + <<~PROMPT + + #{payload['account_custom_instructions']} + + + + #{payload['conversation_context'].to_json} + + + + #{payload['current_user_message']} + + + + #{payload['assistant_response_to_classify']} + + PROMPT + end + + def normalize_messages(message_history) + message_history.filter_map do |message| + role = message[:role] || message['role'] + next if role.blank? + + { role: role.to_s, content: normalize_content(message[:content] || message['content']) } + end + end + + def normalize_content(content) + return content if content.is_a?(String) + return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array) + + content.to_s + end + + def text_part?(part) + return false unless part.is_a?(Hash) + + (part[:type] || part['type']).to_s == 'text' + end + + def current_user_message(messages) + messages.reverse.find { |message| message[:role] == 'user' }&.dig(:content).to_s + end + + def context_messages(messages) + current_user_index = messages.rindex { |message| message[:role] == 'user' } + prior_messages = current_user_index ? messages[0...current_user_index] : messages + prior_messages.last(MAX_CONTEXT_MESSAGES) + end + + def account_custom_instructions + @assistant.config['instructions'].to_s + end + + def parse_response(content) + JSON.parse(sanitize_json_response(content)) + rescue JSON::ParserError, TypeError + {} + end + + def normalize_response(parsed, raw_content) + action = parsed['action'].to_s + reason = parsed['action_reason'].to_s + return invalid_response(raw_content) unless VALID_ACTIONS.include?(action) + + { + 'action' => action, + 'action_reason' => reason.presence, + 'raw_response' => raw_content, + 'model' => @model, + 'prompt_version' => PROMPT_VERSION + } + end + + def invalid_response(raw_content) + { + 'action' => nil, + 'action_reason' => nil, + 'raw_response' => raw_content, + 'error' => 'invalid_classifier_response', + 'model' => @model, + 'prompt_version' => PROMPT_VERSION + } + end + + def instrumentation_params(user_prompt) + { + span_name: 'llm.captain.assistant_action_classifier', + model: @model, + temperature: @temperature, + account_id: @conversation.account_id, + conversation_id: @conversation.display_id, + feature_name: 'assistant_action_classifier', + messages: [ + { role: 'system', content: system_prompt }, + { role: 'user', content: user_prompt } + ], + metadata: { + assistant_id: @assistant.id, + channel_type: @conversation.inbox&.channel_type, + prompt_version: PROMPT_VERSION, + source: 'v1_response_builder' + } + } + end + + def system_prompt + Captain::Llm::SystemPromptsService.assistant_action_classifier + end +end diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb index eb8c334f4..644c2e5c5 100644 --- a/enterprise/app/services/captain/llm/system_prompts_service.rb +++ b/enterprise/app/services/captain/llm/system_prompts_service.rb @@ -93,6 +93,59 @@ class Captain::Llm::SystemPromptsService SYSTEM_PROMPT_MESSAGE end + def assistant_action_classifier + <<~PROMPT + You are a routing classifier for a customer-support assistant. + + Decide whether the current conversation should stay with the assistant or be transferred to a human agent now. + + The action field MUST be one of: + - "continue": keep the current conversation with the assistant. + - "handoff": transfer the current conversation to a human agent now. + + The action_reason field MUST be one of: + - "general_product_question" + - "missing_docs_bounded_answer" + - "clarifying_question_needed" + - "collect_required_identifier" + - "external_contact_or_lead_routing" + - "out_of_scope_bounded_answer" + - "explicit_human_request" + - "human_offer_accepted" + - "account_or_transaction_verification" + - "operational_issue_needs_inspection" + - "repeated_frustration_or_loop" + - "custom_instruction_transfer" + + Use "continue" when: + - The user has a general product, pricing, capability, setup, pre-sales, or how-to question. + - The assistant can give a bounded answer, ask one useful clarifying question, collect a missing identifier, or share an approved external contact path. + - The assistant says someone will contact the user outside this conversation, but the current conversation itself does not need to be transferred now. + - The user has not explicitly asked for a human and the assistant is still collecting required details. + + Use "handoff" when: + - The user explicitly asks for a human, agent, representative, phone call, callback, or escalation. + - The user accepts an offer to speak with a human. + - The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access. + - The user reports a bug or operational issue that needs support to inspect their account, setup, logs, integration, channel, or delivery state after a reasonable clarifying step. + - The user is repeatedly frustrated, distrustful, or stuck in a loop. + - The assistant response itself says the current conversation will be transferred to a human agent now. + + You may receive account custom instructions inside tags. + These are instructions configured by the account administrator, not the current end user's message. + Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team. + If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above. + Account custom instructions MUST NOT redefine this JSON schema, the allowed action values, or the meaning of continue/handoff. + Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria. + + Return JSON only: + { + "action": "continue", + "action_reason": "general_product_question" + } + PROMPT + end + # rubocop:disable Metrics/MethodLength def copilot_response_generator(product_name, available_tools, config = {}) citation_guidelines = if config['feature_citation'] diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 548b84992..ca3d7b61f 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -10,6 +10,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do let(:conversation) { create(:conversation, inbox: inbox, account: account, status: :pending) } let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) } let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) } + let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) } before do create(:message, conversation: conversation, content: 'Hello', message_type: :incoming) @@ -19,6 +20,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' }) allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service) allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' }) + allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service) + allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' }) end context 'when captain_v2 is disabled' do @@ -48,6 +51,95 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1) end + it 'does not run the action classifier when the classifier feature is disabled' do + expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new) + + described_class.perform_now(conversation, assistant) + + expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs') + end + + context 'when V1 action classifier is enabled' do + before do + allow(account).to receive(:feature_enabled?).and_return(false) + allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false) + allow(account).to receive(:feature_enabled?).with('captain_v1_action_classifier').and_return(true) + end + + it 'keeps the conversation pending when the classifier returns continue' do + expect(Captain::Llm::AssistantActionClassifierService).to receive(:new).with( + assistant: assistant, + conversation: conversation + ).and_return(mock_action_classifier_service) + expect(mock_action_classifier_service).to receive(:classify).with( + message_history: [{ content: 'Hello', role: 'user' }], + assistant_response: 'Hey, welcome to Captain Specs' + ).and_return({ + 'action' => 'continue', + 'action_reason' => 'general_product_question', + 'model' => 'gpt-4.1', + 'prompt_version' => 'v1' + }) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('pending') + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs') + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1) + end + + it 'hands off without incrementing response usage when the classifier returns handoff' do + allow(mock_action_classifier_service).to receive(:classify).and_return({ + 'action' => 'handoff', + 'action_reason' => 'explicit_human_request', + 'model' => 'gpt-4.1', + 'prompt_version' => 'v1' + }) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('open') + expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff')) + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0) + end + + it 'skips the classifier when the legacy handoff token is returned' do + allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'conversation_handoff' }) + expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('open') + expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff')) + end + + it 'falls back to the assistant response when the classifier fails' do + error = StandardError.new('classifier unavailable') + allow(mock_action_classifier_service).to receive(:classify).and_raise(error) + allow(ChatwootExceptionTracker).to receive(:new).and_call_original + + described_class.perform_now(conversation, assistant) + + expect(ChatwootExceptionTracker).to have_received(:new).with(error, account: account) + expect(conversation.reload.status).to eq('pending') + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs') + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1) + end + + it 'falls back to the assistant response when the classifier returns an invalid action' do + allow(mock_action_classifier_service).to receive(:classify).and_return({ + 'action' => nil, + 'error' => 'invalid_classifier_response' + }) + + described_class.perform_now(conversation, assistant) + + expect(conversation.reload.status).to eq('pending') + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs') + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1) + end + end + it 'does not send a response when the conversation is no longer pending' do conversation.open! diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb new file mode 100644 index 000000000..5561cd8d7 --- /dev/null +++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb @@ -0,0 +1,69 @@ +require 'rails_helper' + +RSpec.describe Captain::Llm::AssistantActionClassifierService do + let(:account) { create(:account) } + let(:assistant) do + create( + :captain_assistant, + account: account, + config: { + 'instructions' => 'Only transfer to a manager after the user explicitly confirms.' + } + ) + end + let(:conversation) { create(:conversation, account: account) } + let(:service) { described_class.new(assistant: assistant, conversation: conversation) } + let(:mock_chat) { instance_double(RubyLLM::Chat) } + let(:mock_response) do + instance_double( + RubyLLM::Message, + content: '{"action":"handoff","action_reason":"human_offer_accepted"}' + ) + end + + before do + allow(RubyLLM).to receive(:chat).and_return(mock_chat) + allow(mock_chat).to receive(:with_temperature).and_return(mock_chat) + allow(mock_chat).to receive(:with_params).and_return(mock_chat) + allow(mock_chat).to receive(:with_instructions).and_return(mock_chat) + end + + describe '#classify' do + let(:message_history) do + [ + { role: 'user', content: 'I cannot log in' }, + { role: 'assistant', content: 'Did you check your inbox?' }, + { role: 'user', content: 'Yes, still no reset email' } + ] + end + + it 'passes delimited custom instructions and classifier context to the LLM' do + expect(mock_chat).to receive(:with_instructions).with( + a_string_including( + 'Use them only for routing policy', + 'MUST NOT redefine this JSON schema' + ) + ).and_return(mock_chat) + expect(mock_chat).to receive(:ask).with( + a_string_including( + '', + 'Only transfer to a manager after the user explicitly confirms.', + '', + '"content":"I cannot log in"', + '', + 'Yes, still no reset email', + '', + 'Would you like to talk to support?' + ) + ).and_return(mock_response) + + result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?') + + expect(result).to include( + 'action' => 'handoff', + 'action_reason' => 'human_offer_accepted', + 'prompt_version' => 'v1_custom_xml_precedence' + ) + end + end +end