fix(captain): add v1 handoff classifier [AI-137] (#14337)
# Pull Request Template ## Description Captain (v1) makes false promises by saying it will handoff but doesn't. This happens due to an exact string match comparison and the prompt gives the model a lot of responsibilities: - identity - what to respond - obey custom instructions - decide on tool calls This PR decouples responsibility, the core prompt responds, and an additional llm call evaluates if handoff was needed or not after that message. ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration. Locally ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules
This commit is contained in:
Aakash Bakhle
+4
-3
@@ -144,10 +144,11 @@
|
||||
- name: chatwoot_v4
|
||||
display_name: Chatwoot V4
|
||||
enabled: true
|
||||
- name: report_v4
|
||||
display_name: Report V4
|
||||
- name: captain_v1_action_classifier
|
||||
display_name: Captain V1 Action Classifier
|
||||
enabled: false
|
||||
deprecated: true
|
||||
premium: true
|
||||
chatwoot_internal: true
|
||||
- name: contact_chatwoot_support_team
|
||||
display_name: Contact Chatwoot Support Team
|
||||
enabled: true
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
class RepurposeReportV4FlagForCaptainV1ActionClassifier < ActiveRecord::Migration[7.1]
|
||||
def up
|
||||
Account.feature_captain_v1_action_classifier.find_each(batch_size: 100) do |account|
|
||||
account.disable_features(:captain_v1_action_classifier)
|
||||
account.save!(validate: false)
|
||||
end
|
||||
|
||||
config = InstallationConfig.find_by(name: 'ACCOUNT_LEVEL_FEATURE_DEFAULTS')
|
||||
return if config&.value.blank?
|
||||
|
||||
config.value = config.value.reject { |feature| feature['name'] == 'report_v4' }
|
||||
config.save!
|
||||
GlobalConfig.clear_cache
|
||||
end
|
||||
end
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[7.1].define(version: 2026_04_28_120000) do
|
||||
ActiveRecord::Schema[7.1].define(version: 2026_04_30_114500) do
|
||||
# These extensions should be enabled to support this database
|
||||
enable_extension "pg_stat_statements"
|
||||
enable_extension "pg_trgm"
|
||||
|
||||
@@ -35,7 +35,10 @@ module Captain::ChatResponseHelper
|
||||
|
||||
def credit_used_for_response?(parsed_response)
|
||||
response = parsed_response['response']
|
||||
response.present? && response != 'conversation_handoff'
|
||||
|
||||
# The classifier can still decide to hand off after this trace is written.
|
||||
# Actual response usage is charged later in ResponseBuilderJob, so billing stays correct.
|
||||
response.present? && response != 'conversation_handoff' && parsed_response['action'] != 'handoff'
|
||||
end
|
||||
|
||||
def captain_v1_assistant?
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
class Captain::Conversation::ResponseBuilderJob < ApplicationJob
|
||||
include Captain::Conversation::V1ActionClassifier
|
||||
|
||||
MAX_MESSAGE_LENGTH = 10_000
|
||||
retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
|
||||
retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds
|
||||
@@ -31,9 +33,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
|
||||
delegate :account, :inbox, to: :@conversation
|
||||
|
||||
def generate_and_process_response
|
||||
message_history = collect_previous_messages
|
||||
@response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation: @conversation).generate_response(
|
||||
message_history: collect_previous_messages
|
||||
message_history: message_history
|
||||
)
|
||||
classify_v1_response_action(message_history) if conversation_pending?
|
||||
process_response
|
||||
end
|
||||
|
||||
@@ -102,6 +106,14 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
|
||||
end
|
||||
|
||||
def v1_handoff_requested?
|
||||
legacy_v1_handoff_token? || classifier_v1_handoff_requested?
|
||||
end
|
||||
|
||||
def classifier_v1_handoff_requested?
|
||||
@response['action'] == 'handoff'
|
||||
end
|
||||
|
||||
def legacy_v1_handoff_token?
|
||||
@response['response'] == 'conversation_handoff'
|
||||
end
|
||||
|
||||
@@ -111,8 +123,13 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
|
||||
|
||||
def process_v1_handoff
|
||||
I18n.with_locale(@assistant.account.locale) do
|
||||
Rails.logger.info(
|
||||
"[CAPTAIN][ResponseBuilderJob] V1 handoff requested for account=#{account.id} conversation=#{@conversation.display_id} " \
|
||||
"source=#{@response&.dig('action_source') || 'legacy'} reason=#{@response&.dig('action_reason')}"
|
||||
)
|
||||
create_handoff_message
|
||||
@conversation.bot_handoff!
|
||||
report_v1_handoff_not_executed if conversation_pending?
|
||||
send_out_of_office_message_if_applicable
|
||||
end
|
||||
end
|
||||
@@ -166,6 +183,9 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
|
||||
|
||||
def handle_error(error)
|
||||
log_error(error)
|
||||
@response ||= {}
|
||||
@response['action_source'] ||= 'error'
|
||||
@response['action_reason'] ||= error_action_reason(error)
|
||||
process_v1_handoff if conversation_pending?
|
||||
true
|
||||
end
|
||||
@@ -174,10 +194,23 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
|
||||
ChatwootExceptionTracker.new(error, account: account).capture_exception
|
||||
end
|
||||
|
||||
def error_action_reason(error)
|
||||
error.class.name.underscore.tr('/', '_')
|
||||
end
|
||||
|
||||
def captain_v2_enabled?
|
||||
account.feature_enabled?('captain_integration_v2')
|
||||
end
|
||||
|
||||
def report_v1_handoff_not_executed
|
||||
error = StandardError.new("Captain V1 handoff requested but conversation #{@conversation.display_id} is still pending")
|
||||
ChatwootExceptionTracker.new(error, account: account).capture_exception
|
||||
Rails.logger.error(
|
||||
"[CAPTAIN][ResponseBuilderJob] V1 handoff requested but not executed for account=#{account.id} " \
|
||||
"conversation=#{@conversation.display_id}"
|
||||
)
|
||||
end
|
||||
|
||||
def conversation_pending?
|
||||
status = Conversation.uncached { Conversation.where(id: @conversation.id).pick(:status) }
|
||||
status == 'pending' || status == Conversation.statuses[:pending]
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
module Captain::Conversation::V1ActionClassifier
|
||||
private
|
||||
|
||||
def v1_action_classifier_enabled?
|
||||
account.feature_enabled?('captain_v1_action_classifier')
|
||||
end
|
||||
|
||||
def classify_v1_response_action(message_history)
|
||||
return unless v1_action_classifier_enabled?
|
||||
return if legacy_v1_handoff_token?
|
||||
|
||||
classification = Captain::Llm::AssistantActionClassifierService.new(
|
||||
assistant: @assistant,
|
||||
conversation: @conversation
|
||||
).classify(message_history: message_history, assistant_response: @response['response'])
|
||||
|
||||
apply_v1_action_classification(classification)
|
||||
rescue StandardError => e
|
||||
ChatwootExceptionTracker.new(e, account: account).capture_exception
|
||||
Rails.logger.warn(
|
||||
"[CAPTAIN][ResponseBuilderJob] V1 action classifier failed for account=#{account.id} " \
|
||||
"conversation=#{@conversation.display_id}: #{e.class.name}: #{e.message}"
|
||||
)
|
||||
end
|
||||
|
||||
def apply_v1_action_classification(classification)
|
||||
action = classification['action']
|
||||
return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action)
|
||||
|
||||
@response.merge!(
|
||||
'action' => action,
|
||||
'action_reason' => classification['action_reason'],
|
||||
'action_source' => 'classifier',
|
||||
'action_classifier_model' => classification['model']
|
||||
)
|
||||
|
||||
log_v1_action_classification(action, classification)
|
||||
end
|
||||
|
||||
def log_v1_action_classification(action, classification)
|
||||
Rails.logger.info(
|
||||
"[CAPTAIN][ResponseBuilderJob] V1 action classifier account=#{account.id} conversation=#{@conversation.display_id} " \
|
||||
"action=#{action} reason=#{classification['action_reason']} model=#{classification['model']}"
|
||||
)
|
||||
end
|
||||
|
||||
def valid_v1_action_classification?(action)
|
||||
Captain::AssistantActionSchema::ACTIONS.include?(action)
|
||||
end
|
||||
|
||||
def log_invalid_v1_action_classification(classification)
|
||||
Rails.logger.warn(
|
||||
'[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \
|
||||
"for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
|
||||
)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,148 @@
|
||||
class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
include Integrations::LlmInstrumentation
|
||||
|
||||
MAX_CONTEXT_MESSAGES = 10
|
||||
|
||||
def initialize(assistant:, conversation:)
|
||||
super()
|
||||
@assistant = assistant
|
||||
@conversation = conversation
|
||||
@temperature = 0.0
|
||||
end
|
||||
|
||||
def classify(message_history:, assistant_response:)
|
||||
user_prompt = classification_user_prompt(
|
||||
message_history: message_history,
|
||||
assistant_response: assistant_response
|
||||
)
|
||||
|
||||
response = instrument_llm_call(instrumentation_params(user_prompt)) do
|
||||
chat(model: @model, temperature: @temperature)
|
||||
.with_schema(Captain::AssistantActionSchema)
|
||||
.with_instructions(system_prompt)
|
||||
.ask(user_prompt)
|
||||
end
|
||||
|
||||
parsed = parse_response(response.content)
|
||||
normalize_response(parsed, response.content)
|
||||
rescue StandardError => e
|
||||
ChatwootExceptionTracker.new(e, account: @conversation.account).capture_exception
|
||||
Rails.logger.warn(
|
||||
"[CAPTAIN][AssistantActionClassifier] Failed for conversation #{@conversation.display_id}: #{e.class.name}: #{e.message}"
|
||||
)
|
||||
{ 'action' => nil, 'action_reason' => nil, 'error' => e.message, 'model' => @model }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def classification_user_prompt(message_history:, assistant_response:)
|
||||
<<~PROMPT
|
||||
<account_custom_instructions>
|
||||
#{@assistant.config['instructions']}
|
||||
</account_custom_instructions>
|
||||
|
||||
<conversation_context>
|
||||
#{format_conversation_context(message_history)}
|
||||
</conversation_context>
|
||||
|
||||
<assistant_response_to_classify>
|
||||
#{assistant_response}
|
||||
</assistant_response_to_classify>
|
||||
PROMPT
|
||||
end
|
||||
|
||||
def normalize_messages(message_history)
|
||||
message_history.filter_map do |message|
|
||||
role = message[:role] || message['role']
|
||||
next if role.blank?
|
||||
|
||||
{ role: role.to_s, content: normalize_content(message[:content] || message['content']) }
|
||||
end
|
||||
end
|
||||
|
||||
def normalize_content(content)
|
||||
return content if content.is_a?(String)
|
||||
return content.filter_map { |part| part[:text] || part['text'] if text_part?(part) }.join("\n") if content.is_a?(Array)
|
||||
|
||||
content.to_s
|
||||
end
|
||||
|
||||
def text_part?(part)
|
||||
return false unless part.is_a?(Hash)
|
||||
|
||||
(part[:type] || part['type']).to_s == 'text'
|
||||
end
|
||||
|
||||
def format_conversation_context(messages)
|
||||
normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
|
||||
content = message[:content].to_s.strip
|
||||
next if content.blank?
|
||||
|
||||
"#{role_label(message[:role])}: #{content}"
|
||||
end.join("\n")
|
||||
end
|
||||
|
||||
def role_label(role)
|
||||
return 'User' if role == 'user'
|
||||
return 'Assistant' if role == 'assistant'
|
||||
|
||||
role.to_s.titleize
|
||||
end
|
||||
|
||||
def parse_response(content)
|
||||
return content if content.is_a?(Hash)
|
||||
|
||||
JSON.parse(sanitize_json_response(content))
|
||||
rescue JSON::ParserError, TypeError
|
||||
{}
|
||||
end
|
||||
|
||||
def normalize_response(parsed, raw_content)
|
||||
action = parsed['action'].to_s
|
||||
reason = parsed['action_reason'].to_s
|
||||
return invalid_response(raw_content) unless Captain::AssistantActionSchema::ACTIONS.include?(action)
|
||||
|
||||
{
|
||||
'action' => action,
|
||||
'action_reason' => reason.presence,
|
||||
'raw_response' => raw_content,
|
||||
'model' => @model
|
||||
}
|
||||
end
|
||||
|
||||
def invalid_response(raw_content)
|
||||
{
|
||||
'action' => nil,
|
||||
'action_reason' => nil,
|
||||
'raw_response' => raw_content,
|
||||
'error' => 'invalid_classifier_response',
|
||||
'model' => @model
|
||||
}
|
||||
end
|
||||
|
||||
def instrumentation_params(user_prompt)
|
||||
{
|
||||
span_name: 'llm.captain.assistant_action_classifier',
|
||||
model: @model,
|
||||
temperature: @temperature,
|
||||
account_id: @conversation.account_id,
|
||||
conversation_id: @conversation.display_id,
|
||||
feature_name: 'assistant_action_classifier',
|
||||
messages: [
|
||||
{ role: 'system', content: system_prompt },
|
||||
{ role: 'user', content: user_prompt }
|
||||
],
|
||||
metadata: {
|
||||
assistant_id: @assistant.id,
|
||||
channel_type: @conversation.inbox&.channel_type,
|
||||
source: 'v1_response_builder'
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def system_prompt
|
||||
Captain::Llm::SystemPromptsService.assistant_action_classifier(
|
||||
has_custom_instructions: @assistant.config['instructions'].present?
|
||||
)
|
||||
end
|
||||
end
|
||||
@@ -93,6 +93,50 @@ class Captain::Llm::SystemPromptsService
|
||||
SYSTEM_PROMPT_MESSAGE
|
||||
end
|
||||
|
||||
def assistant_action_classifier(has_custom_instructions: false)
|
||||
<<~PROMPT
|
||||
You are a routing classifier for a customer-support assistant.
|
||||
|
||||
Decide whether the current conversation should stay with the assistant or be transferred to a human agent now.
|
||||
|
||||
The action field MUST be one of:
|
||||
- "continue": keep the current conversation with the assistant.
|
||||
- "handoff": transfer the current conversation to a human agent now.
|
||||
|
||||
The action_reason field MUST be one of:
|
||||
- "general_product_question"
|
||||
- "missing_docs_bounded_answer"
|
||||
- "clarifying_question_needed"
|
||||
- "collect_required_identifier"
|
||||
- "external_contact_or_lead_routing"
|
||||
- "out_of_scope_bounded_answer"
|
||||
- "explicit_human_request"
|
||||
- "human_offer_accepted"
|
||||
- "account_or_transaction_verification"
|
||||
- "operational_issue_needs_inspection"
|
||||
- "repeated_frustration_or_loop"
|
||||
- "custom_instruction_transfer"
|
||||
|
||||
Use "continue" when:
|
||||
- The user has a general product, pricing, capability, setup, pre-sales, or how-to question.
|
||||
- The assistant can give a bounded answer, ask one useful clarifying question, collect a missing identifier, or share an approved external contact path.
|
||||
- The assistant says someone will contact the user outside this conversation, but the current conversation itself does not need to be transferred now.
|
||||
- The user has not explicitly asked for a human and the assistant is still collecting required details.
|
||||
|
||||
Use "handoff" when:
|
||||
- The user explicitly asks for a human, agent, representative, phone call, callback, or escalation.
|
||||
- The user accepts an offer to speak with a human.
|
||||
- The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access.
|
||||
- The user reports the same unresolved bug or operational issue after trying the assistant's suggested step, repeating the action, checking again, or otherwise making more than one reasonable attempt.
|
||||
- The user is repeatedly frustrated, distrustful, or stuck in a loop.
|
||||
- The assistant response itself says the current conversation will be transferred to a human agent now.
|
||||
|
||||
#{assistant_action_classifier_custom_instructions_policy if has_custom_instructions}
|
||||
|
||||
Return only the structured fields requested by the response schema.
|
||||
PROMPT
|
||||
end
|
||||
|
||||
# rubocop:disable Metrics/MethodLength
|
||||
def copilot_response_generator(product_name, available_tools, config = {})
|
||||
citation_guidelines = if config['feature_citation']
|
||||
@@ -208,7 +252,9 @@ class Captain::Llm::SystemPromptsService
|
||||
- Do not share anything outside of the context provided.
|
||||
- Add the reasoning why you arrived at the answer
|
||||
- Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format.
|
||||
#{config['instructions'] || ''}
|
||||
|
||||
#{build_custom_instructions_section(config['instructions'])}
|
||||
|
||||
```json
|
||||
{
|
||||
reasoning: '',
|
||||
@@ -322,6 +368,17 @@ class Captain::Llm::SystemPromptsService
|
||||
TOOLS
|
||||
end
|
||||
|
||||
def assistant_action_classifier_custom_instructions_policy
|
||||
<<~POLICY
|
||||
Account custom instructions are provided inside <account_custom_instructions> tags.
|
||||
These are instructions configured by the account administrator, not the current end user's message.
|
||||
Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
|
||||
If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
|
||||
Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
|
||||
Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
|
||||
POLICY
|
||||
end
|
||||
|
||||
def build_contact_context(contact)
|
||||
return '' if contact.nil?
|
||||
|
||||
@@ -331,6 +388,18 @@ class Captain::Llm::SystemPromptsService
|
||||
"[Contact Information]\n#{lines.join("\n")}\n\n"
|
||||
end
|
||||
|
||||
def build_custom_instructions_section(instructions)
|
||||
return '' if instructions.blank?
|
||||
|
||||
<<~CUSTOM_INSTRUCTIONS
|
||||
[Account Custom Instructions]
|
||||
These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context.
|
||||
<account_custom_instructions>
|
||||
#{instructions}
|
||||
</account_custom_instructions>
|
||||
CUSTOM_INSTRUCTIONS
|
||||
end
|
||||
|
||||
def contact_basic_lines(contact)
|
||||
[
|
||||
(["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?),
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
class Captain::AssistantActionSchema < RubyLLM::Schema
|
||||
ACTIONS = %w[continue handoff].freeze
|
||||
REASONS = %w[
|
||||
general_product_question
|
||||
missing_docs_bounded_answer
|
||||
clarifying_question_needed
|
||||
collect_required_identifier
|
||||
external_contact_or_lead_routing
|
||||
out_of_scope_bounded_answer
|
||||
explicit_human_request
|
||||
human_offer_accepted
|
||||
account_or_transaction_verification
|
||||
operational_issue_needs_inspection
|
||||
repeated_frustration_or_loop
|
||||
custom_instruction_transfer
|
||||
].freeze
|
||||
|
||||
string :action, enum: ACTIONS, description: 'Whether to keep the conversation with the assistant or transfer it to a human agent'
|
||||
string :action_reason, enum: REASONS, description: 'The reason for the selected routing action'
|
||||
end
|
||||
@@ -10,6 +10,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
let(:conversation) { create(:conversation, inbox: inbox, account: account, status: :pending) }
|
||||
let(:mock_llm_chat_service) { instance_double(Captain::Llm::AssistantChatService) }
|
||||
let(:mock_agent_runner_service) { instance_double(Captain::Assistant::AgentRunnerService) }
|
||||
let(:mock_action_classifier_service) { instance_double(Captain::Llm::AssistantActionClassifierService) }
|
||||
|
||||
before do
|
||||
create(:message, conversation: conversation, content: 'Hello', message_type: :incoming)
|
||||
@@ -19,6 +20,8 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' })
|
||||
allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service)
|
||||
allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' })
|
||||
allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service)
|
||||
allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' })
|
||||
end
|
||||
|
||||
context 'when captain_v2 is disabled' do
|
||||
@@ -48,6 +51,107 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
|
||||
end
|
||||
|
||||
it 'does not run the action classifier when the classifier feature is disabled' do
|
||||
expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(conversation.messages.last.content).to eq('Hey, welcome to Captain Specs')
|
||||
end
|
||||
|
||||
context 'when V1 action classifier is enabled' do
|
||||
before do
|
||||
allow(account).to receive(:feature_enabled?).and_return(false)
|
||||
allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(false)
|
||||
allow(account).to receive(:feature_enabled?).with('captain_v1_action_classifier').and_return(true)
|
||||
end
|
||||
|
||||
it 'keeps the conversation pending when the classifier returns continue' do
|
||||
expect(Captain::Llm::AssistantActionClassifierService).to receive(:new).with(
|
||||
assistant: assistant,
|
||||
conversation: conversation
|
||||
).and_return(mock_action_classifier_service)
|
||||
expect(mock_action_classifier_service).to receive(:classify).with(
|
||||
message_history: [{ content: 'Hello', role: 'user' }],
|
||||
assistant_response: 'Hey, welcome to Captain Specs'
|
||||
).and_return({
|
||||
'action' => 'continue',
|
||||
'action_reason' => 'general_product_question',
|
||||
'model' => 'gpt-4.1'
|
||||
})
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(conversation.reload.status).to eq('pending')
|
||||
expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
|
||||
expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
|
||||
end
|
||||
|
||||
it 'hands off without incrementing response usage when the classifier returns handoff' do
|
||||
allow(mock_action_classifier_service).to receive(:classify).and_return({
|
||||
'action' => 'handoff',
|
||||
'action_reason' => 'explicit_human_request',
|
||||
'model' => 'gpt-4.1'
|
||||
})
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(conversation.reload.status).to eq('open')
|
||||
expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
|
||||
expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
|
||||
end
|
||||
|
||||
it 'skips the classifier when the legacy handoff token is returned' do
|
||||
allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'conversation_handoff' })
|
||||
expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(conversation.reload.status).to eq('open')
|
||||
expect(conversation.messages.outgoing.last.content).to eq(I18n.t('conversations.captain.handoff'))
|
||||
end
|
||||
|
||||
it 'falls back to the assistant response when the classifier fails' do
|
||||
error = StandardError.new('classifier unavailable')
|
||||
allow(mock_action_classifier_service).to receive(:classify).and_raise(error)
|
||||
allow(ChatwootExceptionTracker).to receive(:new).and_call_original
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(ChatwootExceptionTracker).to have_received(:new).with(error, account: account)
|
||||
expect(conversation.reload.status).to eq('pending')
|
||||
expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
|
||||
expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
|
||||
end
|
||||
|
||||
it 'falls back to the assistant response when the classifier returns an invalid action' do
|
||||
allow(mock_action_classifier_service).to receive(:classify).and_return({
|
||||
'action' => nil,
|
||||
'error' => 'invalid_classifier_response'
|
||||
})
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(conversation.reload.status).to eq('pending')
|
||||
expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain Specs')
|
||||
expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(1)
|
||||
end
|
||||
|
||||
it 'skips the classifier when the conversation is no longer pending after response generation' do
|
||||
allow(mock_llm_chat_service).to receive(:generate_response) do
|
||||
conversation.open!
|
||||
{ 'response' => 'Hey, welcome to Captain Specs' }
|
||||
end
|
||||
|
||||
expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new)
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
|
||||
expect(conversation.messages.outgoing.count).to eq(0)
|
||||
expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0)
|
||||
end
|
||||
end
|
||||
|
||||
it 'does not send a response when the conversation is no longer pending' do
|
||||
conversation.open!
|
||||
|
||||
@@ -292,9 +396,11 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
|
||||
it 'handles API errors and triggers handoff' do
|
||||
allow(mock_llm_chat_service).to receive(:generate_response)
|
||||
.and_raise(Faraday::BadRequestError, 'Bad request to image service')
|
||||
allow(Rails.logger).to receive(:info).and_call_original
|
||||
|
||||
described_class.perform_now(conversation, assistant)
|
||||
expect(conversation.reload.status).to eq('open')
|
||||
expect(Rails.logger).to have_received(:info).with(include('source=error reason=faraday_bad_request_error'))
|
||||
end
|
||||
|
||||
it 'succeeds when no error occurs' do
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::Llm::AssistantActionClassifierService do
|
||||
let(:account) { create(:account) }
|
||||
let(:assistant) do
|
||||
create(
|
||||
:captain_assistant,
|
||||
account: account,
|
||||
config: {
|
||||
'instructions' => 'Only transfer to a manager after the user explicitly confirms.'
|
||||
}
|
||||
)
|
||||
end
|
||||
let(:conversation) { create(:conversation, account: account) }
|
||||
let(:service) { described_class.new(assistant: assistant, conversation: conversation) }
|
||||
let(:mock_chat) { instance_double(RubyLLM::Chat) }
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: { 'action' => 'handoff', 'action_reason' => 'human_offer_accepted' }
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
|
||||
end
|
||||
|
||||
describe '#classify' do
|
||||
let(:message_history) do
|
||||
[
|
||||
{ role: 'user', content: 'I cannot log in' },
|
||||
{ role: 'assistant', content: 'Did you check your inbox?' },
|
||||
{ role: 'user', content: 'Yes, still no reset email' }
|
||||
]
|
||||
end
|
||||
|
||||
it 'passes delimited custom instructions and classifier context to the LLM' do
|
||||
expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:with_instructions).with(
|
||||
a_string_including('Account custom instructions are provided inside <account_custom_instructions> tags.')
|
||||
).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:ask) do |prompt|
|
||||
expect(prompt).to include(
|
||||
'<account_custom_instructions>',
|
||||
'Only transfer to a manager after the user explicitly confirms.',
|
||||
'<conversation_context>',
|
||||
'User: I cannot log in',
|
||||
'Assistant: Did you check your inbox?',
|
||||
'User: Yes, still no reset email',
|
||||
'<assistant_response_to_classify>',
|
||||
'Would you like to talk to support?'
|
||||
)
|
||||
expect(prompt).not_to include('"role"', '"content"', '<current_user_message>')
|
||||
|
||||
mock_response
|
||||
end
|
||||
|
||||
result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
|
||||
|
||||
expect(result).to include(
|
||||
'action' => 'handoff',
|
||||
'action_reason' => 'human_offer_accepted'
|
||||
)
|
||||
end
|
||||
|
||||
it 'uses the configured Captain model' do
|
||||
create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4.1-nano')
|
||||
|
||||
expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-nano').and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
|
||||
|
||||
expect(result).to include('model' => 'gpt-4.1-nano')
|
||||
end
|
||||
|
||||
context 'when the assistant has no custom instructions' do
|
||||
before do
|
||||
assistant.update!(config: assistant.config.except('instructions'))
|
||||
end
|
||||
|
||||
it 'does not add custom-instruction policy to the system prompt' do
|
||||
expect(mock_chat).to receive(:with_instructions).with(
|
||||
satisfy { |prompt| prompt.exclude?('Account custom instructions are provided') }
|
||||
).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'account custom instructions in system prompt' do
|
||||
before do
|
||||
assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff'))
|
||||
end
|
||||
|
||||
it 'adds custom instructions in a separate delimited section' do
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
expect(mock_chat).to receive(:with_instructions).with(
|
||||
a_string_including(
|
||||
'<account_custom_instructions>',
|
||||
'if user enters 1112234 suggest handoff',
|
||||
'</account_custom_instructions>'
|
||||
)
|
||||
) do |instructions|
|
||||
expect(instructions).not_to include('<custom-instructions>')
|
||||
expect(instructions.index('<account_custom_instructions>')).to be < instructions.index('```json')
|
||||
mock_chat
|
||||
end
|
||||
|
||||
service = described_class.new(assistant: assistant, conversation: conversation)
|
||||
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user