diff --git a/enterprise/lib/captain/conversation_completion_service.rb b/enterprise/lib/captain/conversation_completion_service.rb index c45559165..37f9add3e 100644 --- a/enterprise/lib/captain/conversation_completion_service.rb +++ b/enterprise/lib/captain/conversation_completion_service.rb @@ -12,7 +12,7 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService pattr_initialize [:account!, :conversation_display_id!] def perform - content = format_messages_as_string + content = format_evaluation_input return default_incomplete_response('No messages found') if content.blank? response = make_api_call( @@ -35,12 +35,58 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService Rails.root.join('enterprise/lib/captain/prompts', "#{file_name}.liquid").read end - def format_messages_as_string - messages = conversation_messages(start_from: 0) - messages.map do |msg| - sender_type = msg[:role] == 'user' ? 'Customer' : 'Assistant' - "#{sender_type}: #{msg[:content]}" + def format_evaluation_input + messages = conversation_message_records(start_from: 0) + return if messages.blank? + + [ + "Conversation status: #{conversation.status}", + format_messages_as_string(messages) + ].join("\n\n") + end + + def conversation_message_records(start_from: 0) + messages = [] + character_count = start_from + + conversation.messages + .where(message_type: [:incoming, :outgoing]) + .where(private: false) + .reorder('id desc') + .each do |message| + content = message.content_for_llm + next if content.blank? + break if character_count + content.length > TOKEN_LIMIT + + messages.prepend({ message: message, content: content }) + character_count += content.length + end + + messages + end + + def format_messages_as_string(messages) + transcript = messages.map do |message_context| + "#{message_sender_label(message_context[:message])}: #{message_context[:content]}" end.join("\n") + + "Conversation transcript:\n#{transcript}" + end + + def message_sender_label(message) + return 'Customer' if message.incoming? + return 'Captain' if captain_reply?(message) + return 'Bot' if bot_reply?(message) + + 'Assistant' + end + + def captain_reply?(message) + message.outgoing? && message.sender_type == 'Captain::Assistant' + end + + def bot_reply?(message) + message.outgoing? && message.sender_type.in?(['AgentBot', 'Captain::Assistant']) end def parse_response(message) diff --git a/enterprise/lib/captain/prompts/conversation_completion.liquid b/enterprise/lib/captain/prompts/conversation_completion.liquid index ed81039af..e039f60b0 100644 --- a/enterprise/lib/captain/prompts/conversation_completion.liquid +++ b/enterprise/lib/captain/prompts/conversation_completion.liquid @@ -2,18 +2,39 @@ You are evaluating whether a customer support conversation is complete and can b The conversation may be in any language. Apply these criteria based on the intent and meaning of messages, regardless of language. +You will receive: +- Conversation status +- Conversation transcript where messages are labeled as Customer, Captain, Bot, or Assistant + +This evaluator runs for inactive pending conversations. Focus on the latest pending exchange or latest unresolved customer request. Older messages may be present only for context. +If the conversation status is "pending", the conversation is still with Captain. Do not assume a handoff happened because Captain mentioned one. + A conversation is INCOMPLETE (keep open) if ANY of these apply: - The assistant asked a question or requested information that the customer hasn't provided - The customer asked a question that wasn't fully answered - The customer asked for something the assistant couldn't do — even if the assistant explained why, the customer's need is unmet - The customer raised multiple questions or issues and not all were addressed +- In the latest pending exchange, Captain, Bot, or Assistant said it handed off, will hand off, escalated, will escalate, or that a human/team/another party will continue the work +- In the latest pending exchange, Captain, Bot, or Assistant promised future action or follow-up instead of resolving the customer's request +- In the latest pending exchange, the customer is waiting for another party's action, response, status update, or investigation result +- The latest customer message is only an attachment placeholder such as "[Attachment]" and there is no later text explaining what it contains or showing the issue was answered +- The customer says they were not helped, asks why nobody replied, repeats the unresolved issue after a previous answer, or otherwise indicates dissatisfaction with the current help + +Do NOT treat these as incomplete by themselves: +- A generic greeting or broad optional offer from Captain/Bot/Assistant, such as "How can I help?", "What would you like to know?", or "Anything else?", when the customer has not made a recognizable request +- A customer greeting, single-word reply, name, phone number, or gibberish with no recognizable question/request, followed only by Captain/Bot/Assistant asking what the customer needs +- An optional invitation for the customer to ask more questions after the assistant already answered the actual request +- Older handoff, escalation, or follow-up messages from a previous exchange when the latest customer message starts a new topic, has no recognizable request, or has already been answered + +Important handoff rule: +- A handoff, escalation, transfer, acknowledgement, or promise of future follow-up is not a resolution by itself +- If conversation status is "pending" and Captain/Bot/Assistant says it handed off, will hand off, or that another party will continue the work in the latest pending exchange, keep the conversation INCOMPLETE. A conversation is COMPLETE only if ALL of these are true: - The assistant's answer fully addressed the customer's question or issue and is self-contained — it requires no further action from the customer - There are no unanswered questions, unmet requests, or outstanding follow-ups from either side - Note: customers often do not explicitly say thanks or confirm resolution. If the assistant gave a complete, self-contained answer and the customer had no follow-up, that is sufficient. Do not require explicit gratitude or confirmation. -- If the customer sent only one or two short messages (single words, names, phone numbers, or gibberish) with no recognizable question or request across the entire conversation, and the - assistant has responded asking for clarification, the conversation is COMPLETE. +- If the customer sent only one or two short text messages (greetings, single words, names, phone numbers, or gibberish) with no recognizable question or request across the entire conversation, and Captain/Bot/Assistant has responded asking what they need or offering help, the conversation is COMPLETE. Analyze the conversation and respond with ONLY a JSON object (no other text): {"complete": true, "reason": "brief explanation"} diff --git a/spec/enterprise/lib/captain/conversation_completion_service_spec.rb b/spec/enterprise/lib/captain/conversation_completion_service_spec.rb index 80b9ab1d8..9cdfc822c 100644 --- a/spec/enterprise/lib/captain/conversation_completion_service_spec.rb +++ b/spec/enterprise/lib/captain/conversation_completion_service_spec.rb @@ -68,6 +68,110 @@ RSpec.describe Captain::ConversationCompletionService do end end + context 'when building evaluation context' do + let(:captain_assistant) { create(:captain_assistant, account: account) } + let(:mock_response) do + instance_double( + RubyLLM::Message, + content: { 'complete' => false, 'reason' => 'Human follow-up is still pending' }, + input_tokens: 100, + output_tokens: 20 + ) + end + + it 'includes conversation status and speaker labels' do + conversation.update!(status: :pending, waiting_since: 2.hours.ago) + create(:message, conversation: conversation, inbox: inbox, account: account, message_type: :incoming, content: 'I need help with a refund') + create( + :message, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :outgoing, + sender: captain_assistant, + content: 'I will transfer this to support for review.' + ) + + expect(mock_chat).to receive(:ask) do |content| + expect(content).to include( + 'Conversation status: pending', + 'Conversation transcript:', + 'Customer: I need help with a refund', + 'Captain: I will transfer this to support for review.' + ) + + mock_response + end + + result = service.perform + + expect(result[:complete]).to be false + end + + it 'includes pending captain handoff evidence in the transcript' do + conversation.update!(status: :pending) + create(:message, conversation: conversation, inbox: inbox, account: account, message_type: :incoming, content: 'Please cancel my order') + create( + :message, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :outgoing, + sender: captain_assistant, + content: 'I will transfer this to a specialist and they will follow up here.' + ) + + expect(mock_chat).to receive(:ask) do |content| + expect(content).to include( + 'Conversation status: pending', + 'Captain: I will transfer this to a specialist and they will follow up here.' + ) + + mock_response + end + + result = service.perform + + expect(result[:complete]).to be false + end + + it 'reuses computed message content while formatting the transcript' do + content_for_llm_calls_by_message_id = Hash.new(0) + allow_any_instance_of(Message).to receive(:content_for_llm).and_wrap_original do |method, *args| # rubocop:disable RSpec/AnyInstance + content_for_llm_calls_by_message_id[method.receiver.id] += 1 + method.call(*args) + end + + incoming_message = create( + :message, + :with_attachment, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :incoming, + content: nil + ) + outgoing_message = create( + :message, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :outgoing, + sender: captain_assistant, + content: 'What do you need help with?' + ) + + allow(mock_chat).to receive(:ask).and_return(mock_response) + + service.perform + + expect(content_for_llm_calls_by_message_id).to include( + incoming_message.id => 1, + outgoing_message.id => 1 + ) + end + end + context 'when conversation has no messages' do it 'returns incomplete with appropriate reason' do result = service.perform