From 1b6a80d84d19310217c83c5d8140d08ba9b675f9 Mon Sep 17 00:00:00 2001 From: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:45:40 +0530 Subject: [PATCH] fix(captain): improve conversation completion evaluation (#14967) # Pull Request Template ## Description Please include a summary of the change and issue(s) fixed. Also, mention relevant motivation, context, and any dependencies that this change requires. Fixes https://linear.app/chatwoot/issue/AI-136/check-conversation-status-while-auto-resolving - After 60mins of inactivity, we run a job that decides if pending conversations are resolvable or need handoff - the prompt was a bit conservative and didn't have conversation state context ## Type of change Please delete options that are not relevant. - [x] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration. locally ran a sample eval ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --------- Co-authored-by: Sony Mathew --- .../conversation_completion_service.rb | 58 +++++++++- .../prompts/conversation_completion.liquid | 25 ++++- .../conversation_completion_service_spec.rb | 104 ++++++++++++++++++ 3 files changed, 179 insertions(+), 8 deletions(-) diff --git a/enterprise/lib/captain/conversation_completion_service.rb b/enterprise/lib/captain/conversation_completion_service.rb index c45559165..37f9add3e 100644 --- a/enterprise/lib/captain/conversation_completion_service.rb +++ b/enterprise/lib/captain/conversation_completion_service.rb @@ -12,7 +12,7 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService pattr_initialize [:account!, :conversation_display_id!] def perform - content = format_messages_as_string + content = format_evaluation_input return default_incomplete_response('No messages found') if content.blank? response = make_api_call( @@ -35,12 +35,58 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService Rails.root.join('enterprise/lib/captain/prompts', "#{file_name}.liquid").read end - def format_messages_as_string - messages = conversation_messages(start_from: 0) - messages.map do |msg| - sender_type = msg[:role] == 'user' ? 'Customer' : 'Assistant' - "#{sender_type}: #{msg[:content]}" + def format_evaluation_input + messages = conversation_message_records(start_from: 0) + return if messages.blank? + + [ + "Conversation status: #{conversation.status}", + format_messages_as_string(messages) + ].join("\n\n") + end + + def conversation_message_records(start_from: 0) + messages = [] + character_count = start_from + + conversation.messages + .where(message_type: [:incoming, :outgoing]) + .where(private: false) + .reorder('id desc') + .each do |message| + content = message.content_for_llm + next if content.blank? + break if character_count + content.length > TOKEN_LIMIT + + messages.prepend({ message: message, content: content }) + character_count += content.length + end + + messages + end + + def format_messages_as_string(messages) + transcript = messages.map do |message_context| + "#{message_sender_label(message_context[:message])}: #{message_context[:content]}" end.join("\n") + + "Conversation transcript:\n#{transcript}" + end + + def message_sender_label(message) + return 'Customer' if message.incoming? + return 'Captain' if captain_reply?(message) + return 'Bot' if bot_reply?(message) + + 'Assistant' + end + + def captain_reply?(message) + message.outgoing? && message.sender_type == 'Captain::Assistant' + end + + def bot_reply?(message) + message.outgoing? && message.sender_type.in?(['AgentBot', 'Captain::Assistant']) end def parse_response(message) diff --git a/enterprise/lib/captain/prompts/conversation_completion.liquid b/enterprise/lib/captain/prompts/conversation_completion.liquid index ed81039af..e039f60b0 100644 --- a/enterprise/lib/captain/prompts/conversation_completion.liquid +++ b/enterprise/lib/captain/prompts/conversation_completion.liquid @@ -2,18 +2,39 @@ You are evaluating whether a customer support conversation is complete and can b The conversation may be in any language. Apply these criteria based on the intent and meaning of messages, regardless of language. +You will receive: +- Conversation status +- Conversation transcript where messages are labeled as Customer, Captain, Bot, or Assistant + +This evaluator runs for inactive pending conversations. Focus on the latest pending exchange or latest unresolved customer request. Older messages may be present only for context. +If the conversation status is "pending", the conversation is still with Captain. Do not assume a handoff happened because Captain mentioned one. + A conversation is INCOMPLETE (keep open) if ANY of these apply: - The assistant asked a question or requested information that the customer hasn't provided - The customer asked a question that wasn't fully answered - The customer asked for something the assistant couldn't do — even if the assistant explained why, the customer's need is unmet - The customer raised multiple questions or issues and not all were addressed +- In the latest pending exchange, Captain, Bot, or Assistant said it handed off, will hand off, escalated, will escalate, or that a human/team/another party will continue the work +- In the latest pending exchange, Captain, Bot, or Assistant promised future action or follow-up instead of resolving the customer's request +- In the latest pending exchange, the customer is waiting for another party's action, response, status update, or investigation result +- The latest customer message is only an attachment placeholder such as "[Attachment]" and there is no later text explaining what it contains or showing the issue was answered +- The customer says they were not helped, asks why nobody replied, repeats the unresolved issue after a previous answer, or otherwise indicates dissatisfaction with the current help + +Do NOT treat these as incomplete by themselves: +- A generic greeting or broad optional offer from Captain/Bot/Assistant, such as "How can I help?", "What would you like to know?", or "Anything else?", when the customer has not made a recognizable request +- A customer greeting, single-word reply, name, phone number, or gibberish with no recognizable question/request, followed only by Captain/Bot/Assistant asking what the customer needs +- An optional invitation for the customer to ask more questions after the assistant already answered the actual request +- Older handoff, escalation, or follow-up messages from a previous exchange when the latest customer message starts a new topic, has no recognizable request, or has already been answered + +Important handoff rule: +- A handoff, escalation, transfer, acknowledgement, or promise of future follow-up is not a resolution by itself +- If conversation status is "pending" and Captain/Bot/Assistant says it handed off, will hand off, or that another party will continue the work in the latest pending exchange, keep the conversation INCOMPLETE. A conversation is COMPLETE only if ALL of these are true: - The assistant's answer fully addressed the customer's question or issue and is self-contained — it requires no further action from the customer - There are no unanswered questions, unmet requests, or outstanding follow-ups from either side - Note: customers often do not explicitly say thanks or confirm resolution. If the assistant gave a complete, self-contained answer and the customer had no follow-up, that is sufficient. Do not require explicit gratitude or confirmation. -- If the customer sent only one or two short messages (single words, names, phone numbers, or gibberish) with no recognizable question or request across the entire conversation, and the - assistant has responded asking for clarification, the conversation is COMPLETE. +- If the customer sent only one or two short text messages (greetings, single words, names, phone numbers, or gibberish) with no recognizable question or request across the entire conversation, and Captain/Bot/Assistant has responded asking what they need or offering help, the conversation is COMPLETE. Analyze the conversation and respond with ONLY a JSON object (no other text): {"complete": true, "reason": "brief explanation"} diff --git a/spec/enterprise/lib/captain/conversation_completion_service_spec.rb b/spec/enterprise/lib/captain/conversation_completion_service_spec.rb index 80b9ab1d8..9cdfc822c 100644 --- a/spec/enterprise/lib/captain/conversation_completion_service_spec.rb +++ b/spec/enterprise/lib/captain/conversation_completion_service_spec.rb @@ -68,6 +68,110 @@ RSpec.describe Captain::ConversationCompletionService do end end + context 'when building evaluation context' do + let(:captain_assistant) { create(:captain_assistant, account: account) } + let(:mock_response) do + instance_double( + RubyLLM::Message, + content: { 'complete' => false, 'reason' => 'Human follow-up is still pending' }, + input_tokens: 100, + output_tokens: 20 + ) + end + + it 'includes conversation status and speaker labels' do + conversation.update!(status: :pending, waiting_since: 2.hours.ago) + create(:message, conversation: conversation, inbox: inbox, account: account, message_type: :incoming, content: 'I need help with a refund') + create( + :message, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :outgoing, + sender: captain_assistant, + content: 'I will transfer this to support for review.' + ) + + expect(mock_chat).to receive(:ask) do |content| + expect(content).to include( + 'Conversation status: pending', + 'Conversation transcript:', + 'Customer: I need help with a refund', + 'Captain: I will transfer this to support for review.' + ) + + mock_response + end + + result = service.perform + + expect(result[:complete]).to be false + end + + it 'includes pending captain handoff evidence in the transcript' do + conversation.update!(status: :pending) + create(:message, conversation: conversation, inbox: inbox, account: account, message_type: :incoming, content: 'Please cancel my order') + create( + :message, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :outgoing, + sender: captain_assistant, + content: 'I will transfer this to a specialist and they will follow up here.' + ) + + expect(mock_chat).to receive(:ask) do |content| + expect(content).to include( + 'Conversation status: pending', + 'Captain: I will transfer this to a specialist and they will follow up here.' + ) + + mock_response + end + + result = service.perform + + expect(result[:complete]).to be false + end + + it 'reuses computed message content while formatting the transcript' do + content_for_llm_calls_by_message_id = Hash.new(0) + allow_any_instance_of(Message).to receive(:content_for_llm).and_wrap_original do |method, *args| # rubocop:disable RSpec/AnyInstance + content_for_llm_calls_by_message_id[method.receiver.id] += 1 + method.call(*args) + end + + incoming_message = create( + :message, + :with_attachment, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :incoming, + content: nil + ) + outgoing_message = create( + :message, + conversation: conversation, + inbox: inbox, + account: account, + message_type: :outgoing, + sender: captain_assistant, + content: 'What do you need help with?' + ) + + allow(mock_chat).to receive(:ask).and_return(mock_response) + + service.perform + + expect(content_for_llm_calls_by_message_id).to include( + incoming_message.id => 1, + outgoing_message.id => 1 + ) + end + end + context 'when conversation has no messages' do it 'returns incomplete with appropriate reason' do result = service.perform