Merge branch 'develop' into feature/cw-7495-models

enterprise/lib/captain/conversation_completion_service.rb

@@ -12,7 +12,7 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService
   pattr_initialize [:account!, :conversation_display_id!]
 
   def perform
-    content = format_messages_as_string
+    content = format_evaluation_input
     return default_incomplete_response('No messages found') if content.blank?
 
     response = make_api_call(
@@ -35,12 +35,58 @@ class Captain::ConversationCompletionService < Captain::BaseTaskService
     Rails.root.join('enterprise/lib/captain/prompts', "#{file_name}.liquid").read
   end
 
-  def format_messages_as_string
-    messages = conversation_messages(start_from: 0)
-    messages.map do |msg|
-      sender_type = msg[:role] == 'user' ? 'Customer' : 'Assistant'
-      "#{sender_type}: #{msg[:content]}"
+  def format_evaluation_input
+    messages = conversation_message_records(start_from: 0)
+    return if messages.blank?
+
+    [
+      "Conversation status: #{conversation.status}",
+      format_messages_as_string(messages)
+    ].join("\n\n")
+  end
+
+  def conversation_message_records(start_from: 0)
+    messages = []
+    character_count = start_from
+
+    conversation.messages
+                .where(message_type: [:incoming, :outgoing])
+                .where(private: false)
+                .reorder('id desc')
+                .each do |message|
+      content = message.content_for_llm
+      next if content.blank?
+      break if character_count + content.length > TOKEN_LIMIT
+
+      messages.prepend({ message: message, content: content })
+      character_count += content.length
+    end
+
+    messages
+  end
+
+  def format_messages_as_string(messages)
+    transcript = messages.map do |message_context|
+      "#{message_sender_label(message_context[:message])}: #{message_context[:content]}"
     end.join("\n")
+
+    "Conversation transcript:\n#{transcript}"
+  end
+
+  def message_sender_label(message)
+    return 'Customer' if message.incoming?
+    return 'Captain' if captain_reply?(message)
+    return 'Bot' if bot_reply?(message)
+
+    'Assistant'
+  end
+
+  def captain_reply?(message)
+    message.outgoing? && message.sender_type == 'Captain::Assistant'
+  end
+
+  def bot_reply?(message)
+    message.outgoing? && message.sender_type.in?(['AgentBot', 'Captain::Assistant'])
   end
 
   def parse_response(message)

enterprise/lib/captain/prompts/conversation_completion.liquid

@@ -2,18 +2,39 @@ You are evaluating whether a customer support conversation is complete and can b
 
 The conversation may be in any language. Apply these criteria based on the intent and meaning of messages, regardless of language.
 
+You will receive:
+- Conversation status
+- Conversation transcript where messages are labeled as Customer, Captain, Bot, or Assistant
+
+This evaluator runs for inactive pending conversations. Focus on the latest pending exchange or latest unresolved customer request. Older messages may be present only for context.
+If the conversation status is "pending", the conversation is still with Captain. Do not assume a handoff happened because Captain mentioned one.
+
 A conversation is INCOMPLETE (keep open) if ANY of these apply:
 - The assistant asked a question or requested information that the customer hasn't provided
 - The customer asked a question that wasn't fully answered
 - The customer asked for something the assistant couldn't do — even if the assistant explained why, the customer's need is unmet
 - The customer raised multiple questions or issues and not all were addressed
+- In the latest pending exchange, Captain, Bot, or Assistant said it handed off, will hand off, escalated, will escalate, or that a human/team/another party will continue the work
+- In the latest pending exchange, Captain, Bot, or Assistant promised future action or follow-up instead of resolving the customer's request
+- In the latest pending exchange, the customer is waiting for another party's action, response, status update, or investigation result
+- The latest customer message is only an attachment placeholder such as "[Attachment]" and there is no later text explaining what it contains or showing the issue was answered
+- The customer says they were not helped, asks why nobody replied, repeats the unresolved issue after a previous answer, or otherwise indicates dissatisfaction with the current help
+
+Do NOT treat these as incomplete by themselves:
+- A generic greeting or broad optional offer from Captain/Bot/Assistant, such as "How can I help?", "What would you like to know?", or "Anything else?", when the customer has not made a recognizable request
+- A customer greeting, single-word reply, name, phone number, or gibberish with no recognizable question/request, followed only by Captain/Bot/Assistant asking what the customer needs
+- An optional invitation for the customer to ask more questions after the assistant already answered the actual request
+- Older handoff, escalation, or follow-up messages from a previous exchange when the latest customer message starts a new topic, has no recognizable request, or has already been answered
+
+Important handoff rule:
+- A handoff, escalation, transfer, acknowledgement, or promise of future follow-up is not a resolution by itself
+- If conversation status is "pending" and Captain/Bot/Assistant says it handed off, will hand off, or that another party will continue the work in the latest pending exchange, keep the conversation INCOMPLETE.
 
 A conversation is COMPLETE only if ALL of these are true:
 - The assistant's answer fully addressed the customer's question or issue and is self-contained — it requires no further action from the customer
 - There are no unanswered questions, unmet requests, or outstanding follow-ups from either side
 - Note: customers often do not explicitly say thanks or confirm resolution. If the assistant gave a complete, self-contained answer and the customer had no follow-up, that is sufficient. Do not require explicit gratitude or confirmation.
-- If the customer sent only one or two short messages (single words, names, phone numbers, or gibberish) with no recognizable question or request across the entire conversation, and the
-  assistant has responded asking for clarification, the conversation is COMPLETE.
+- If the customer sent only one or two short text messages (greetings, single words, names, phone numbers, or gibberish) with no recognizable question or request across the entire conversation, and Captain/Bot/Assistant has responded asking what they need or offering help, the conversation is COMPLETE.
 
 Analyze the conversation and respond with ONLY a JSON object (no other text):
 {"complete": true, "reason": "brief explanation"}

spec/enterprise/lib/captain/conversation_completion_service_spec.rb

@@ -68,6 +68,110 @@ RSpec.describe Captain::ConversationCompletionService do
       end
     end
 
+    context 'when building evaluation context' do
+      let(:captain_assistant) { create(:captain_assistant, account: account) }
+      let(:mock_response) do
+        instance_double(
+          RubyLLM::Message,
+          content: { 'complete' => false, 'reason' => 'Human follow-up is still pending' },
+          input_tokens: 100,
+          output_tokens: 20
+        )
+      end
+
+      it 'includes conversation status and speaker labels' do
+        conversation.update!(status: :pending, waiting_since: 2.hours.ago)
+        create(:message, conversation: conversation, inbox: inbox, account: account, message_type: :incoming, content: 'I need help with a refund')
+        create(
+          :message,
+          conversation: conversation,
+          inbox: inbox,
+          account: account,
+          message_type: :outgoing,
+          sender: captain_assistant,
+          content: 'I will transfer this to support for review.'
+        )
+
+        expect(mock_chat).to receive(:ask) do |content|
+          expect(content).to include(
+            'Conversation status: pending',
+            'Conversation transcript:',
+            'Customer: I need help with a refund',
+            'Captain: I will transfer this to support for review.'
+          )
+
+          mock_response
+        end
+
+        result = service.perform
+
+        expect(result[:complete]).to be false
+      end
+
+      it 'includes pending captain handoff evidence in the transcript' do
+        conversation.update!(status: :pending)
+        create(:message, conversation: conversation, inbox: inbox, account: account, message_type: :incoming, content: 'Please cancel my order')
+        create(
+          :message,
+          conversation: conversation,
+          inbox: inbox,
+          account: account,
+          message_type: :outgoing,
+          sender: captain_assistant,
+          content: 'I will transfer this to a specialist and they will follow up here.'
+        )
+
+        expect(mock_chat).to receive(:ask) do |content|
+          expect(content).to include(
+            'Conversation status: pending',
+            'Captain: I will transfer this to a specialist and they will follow up here.'
+          )
+
+          mock_response
+        end
+
+        result = service.perform
+
+        expect(result[:complete]).to be false
+      end
+
+      it 'reuses computed message content while formatting the transcript' do
+        content_for_llm_calls_by_message_id = Hash.new(0)
+        allow_any_instance_of(Message).to receive(:content_for_llm).and_wrap_original do |method, *args| # rubocop:disable RSpec/AnyInstance
+          content_for_llm_calls_by_message_id[method.receiver.id] += 1
+          method.call(*args)
+        end
+
+        incoming_message = create(
+          :message,
+          :with_attachment,
+          conversation: conversation,
+          inbox: inbox,
+          account: account,
+          message_type: :incoming,
+          content: nil
+        )
+        outgoing_message = create(
+          :message,
+          conversation: conversation,
+          inbox: inbox,
+          account: account,
+          message_type: :outgoing,
+          sender: captain_assistant,
+          content: 'What do you need help with?'
+        )
+
+        allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+        service.perform
+
+        expect(content_for_llm_calls_by_message_id).to include(
+          incoming_message.id => 1,
+          outgoing_message.id => 1
+        )
+      end
+    end
+
     context 'when conversation has no messages' do
       it 'returns incomplete with appropriate reason' do
         result = service.perform