From 9749a3dc96b451657face269f0d4401474b06595 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 16 Jul 2026 18:20:44 +0530 Subject: [PATCH] feat: capture captain sessions for v2 assistant responses [CW-7485] (#14971) Records a `Captain::Session` row for every Captain V2 assistant response delivered in a conversation, so we can show how a response was generated and report on credit, FAQ, and document usage. Stacked on #14970 (the `captain_sessions` model). ## What changed - `FaqLookupTool` now records the retrieved FAQ ids (and their backing document ids) into the shared run state, accumulated across tool calls. - `AgentRunnerService` exposes the raw ai-agents run result via `last_run_result`; the `generate_response` return shape is unchanged, so the playground path is unaffected. - New `Captain::Assistant::SessionCaptureService` builds the session: scenario resolved from the answering agent name, model from `assistant.agent_model`, token usage plus the trimmed current-turn conversation history stored in `run_context`. - `ResponseBuilderJob` captures after delivery: `credits_consumed` mirrors the actual charge (1.0 for a billed response, 0.0 for handoffs, where the session points at the customer-facing handoff message). Capture runs outside the delivery transaction and swallows its own failures, so a logging bug can never block or roll back a customer reply. V1 responses and copilot are out of scope; copilot capture comes next. ## How to test On an account with `captain_integration_v2` enabled and an inbox connected to an assistant with approved FAQs, send a customer message on a pending conversation. After the assistant replies, a `Captain::Session` row should exist with the conversation as subject, the reply message as result, the FAQs/documents used, and the run context for that turn. Asking for a human agent should produce a zero-credit session pointing at the handoff message. CleanShot 2026-07-15 at 17 25 40@2x --- .../captain/conversation/message_builder.rb | 53 ++++++ .../conversation/response_builder_job.rb | 74 ++------ enterprise/app/models/concerns/agentable.rb | 14 +- .../captain/assistant/agent_runner_service.rb | 43 +---- .../captain/assistant/runner_state_helper.rb | 42 +++++ .../assistant/session_capture_service.rb | 64 +++++++ .../lib/captain/tools/faq_lookup_tool.rb | 13 +- .../conversation/response_builder_job_spec.rb | 99 +++++++++++ .../lib/captain/tools/faq_lookup_tool_spec.rb | 21 +++ .../assistant/agent_runner_service_spec.rb | 12 ++ .../assistant/session_capture_service_spec.rb | 167 ++++++++++++++++++ 11 files changed, 498 insertions(+), 104 deletions(-) create mode 100644 enterprise/app/jobs/captain/conversation/message_builder.rb create mode 100644 enterprise/app/services/captain/assistant/runner_state_helper.rb create mode 100644 enterprise/app/services/captain/assistant/session_capture_service.rb create mode 100644 spec/enterprise/services/captain/assistant/session_capture_service_spec.rb diff --git a/enterprise/app/jobs/captain/conversation/message_builder.rb b/enterprise/app/jobs/captain/conversation/message_builder.rb new file mode 100644 index 000000000..7954e9465 --- /dev/null +++ b/enterprise/app/jobs/captain/conversation/message_builder.rb @@ -0,0 +1,53 @@ +module Captain::Conversation::MessageBuilder + private + + def collect_previous_messages + @conversation + .messages + .where(message_type: [:incoming, :outgoing]) + .where(private: false) + .map do |message| + message_hash = { + content: prepare_multimodal_message_content(message), + role: determine_role(message) + } + + # Include agent_name if present in additional_attributes + message_hash[:agent_name] = message.additional_attributes['agent_name'] if message.additional_attributes&.dig('agent_name').present? + + message_hash + end + end + + def determine_role(message) + message.message_type == 'incoming' ? 'user' : 'assistant' + end + + def prepare_multimodal_message_content(message) + Captain::OpenAiMessageBuilderService.new(message: message).generate_content + end + + def create_messages + validate_message_content!(@response['response']) + create_outgoing_message(@response['response'], agent_name: @response['agent_name']) + end + + def validate_message_content!(content) + raise ArgumentError, 'Message content cannot be blank' if content.blank? + end + + def create_outgoing_message(message_content, agent_name: nil, preserve_waiting_since: false) + additional_attrs = {} + additional_attrs[:agent_name] = agent_name if agent_name.present? + + @conversation.messages.create!( + message_type: :outgoing, + account_id: account.id, + inbox_id: inbox.id, + sender: @assistant, + content: message_content, + additional_attributes: additional_attrs, + preserve_waiting_since: preserve_waiting_since + ) + end +end diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index 282d94862..fb0e72721 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -1,6 +1,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob include Captain::Conversation::V1ActionClassifier include Captain::Conversation::V1FalsePromiseHandler + include Captain::Conversation::MessageBuilder MAX_MESSAGE_LENGTH = 10_000 retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds @@ -44,9 +45,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def generate_response_with_v2 - @response = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation).generate_response( - message_history: collect_previous_messages_with_resolution_markers - ) + runner_service = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation) + message_history = Captain::Conversation::MessageHistoryBuilderService.new(conversation: @conversation).perform + @response = runner_service.generate_response(message_history: message_history) + @run_result = runner_service.last_run_result + process_response end @@ -65,6 +68,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob # left is the customer-facing follow-up message. process_v2_handoff end + capture_assistant_session(result_message: @handoff_message, credits_consumed: 0.0) elsif v1_handoff_requested? # V1 only signals via the response string — no state has been touched yet. If # the conversation isn't pending anymore, a human took over mid-run; bail out @@ -73,44 +77,16 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob process_v1_handoff elsif conversation_pending? + message = nil ActiveRecord::Base.transaction do - create_messages + message = create_messages Rails.logger.info("[CAPTAIN][ResponseBuilderJob] Incrementing response usage for #{account.id}") account.increment_response_usage end + capture_assistant_session(result_message: message, credits_consumed: 1.0) end end - def collect_previous_messages - @conversation - .messages - .where(message_type: [:incoming, :outgoing]) - .where(private: false) - .map do |message| - message_hash = { - content: prepare_multimodal_message_content(message), - role: determine_role(message) - } - - # Include agent_name if present in additional_attributes - message_hash[:agent_name] = message.additional_attributes['agent_name'] if message.additional_attributes&.dig('agent_name').present? - - message_hash - end - end - - def collect_previous_messages_with_resolution_markers - Captain::Conversation::MessageHistoryBuilderService.new(conversation: @conversation).perform - end - - def determine_role(message) - message.message_type == 'incoming' ? 'user' : 'assistant' - end - - def prepare_multimodal_message_content(message) - Captain::OpenAiMessageBuilderService.new(message: message).generate_content - end - def v1_handoff_requested? legacy_v1_handoff_token? || classifier_v1_handoff_requested? end @@ -157,34 +133,18 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def create_handoff_message(preserve_waiting_since: false) - create_outgoing_message( + @handoff_message = create_outgoing_message( @assistant.config['handoff_message'].presence || I18n.t('conversations.captain.handoff'), preserve_waiting_since: preserve_waiting_since ) end - def create_messages - validate_message_content!(@response['response']) - create_outgoing_message(@response['response'], agent_name: @response['agent_name']) - end - - def validate_message_content!(content) - raise ArgumentError, 'Message content cannot be blank' if content.blank? - end - - def create_outgoing_message(message_content, agent_name: nil, preserve_waiting_since: false) - additional_attrs = {} - additional_attrs[:agent_name] = agent_name if agent_name.present? - - @conversation.messages.create!( - message_type: :outgoing, - account_id: account.id, - inbox_id: inbox.id, - sender: @assistant, - content: message_content, - additional_attributes: additional_attrs, - preserve_waiting_since: preserve_waiting_since - ) + # Capture runs outside the delivery transaction and never raises (the service + # swallows its own failures): a session-logging bug must never roll back the + # customer reply or trigger the top-level handle_error handoff on top of it. + def capture_assistant_session(result_message:, credits_consumed:) + Captain::Assistant::SessionCaptureService.new(assistant: @assistant, conversation: @conversation, run_result: @run_result, + result_message: result_message, credits_consumed: credits_consumed).capture end def handle_error(error) diff --git a/enterprise/app/models/concerns/agentable.rb b/enterprise/app/models/concerns/agentable.rb index 086deedc1..c12e15f52 100644 --- a/enterprise/app/models/concerns/agentable.rb +++ b/enterprise/app/models/concerns/agentable.rb @@ -31,6 +31,13 @@ module Concerns::Agentable Captain::PromptRenderer.render(template_name, enhanced_context.with_indifferent_access) end + def agent_model + route = Llm::FeatureRouter.resolve(feature: 'assistant', account: account) + return route[:model] if route[:source] == :account_override || account&.feature_enabled?('captain_integration_v2') + + installation_model.presence || route[:model] + end + private def agent_name @@ -45,13 +52,6 @@ module Concerns::Agentable [] # Default implementation, override if needed end - def agent_model - route = Llm::FeatureRouter.resolve(feature: 'assistant', account: account) - return route[:model] if route[:source] == :account_override || account&.feature_enabled?('captain_integration_v2') - - installation_model.presence || route[:model] - end - def installation_model InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_MODEL')&.value end diff --git a/enterprise/app/services/captain/assistant/agent_runner_service.rb b/enterprise/app/services/captain/assistant/agent_runner_service.rb index 09070eba6..197b2e39b 100644 --- a/enterprise/app/services/captain/assistant/agent_runner_service.rb +++ b/enterprise/app/services/captain/assistant/agent_runner_service.rb @@ -5,20 +5,10 @@ class Captain::Assistant::AgentRunnerService include Integrations::LlmInstrumentationConstants include Captain::Assistant::RunnerCallbacksHelper include Captain::Assistant::TracePayloadHelper + include Captain::Assistant::RunnerStateHelper - CONVERSATION_STATE_ATTRIBUTES = %i[ - id display_id inbox_id contact_id status priority - label_list custom_attributes additional_attributes - ].freeze + attr_reader :last_run_result - CONTACT_STATE_ATTRIBUTES = %i[ - id name email phone_number identifier contact_type - custom_attributes additional_attributes - ].freeze - - CONTACT_INBOX_STATE_ATTRIBUTES = %i[id hmac_verified].freeze - - CAMPAIGN_STATE_ATTRIBUTES = %i[id title message campaign_type description].freeze def initialize(assistant:, conversation: nil, callbacks: {}, source: nil) @assistant = assistant @conversation = conversation @@ -29,9 +19,9 @@ class Captain::Assistant::AgentRunnerService def generate_response(message_history: []) message_to_process, context = run_payload(message_history) - result = runner.run(message_to_process, context: context, max_turns: 10) + @last_run_result = runner.run(message_to_process, context: context, max_turns: 10) - process_agent_result(result) + process_agent_result(@last_run_result) rescue StandardError => e # In rake/local runs, conversation may not be present, so account is optional here. ChatwootExceptionTracker.new(e, account: @conversation&.account).capture_exception @@ -111,31 +101,6 @@ class Captain::Assistant::AgentRunnerService } end - def build_state - state = { - account_id: @assistant.account_id, - assistant_id: @assistant.id, - assistant_config: @assistant.config, - timezone: @conversation&.inbox&.timezone.presence || 'UTC' - } - state[:source] = @source if @source.present? - - build_conversation_state(state) if @conversation - state - end - - def build_conversation_state(state) - state[:conversation] = slice_attrs(@conversation, CONVERSATION_STATE_ATTRIBUTES) - state[:channel_type] = @conversation.inbox&.channel_type - state[:contact] = slice_attrs(@conversation.contact, CONTACT_STATE_ATTRIBUTES) if @conversation.contact - state[:campaign] = slice_attrs(@conversation.campaign, CAMPAIGN_STATE_ATTRIBUTES) if @conversation.campaign - state[:contact_inbox] = slice_attrs(@conversation.contact_inbox, CONTACT_INBOX_STATE_ATTRIBUTES) if @conversation.contact_inbox - end - - def slice_attrs(record, keys) - record.attributes.symbolize_keys.slice(*keys) - end - def build_and_wire_agents assistant_agent = @assistant.agent scenario_agents = @assistant.scenarios.enabled.map(&:agent) diff --git a/enterprise/app/services/captain/assistant/runner_state_helper.rb b/enterprise/app/services/captain/assistant/runner_state_helper.rb new file mode 100644 index 000000000..d2d6146f8 --- /dev/null +++ b/enterprise/app/services/captain/assistant/runner_state_helper.rb @@ -0,0 +1,42 @@ +module Captain::Assistant::RunnerStateHelper + CONVERSATION_STATE_ATTRIBUTES = %i[ + id display_id inbox_id contact_id status priority + label_list custom_attributes additional_attributes + ].freeze + + CONTACT_STATE_ATTRIBUTES = %i[ + id name email phone_number identifier contact_type + custom_attributes additional_attributes + ].freeze + + CONTACT_INBOX_STATE_ATTRIBUTES = %i[id hmac_verified].freeze + + CAMPAIGN_STATE_ATTRIBUTES = %i[id title message campaign_type description].freeze + + private + + def build_state + state = { + account_id: @assistant.account_id, + assistant_id: @assistant.id, + assistant_config: @assistant.config, + timezone: @conversation&.inbox&.timezone.presence || 'UTC' + } + state[:source] = @source if @source.present? + + build_conversation_state(state) if @conversation + state + end + + def build_conversation_state(state) + state[:conversation] = slice_attrs(@conversation, CONVERSATION_STATE_ATTRIBUTES) + state[:channel_type] = @conversation.inbox&.channel_type + state[:contact] = slice_attrs(@conversation.contact, CONTACT_STATE_ATTRIBUTES) if @conversation.contact + state[:campaign] = slice_attrs(@conversation.campaign, CAMPAIGN_STATE_ATTRIBUTES) if @conversation.campaign + state[:contact_inbox] = slice_attrs(@conversation.contact_inbox, CONTACT_INBOX_STATE_ATTRIBUTES) if @conversation.contact_inbox + end + + def slice_attrs(record, keys) + record.attributes.symbolize_keys.slice(*keys) + end +end diff --git a/enterprise/app/services/captain/assistant/session_capture_service.rb b/enterprise/app/services/captain/assistant/session_capture_service.rb new file mode 100644 index 000000000..36af50308 --- /dev/null +++ b/enterprise/app/services/captain/assistant/session_capture_service.rb @@ -0,0 +1,64 @@ +class Captain::Assistant::SessionCaptureService + SCENARIO_AGENT_REGEX = /\A#{Captain::Scenario::HANDOFF_KEY_PREFIX}_(\d+)_/ + + def initialize(assistant:, conversation:, run_result:, result_message:, credits_consumed:) + @assistant = assistant + @conversation = conversation + @run_result = run_result + @result_message = result_message + @credits_consumed = credits_consumed + end + + def capture + # TODO: Capture failed runs once error-session semantics are defined. For now, + # only successful runs that produce a customer-facing reply or handoff are recorded. + return unless @run_result&.success? + + capture! + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: @assistant.account).capture_exception + Rails.logger.error("[CAPTAIN][SessionCaptureService] Capture failed for conversation=#{@conversation.display_id}: #{e.message}") + end + + def capture! + model = @assistant.agent_model + metadata = context.dig(:state, :cw_metadata) || {} + + Captain::AgentSession.create!( + assistant: @assistant, + session_type: :assistant, + subject: @conversation, + result: @result_message, + llm_model: "#{Llm::Models.provider_for(model)}-#{model}", + credits_consumed: @credits_consumed, + faq_ids: metadata[:faq_ids] || [], + document_ids: metadata[:document_ids] || [], + scenario_ids: scenario_ids, + run_context: current_turn_history + ) + end + + private + + def context + @run_result.context || {} + end + + def scenario_ids + ids = current_turn_history.filter_map do |message| + next unless message[:role].to_s == 'assistant' + + message[:agent_name].to_s.match(SCENARIO_AGENT_REGEX)&.[](1)&.to_i + end.uniq + + ids & @assistant.scenarios.where(id: ids).pluck(:id) + end + + # Trim to the current turn: the last user message and everything after it + # (assistant replies, tool calls/results, handoff hops). + def current_turn_history + history = Array(context[:conversation_history]) + last_user_index = history.rindex { |message| message[:role].to_s == 'user' } + last_user_index ? history[last_user_index..] : history + end +end diff --git a/enterprise/lib/captain/tools/faq_lookup_tool.rb b/enterprise/lib/captain/tools/faq_lookup_tool.rb index 93dd90259..2a16e7e99 100644 --- a/enterprise/lib/captain/tools/faq_lookup_tool.rb +++ b/enterprise/lib/captain/tools/faq_lookup_tool.rb @@ -2,11 +2,12 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool description 'Search FAQ responses using semantic similarity to find relevant answers' param :query, type: 'string', desc: 'The question or topic to search for in the FAQ database' - def perform(_tool_context, query:) + def perform(tool_context, query:) log_tool_usage('searching', { query: query }) # Use existing vector search on approved responses responses = @assistant.responses.approved.search(query).to_a + record_retrieved_sources(tool_context, responses) if responses.empty? log_tool_usage('no_results', { query: query }) @@ -19,6 +20,16 @@ class Captain::Tools::FaqLookupTool < Captain::Tools::BasePublicTool private + def record_retrieved_sources(tool_context, responses) + return if responses.empty? + + metadata = tool_context.state[:cw_metadata] ||= {} + metadata[:faq_ids] = Array(metadata[:faq_ids]) | responses.map(&:id) + + document_ids = responses.filter_map { |response| response.documentable_id if response.documentable_type == 'Captain::Document' } + metadata[:document_ids] = Array(metadata[:document_ids]) | document_ids + end + def format_responses(responses) responses.map { |response| format_response(response) }.join end diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 266954d0f..eb1cb641c 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -22,6 +22,7 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do allow(mock_llm_chat_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain Specs' }) allow(Captain::Assistant::AgentRunnerService).to receive(:new).and_return(mock_agent_runner_service) allow(mock_agent_runner_service).to receive(:generate_response).and_return({ 'response' => 'Hey, welcome to Captain V2' }) + allow(mock_agent_runner_service).to receive(:last_run_result).and_return(nil) allow(Captain::Llm::AssistantActionClassifierService).to receive(:new).and_return(mock_action_classifier_service) allow(mock_action_classifier_service).to receive(:classify).and_return({ 'action' => 'continue' }) allow(Captain::Llm::AssistantFalsePromiseService).to receive(:new).and_return(mock_false_promise_service) @@ -72,6 +73,12 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1) end + it 'does not create a captain session' do + expect do + described_class.perform_now(conversation, assistant) + end.not_to change(Captain::AgentSession, :count) + end + it 'does not run the action classifier when the classifier feature is disabled' do expect(Captain::Llm::AssistantActionClassifierService).not_to receive(:new) @@ -490,6 +497,98 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do end end + context 'when capturing assistant sessions' do + let(:run_context) do + { + session_id: "#{account.id}_#{conversation.display_id}", + current_agent: 'Assistant', + turn_count: 1, + conversation_history: [ + { role: :user, content: 'Hello' }, + { role: :assistant, content: 'Hey, welcome to Captain V2', agent_name: 'Assistant' } + ], + state: { cw_metadata: { faq_ids: [7, 9], document_ids: [3] } } + } + end + let(:usage) do + Agents::RunContext::Usage.new.tap do |u| + u.input_tokens = 100 + u.output_tokens = 20 + u.total_tokens = 120 + end + end + let(:run_result) { Agents::RunResult.new(output: { 'response' => 'Hey, welcome to Captain V2' }, usage: usage, context: run_context) } + + before do + allow(account).to receive(:feature_enabled?).and_return(false) + allow(account).to receive(:feature_enabled?).with('captain_integration_v2').and_return(true) + allow(mock_agent_runner_service).to receive(:last_run_result).and_return(run_result) + end + + it 'creates a session for a delivered response' do + described_class.perform_now(conversation, assistant) + + session = Captain::AgentSession.last + expect(session).to have_attributes( + account_id: account.id, + assistant_id: assistant.id, + subject_id: conversation.id, + subject_type: 'Conversation', + result_id: conversation.messages.outgoing.last.id, + result_type: 'Message', + llm_model: 'openai-gpt-5.2', + credits_consumed: 1.0, + faq_ids: [7, 9], + document_ids: [3], + scenario_ids: [], + user_id: nil + ) + expect(session).to be_session_assistant + expect(session.run_context.first).to include('role' => 'user', 'content' => 'Hello') + end + + it 'creates a zero-credit session when the handoff tool fired' do + allow(mock_agent_runner_service).to receive(:generate_response) do + conversation.update!(status: :open) + { 'response' => 'Let me connect you', 'handoff_tool_called' => true } + end + + described_class.perform_now(conversation, assistant) + + session = Captain::AgentSession.last + expect(session.credits_consumed).to eq(0.0) + expect(session.result_id).to eq(conversation.messages.outgoing.where(private: false).last.id) + expect(account.reload.usage_limits[:captain][:responses][:consumed]).to eq(0) + end + + it 'creates a zero-credit session when the handoff tool fired but failed to commit' do + allow(mock_agent_runner_service).to receive(:generate_response).and_return({ + 'response' => 'I tried to hand off', + 'handoff_tool_called' => true + }) + + described_class.perform_now(conversation, assistant) + + session = Captain::AgentSession.last + expect(session.credits_consumed).to eq(0.0) + expect(session.result_id).to eq(conversation.messages.outgoing.where(private: false).last.id) + end + + it 'still delivers the reply when session capture fails' do + allow(Captain::AgentSession).to receive(:create!).and_raise(StandardError, 'capture failed') + allow(ChatwootExceptionTracker).to receive(:new).and_call_original + + expect do + described_class.perform_now(conversation, assistant) + end.not_to raise_error + + expect(conversation.messages.outgoing.count).to eq(1) + expect(conversation.messages.outgoing.last.content).to eq('Hey, welcome to Captain V2') + expect(conversation.reload.status).to eq('pending') + expect(ChatwootExceptionTracker).to have_received(:new) + end + end + # Regression (PR #13417): wrapping create_handoff_message and bot_handoff! in the # same transaction defers the message's after_create_commit until commit, at which # point it clears waiting_since (bot_response). The handoff path must stay outside diff --git a/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb b/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb index ccae44ac2..dc2b9f6c6 100644 --- a/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/faq_lookup_tool_spec.rb @@ -80,6 +80,21 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do tool.perform(tool_context, query: 'password reset') end + + it 'records retrieved faq ids and document ids into Chatwoot metadata' do + tool.perform(tool_context, query: 'password reset') + + expect(tool_context.state.dig(:cw_metadata, :faq_ids)).to contain_exactly(response1.id, response2.id) + expect(tool_context.state.dig(:cw_metadata, :document_ids)).to contain_exactly(document.id) + end + + it 'accumulates unique ids across multiple calls' do + tool.perform(tool_context, query: 'password reset') + tool.perform(tool_context, query: 'password reset again') + + expect(tool_context.state.dig(:cw_metadata, :faq_ids)).to contain_exactly(response1.id, response2.id) + expect(tool_context.state.dig(:cw_metadata, :document_ids)).to contain_exactly(document.id) + end end context 'when no FAQs found' do @@ -99,6 +114,12 @@ RSpec.describe Captain::Tools::FaqLookupTool, type: :model do tool.perform(tool_context, query: 'nonexistent topic') end + + it 'leaves shared state untouched' do + tool.perform(tool_context, query: 'nonexistent topic') + + expect(tool_context.state).to eq({}) + end end context 'with blank query' do diff --git a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb index 6fd8d50ab..e88d040b3 100644 --- a/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb +++ b/spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb @@ -170,6 +170,12 @@ RSpec.describe Captain::Assistant::AgentRunnerService do expect(result).to eq({ 'response' => 'Test response', 'agent_name' => nil, 'handoff_tool_called' => false }) end + it 'exposes the raw run result via last_run_result' do + service.generate_response(message_history: message_history) + + expect(service.last_run_result).to eq(mock_result) + end + context 'when handoff tool was called during agent execution' do let(:runner_context) { { captain_v2_handoff_tool_called: true } } let(:mock_result) do @@ -246,6 +252,12 @@ RSpec.describe Captain::Assistant::AgentRunnerService do service.generate_response(message_history: message_history) end + it 'leaves last_run_result nil' do + service.generate_response(message_history: message_history) + + expect(service.last_run_result).to be_nil + end + context 'when conversation is nil' do subject(:service) { described_class.new(assistant: assistant, conversation: nil) } diff --git a/spec/enterprise/services/captain/assistant/session_capture_service_spec.rb b/spec/enterprise/services/captain/assistant/session_capture_service_spec.rb new file mode 100644 index 000000000..ef25b1e2b --- /dev/null +++ b/spec/enterprise/services/captain/assistant/session_capture_service_spec.rb @@ -0,0 +1,167 @@ +require 'rails_helper' + +RSpec.describe Captain::Assistant::SessionCaptureService do + let(:account) { create(:account) } + let(:assistant) { create(:captain_assistant, account: account) } + let(:conversation) { create(:conversation, account: account) } + let(:result_message) { create(:message, account: account, conversation: conversation) } + + let(:usage) do + Agents::RunContext::Usage.new.tap do |u| + u.input_tokens = 120 + u.output_tokens = 40 + u.total_tokens = 160 + end + end + + let(:conversation_history) do + [ + { role: :user, content: 'Hi, my internet is not working' }, + { role: :assistant, content: 'Let me check', agent_name: 'Assistant' }, + { role: :user, content: 'CUST001' }, + { role: :assistant, content: '', agent_name: 'Assistant', tool_calls: [{ 'id' => 'call_1', 'name' => 'faq_lookup' }] }, + { role: :tool, content: 'Restart the modem', tool_call_id: 'call_1' }, + { role: :assistant, content: 'Please restart your modem', agent_name: 'Assistant' } + ] + end + + let(:run_context) do + { + session_id: "#{account.id}_#{conversation.display_id}", + current_agent: 'Assistant', + turn_count: 2, + conversation_history: conversation_history, + state: { cw_metadata: { faq_ids: [11, 12], document_ids: [5] } } + } + end + + let(:run_result) { Agents::RunResult.new(output: { 'response' => 'Please restart your modem' }, usage: usage, context: run_context) } + + let(:service) do + described_class.new( + assistant: assistant, + conversation: conversation, + run_result: run_result, + result_message: result_message, + credits_consumed: 1.0 + ) + end + + before do + allow(assistant).to receive(:agent_model).and_return('gpt-5.2') + end + + describe '#capture' do + it 'creates the session' do + expect { service.capture }.to change(Captain::AgentSession, :count).by(1) + end + + it 'does nothing when there is no run result' do + service = described_class.new( + assistant: assistant, conversation: conversation, run_result: nil, + result_message: result_message, credits_consumed: 1.0 + ) + + expect { service.capture }.not_to change(Captain::AgentSession, :count) + end + + it 'does nothing when the run failed' do + failed_result = Agents::RunResult.new(output: nil, error: StandardError.new('run failed'), context: run_context, usage: usage) + service = described_class.new( + assistant: assistant, conversation: conversation, run_result: failed_result, + result_message: nil, credits_consumed: 0.0 + ) + + expect { service.capture }.not_to change(Captain::AgentSession, :count) + end + + it 'reports failures without raising' do + allow(Captain::AgentSession).to receive(:create!).and_raise(StandardError, 'capture failed') + allow(ChatwootExceptionTracker).to receive(:new).and_call_original + + expect { service.capture }.not_to raise_error + expect(ChatwootExceptionTracker).to have_received(:new) + end + end + + describe '#capture!' do + it 'creates an assistant session with all attributes' do + session = service.capture! + + expect(session).to have_attributes( + account_id: account.id, + assistant_id: assistant.id, + subject_id: conversation.id, + subject_type: 'Conversation', + result_id: result_message.id, + result_type: 'Message', + llm_model: 'openai-gpt-5.2', + credits_consumed: 1.0, + faq_ids: [11, 12], + document_ids: [5], + scenario_ids: [], + user_id: nil + ) + expect(session).to be_session_assistant + end + + it 'stores the trimmed current turn in run_context' do + history = service.capture!.run_context + expect(history.size).to eq(4) + expect(history.first).to include('role' => 'user', 'content' => 'CUST001') + end + + it 'stores the full history when it contains no user message' do + run_context[:conversation_history] = conversation_history.reject { |message| message[:role] == :user } + + history = service.capture!.run_context + + expect(history.size).to eq(4) + end + + it 'handles a successful run result without context or usage' do + run_result = Agents::RunResult.new(output: { 'response' => 'Hello' }) + service = described_class.new( + assistant: assistant, conversation: conversation, run_result: run_result, + result_message: result_message, credits_consumed: 1.0 + ) + + session = service.capture! + + expect(session.result).to eq(result_message) + expect(session.faq_ids).to eq([]) + expect(session.document_ids).to eq([]) + expect(session.run_context).to eq([]) + end + + it 'extracts every scenario that authored a message in the current turn' do + first_scenario = create(:captain_scenario, assistant: assistant, account: account) + second_scenario = create(:captain_scenario, assistant: assistant, account: account) + run_context[:conversation_history] = [ + { role: :user, content: 'Help with my refund' }, + { role: :assistant, content: '', agent_name: first_scenario.handoff_key, tool_calls: [] }, + { role: :assistant, content: 'Checking', agent_name: second_scenario.handoff_key }, + { role: :assistant, content: 'Done', agent_name: first_scenario.handoff_key } + ] + run_context[:current_agent] = 'Assistant' + + expect(service.capture!.scenario_ids).to eq([first_scenario.id, second_scenario.id]) + end + + it 'leaves scenario ids empty for the primary assistant agent' do + run_context[:current_agent] = 'Assistant' + + expect(service.capture!.scenario_ids).to eq([]) + end + + it 'does not capture a scenario belonging to another assistant' do + scenario = create(:captain_scenario, assistant: create(:captain_assistant, account: account), account: account) + run_context[:conversation_history] = [ + { role: :user, content: 'Help' }, + { role: :assistant, content: 'No', agent_name: scenario.handoff_key } + ] + + expect(service.capture!.scenario_ids).to eq([]) + end + end +end