Records a `Captain::Session` row for every Captain V2 assistant response delivered in a conversation, so we can show how a response was generated and report on credit, FAQ, and document usage. Stacked on #14970 (the `captain_sessions` model). ## What changed - `FaqLookupTool` now records the retrieved FAQ ids (and their backing document ids) into the shared run state, accumulated across tool calls. - `AgentRunnerService` exposes the raw ai-agents run result via `last_run_result`; the `generate_response` return shape is unchanged, so the playground path is unaffected. - New `Captain::Assistant::SessionCaptureService` builds the session: scenario resolved from the answering agent name, model from `assistant.agent_model`, token usage plus the trimmed current-turn conversation history stored in `run_context`. - `ResponseBuilderJob` captures after delivery: `credits_consumed` mirrors the actual charge (1.0 for a billed response, 0.0 for handoffs, where the session points at the customer-facing handoff message). Capture runs outside the delivery transaction and swallows its own failures, so a logging bug can never block or roll back a customer reply. V1 responses and copilot are out of scope; copilot capture comes next. ## How to test On an account with `captain_integration_v2` enabled and an inbox connected to an assistant with approved FAQs, send a customer message on a pending conversation. After the assistant replies, a `Captain::Session` row should exist with the conversation as subject, the reply message as result, the FAQs/documents used, and the run context for that turn. Asking for a human agent should produce a zero-credit session pointing at the handoff message. <img width="2428" height="1058" alt="CleanShot 2026-07-15 at 17 25 40@2x" src="https://github.com/user-attachments/assets/d8e44923-c17b-494f-8c33-c8fa4219438c" />
142 lines
5.4 KiB
Ruby
142 lines
5.4 KiB
Ruby
require 'rails_helper'
|
|
|
|
RSpec.describe Captain::Tools::FaqLookupTool, type: :model do
|
|
let(:account) { create(:account) }
|
|
let(:assistant) { create(:captain_assistant, account: account) }
|
|
let(:tool) { described_class.new(assistant) }
|
|
let(:tool_context) { Struct.new(:state).new({}) }
|
|
|
|
before do
|
|
# Create installation config for OpenAI API key to avoid errors
|
|
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
|
|
|
|
# Mock embedding service to avoid actual API calls
|
|
embedding_service = instance_double(Captain::Llm::EmbeddingService)
|
|
allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
|
|
allow(embedding_service).to receive(:get_embedding).and_return(Array.new(1536, 0.1))
|
|
end
|
|
|
|
describe '#description' do
|
|
it 'returns the correct description' do
|
|
expect(tool.description).to eq('Search FAQ responses using semantic similarity to find relevant answers')
|
|
end
|
|
end
|
|
|
|
describe '#parameters' do
|
|
it 'returns the correct parameters' do
|
|
expect(tool.parameters).to have_key(:query)
|
|
expect(tool.parameters[:query].name).to eq(:query)
|
|
expect(tool.parameters[:query].type).to eq('string')
|
|
expect(tool.parameters[:query].description).to eq('The question or topic to search for in the FAQ database')
|
|
end
|
|
end
|
|
|
|
describe '#perform' do
|
|
context 'when FAQs exist' do
|
|
let(:document) { create(:captain_document, assistant: assistant) }
|
|
let!(:response1) do
|
|
create(:captain_assistant_response,
|
|
assistant: assistant,
|
|
question: 'How to reset password?',
|
|
answer: 'Click on forgot password link',
|
|
documentable: document,
|
|
status: 'approved')
|
|
end
|
|
let!(:response2) do
|
|
create(:captain_assistant_response,
|
|
assistant: assistant,
|
|
question: 'How to change email?',
|
|
answer: 'Go to settings and update email',
|
|
status: 'approved')
|
|
end
|
|
|
|
before do
|
|
# Mock nearest_neighbors to return our test responses
|
|
allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(
|
|
Captain::AssistantResponse.where(id: [response1.id, response2.id])
|
|
)
|
|
end
|
|
|
|
it 'searches FAQs and returns formatted responses' do
|
|
result = tool.perform(tool_context, query: 'password reset')
|
|
|
|
expect(result).to include('Question: How to reset password?')
|
|
expect(result).to include('Answer: Click on forgot password link')
|
|
expect(result).to include('Question: How to change email?')
|
|
expect(result).to include('Answer: Go to settings and update email')
|
|
end
|
|
|
|
it 'includes source link when document has external_link' do
|
|
document.update!(external_link: 'https://help.example.com/password')
|
|
|
|
result = tool.perform(tool_context, query: 'password')
|
|
|
|
expect(result).to include('Source: https://help.example.com/password')
|
|
end
|
|
|
|
it 'logs tool usage for search' do
|
|
expect(tool).to receive(:log_tool_usage).with('searching', { query: 'password reset' })
|
|
expect(tool).to receive(:log_tool_usage).with('found_results', { query: 'password reset', count: 2 })
|
|
|
|
tool.perform(tool_context, query: 'password reset')
|
|
end
|
|
|
|
it 'records retrieved faq ids and document ids into Chatwoot metadata' do
|
|
tool.perform(tool_context, query: 'password reset')
|
|
|
|
expect(tool_context.state.dig(:cw_metadata, :faq_ids)).to contain_exactly(response1.id, response2.id)
|
|
expect(tool_context.state.dig(:cw_metadata, :document_ids)).to contain_exactly(document.id)
|
|
end
|
|
|
|
it 'accumulates unique ids across multiple calls' do
|
|
tool.perform(tool_context, query: 'password reset')
|
|
tool.perform(tool_context, query: 'password reset again')
|
|
|
|
expect(tool_context.state.dig(:cw_metadata, :faq_ids)).to contain_exactly(response1.id, response2.id)
|
|
expect(tool_context.state.dig(:cw_metadata, :document_ids)).to contain_exactly(document.id)
|
|
end
|
|
end
|
|
|
|
context 'when no FAQs found' do
|
|
before do
|
|
# Return empty result set
|
|
allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
|
|
end
|
|
|
|
it 'returns no results message' do
|
|
result = tool.perform(tool_context, query: 'nonexistent topic')
|
|
expect(result).to eq('No relevant FAQs found for: nonexistent topic')
|
|
end
|
|
|
|
it 'logs tool usage for no results' do
|
|
expect(tool).to receive(:log_tool_usage).with('searching', { query: 'nonexistent topic' })
|
|
expect(tool).to receive(:log_tool_usage).with('no_results', { query: 'nonexistent topic' })
|
|
|
|
tool.perform(tool_context, query: 'nonexistent topic')
|
|
end
|
|
|
|
it 'leaves shared state untouched' do
|
|
tool.perform(tool_context, query: 'nonexistent topic')
|
|
|
|
expect(tool_context.state).to eq({})
|
|
end
|
|
end
|
|
|
|
context 'with blank query' do
|
|
it 'handles empty query' do
|
|
# Return empty result set
|
|
allow(Captain::AssistantResponse).to receive(:nearest_neighbors).and_return(Captain::AssistantResponse.none)
|
|
|
|
result = tool.perform(tool_context, query: '')
|
|
expect(result).to eq('No relevant FAQs found for: ')
|
|
end
|
|
end
|
|
end
|
|
|
|
describe '#active?' do
|
|
it 'returns true for public tools' do
|
|
expect(tool.active?).to be true
|
|
end
|
|
end
|
|
end
|