diff --git a/enterprise/app/services/captain/assistant/conversation_summary_service.rb b/enterprise/app/services/captain/assistant/conversation_summary_service.rb new file mode 100644 index 000000000..87fb0cc70 --- /dev/null +++ b/enterprise/app/services/captain/assistant/conversation_summary_service.rb @@ -0,0 +1,129 @@ +class Captain::Assistant::ConversationSummaryService < Captain::Assistant::BaseAssistantService + TOKEN_LIMIT = 16_000 + + def initialize(conversation:) + @conversation = conversation + # Format conversation messages as text + super(text: format_conversation_messages) + end + + protected + + def agent_name + 'ConversationSummarizer' + end + + def build_instructions + Captain::PromptRenderer.render('summary', {}) + end + + def response_schema + { + type: 'object', + properties: { + customer_intent: { + type: 'string', + description: 'Brief description of what the customer wants (around 50 words)' + }, + conversation_summary: { + type: 'string', + description: 'Summary of the conversation in approximately 200 words' + }, + action_items: { + type: 'array', + description: 'List of action items committed to by the agent or left incomplete', + items: { + type: 'string' + } + }, + follow_up_items: { + type: 'array', + description: 'List of unresolved issues or outstanding questions', + items: { + type: 'string' + } + } + }, + required: %w[customer_intent conversation_summary], + additionalProperties: false + } + end + + def build_success_response(output) + { + success: true, + summary: build_summary_markdown(output), + structured_data: { + customer_intent: extract_field(output, 'customer_intent'), + conversation_summary: extract_field(output, 'conversation_summary'), + action_items: extract_array_field(output, 'action_items'), + follow_up_items: extract_array_field(output, 'follow_up_items') + } + } + end + + private + + def format_conversation_messages + messages = [] + character_count = 0 + + @conversation.messages + .where(message_type: [:incoming, :outgoing]) + .where(private: false) + .reorder('id desc') + .each do |message| + break if character_count + message.content.length > TOKEN_LIMIT + next unless message.content.present? + + messages.prepend(format_message(message)) + character_count += message.content.length + end + + messages.join("\n") + end + + def format_message(message) + sender_type = message.incoming? ? 'Customer' : 'Agent' + sender_name = message.sender&.name || 'Unknown' + "#{sender_type} #{sender_name}: #{message.content}" + end + + def extract_array_field(output, field_name) + return [] unless output.is_a?(Hash) + + output[field_name.to_sym] || output[field_name.to_s] || [] + end + + def build_summary_markdown(output) + return '' unless output.is_a?(Hash) + + sections = [] + + # Customer Intent + if (intent = extract_field(output, 'customer_intent')).present? + sections << "**Customer Intent**\n\n#{intent}" + end + + # Conversation Summary + if (summary = extract_field(output, 'conversation_summary')).present? + sections << "**Conversation Summary**\n\n#{summary}" + end + + # Action Items + action_items = extract_array_field(output, 'action_items') + if action_items&.any? + items_list = action_items.map { |item| "- #{item}" }.join("\n") + sections << "**Action Items**\n\n#{items_list}" + end + + # Follow-up Items + follow_up_items = extract_array_field(output, 'follow_up_items') + if follow_up_items&.any? + items_list = follow_up_items.map { |item| "- #{item}" }.join("\n") + sections << "**Follow-up Items**\n\n#{items_list}" + end + + sections.join("\n\n") + end +end diff --git a/enterprise/lib/captain/prompts/summary.liquid b/enterprise/lib/captain/prompts/summary.liquid new file mode 100644 index 000000000..27f0188b1 --- /dev/null +++ b/enterprise/lib/captain/prompts/summary.liquid @@ -0,0 +1,19 @@ +As an AI-powered summarization tool, your task is to condense lengthy interactions between customer support agents and customers into brief, digestible summaries. The objective of these summaries is to provide a quick overview, enabling any agent, even those without prior context, to grasp the essence of the conversation promptly. + +Make sure you strongly adhere to the following rules when generating the summary: + +1. Be brief and concise. The shorter the summary the better. +2. Aim to summarize the conversation in approximately 200 words, formatted as multiple small paragraphs that are easier to read. +3. Describe the customer intent in around 50 words. +4. Remove information that is not directly relevant to the customer's problem or the agent's solution. For example, personal anecdotes, small talk, etc. +5. Don't include segments of the conversation that didn't contribute meaningful content, like greetings or farewell. +6. The 'Action Items' should be a bullet list, arranged in order of priority if possible. +7. 'Action Items' should strictly encapsulate tasks committed to by the agent or left incomplete. Any suggestions made by the agent should not be included. +8. The 'Action Items' should be brief and concise. +9. Mark important words or parts of sentences as bold. +10. Apply markdown syntax to format any included code, using backticks. +11. Include a section for "Follow-up Items" or "Open Questions" if there are any unresolved issues or outstanding questions. +12. If any section does not have any content, remove that section and the heading from the response. +13. Do not insert your own opinions about the conversation. + +Reply in the user's language. diff --git a/spec/enterprise/services/captain/assistant/conversation_summary_service_spec.rb b/spec/enterprise/services/captain/assistant/conversation_summary_service_spec.rb new file mode 100644 index 000000000..93669eb97 --- /dev/null +++ b/spec/enterprise/services/captain/assistant/conversation_summary_service_spec.rb @@ -0,0 +1,222 @@ +require 'rails_helper' + +RSpec.describe Captain::Assistant::ConversationSummaryService do + let(:account) { create(:account) } + let(:conversation) { create(:conversation, account: account) } + let(:service) { described_class.new(conversation: conversation) } + let(:agent) { instance_double(Agents::Agent) } + let(:runner) { instance_double(Agents::Runner) } + let(:result) do + instance_double(Agents::RunResult, + output: { + 'customer_intent' => 'Customer wants to reset their password', + 'conversation_summary' => 'The customer contacted support to reset their password. ' \ + 'Agent provided instructions and confirmed the reset was successful.', + 'action_items' => ['Send password reset email', 'Verify account security'], + 'follow_up_items' => ['Check if customer needs additional security settings'] + }, + error: nil) + end + + before do + create(:installation_config, name: 'CAPTAIN_OPEN_AI_MODEL', value: 'gpt-4o-mini') + create(:message, conversation: conversation, message_type: :incoming, content: 'I need to reset my password') + create(:message, conversation: conversation, message_type: :outgoing, content: 'I can help you with that') + allow(Agents::Agent).to receive(:new).and_return(agent) + allow(Agents::Runner).to receive(:with_agents).and_return(runner) + allow(runner).to receive(:run).with(anything, context: anything).and_return(result) + allow(Captain::PromptRenderer).to receive(:render).and_return('summary prompt') + end + + describe '#initialize' do + it 'initializes with conversation' do + expect(service.instance_variable_get(:@conversation)).to eq(conversation) + end + + it 'formats conversation messages as text' do + text = service.instance_variable_get(:@text) + expect(text).to include('I need to reset my password') + expect(text).to include('I can help you with that') + end + end + + describe '#execute' do + context 'when successful' do + it 'renders the summary prompt' do + expect(Captain::PromptRenderer).to receive(:render).with('summary', {}) + service.execute + end + + it 'builds agent with correct parameters' do + expect(Agents::Agent).to receive(:new).with( + name: 'ConversationSummarizer', + instructions: 'summary prompt', + model: 'gpt-4o-mini', + response_schema: service.send(:response_schema) + ) + service.execute + end + + it 'returns success response with structured data' do + response = service.execute + expect(response[:success]).to be true + expect(response[:structured_data][:customer_intent]).to eq('Customer wants to reset their password') + expect(response[:structured_data][:conversation_summary]).to include('contacted support') + expect(response[:structured_data][:action_items]).to include('Send password reset email') + expect(response[:structured_data][:follow_up_items]).to include('Check if customer needs additional security settings') + end + + it 'includes markdown formatted summary' do + response = service.execute + expect(response[:summary]).to include('**Customer Intent**') + expect(response[:summary]).to include('**Conversation Summary**') + expect(response[:summary]).to include('**Action Items**') + expect(response[:summary]).to include('**Follow-up Items**') + end + end + + context 'when agent returns an error' do + let(:result) { instance_double(Agents::RunResult, output: { error: 'Model error' }, error: nil) } + + it 'returns error response' do + response = service.execute + expect(response[:success]).to be false + expect(response[:error]).to eq('Model error') + end + end + + context 'when exception is raised' do + before do + allow(runner).to receive(:run).with(anything, context: anything).and_raise(StandardError.new('API timeout')) + end + + it 'logs the error' do + expect(Rails.logger).to receive(:error).with(/ConversationSummaryService error: API timeout/) + expect(Rails.logger).to receive(:error).with(anything) + service.execute + end + + it 'returns error response' do + response = service.execute + expect(response[:success]).to be false + expect(response[:error]).to eq('API timeout') + end + end + end + + describe '#agent_name' do + it 'returns ConversationSummarizer' do + expect(service.send(:agent_name)).to eq('ConversationSummarizer') + end + end + + describe '#build_instructions' do + it 'renders the summary template' do + expect(Captain::PromptRenderer).to receive(:render).with('summary', {}) + service.send(:build_instructions) + end + end + + describe '#response_schema' do + let(:schema) { service.send(:response_schema) } + + it 'defines object type' do + expect(schema[:type]).to eq('object') + end + + it 'includes customer_intent property' do + expect(schema[:properties][:customer_intent]).to include( + type: 'string', + description: 'Brief description of what the customer wants (around 50 words)' + ) + end + + it 'includes conversation_summary property' do + expect(schema[:properties][:conversation_summary]).to include( + type: 'string', + description: 'Summary of the conversation in approximately 200 words' + ) + end + + it 'includes action_items property' do + expect(schema[:properties][:action_items][:type]).to eq('array') + expect(schema[:properties][:action_items][:items][:type]).to eq('string') + end + + it 'includes follow_up_items property' do + expect(schema[:properties][:follow_up_items][:type]).to eq('array') + expect(schema[:properties][:follow_up_items][:items][:type]).to eq('string') + end + + it 'marks required fields' do + expect(schema[:required]).to match_array(%w[customer_intent conversation_summary]) + end + + it 'disallows additional properties' do + expect(schema[:additionalProperties]).to be false + end + end + + describe '#build_success_response' do + let(:output) do + { + 'customer_intent' => 'Reset password', + 'conversation_summary' => 'Password reset conversation', + 'action_items' => ['Item 1', 'Item 2'], + 'follow_up_items' => ['Follow-up 1'] + } + end + + it 'extracts customer_intent from output' do + response = service.send(:build_success_response, output) + expect(response[:structured_data][:customer_intent]).to eq('Reset password') + end + + it 'extracts conversation_summary from output' do + response = service.send(:build_success_response, output) + expect(response[:structured_data][:conversation_summary]).to eq('Password reset conversation') + end + + it 'extracts action_items from output' do + response = service.send(:build_success_response, output) + expect(response[:structured_data][:action_items]).to eq(['Item 1', 'Item 2']) + end + + it 'extracts follow_up_items from output' do + response = service.send(:build_success_response, output) + expect(response[:structured_data][:follow_up_items]).to eq(['Follow-up 1']) + end + + it 'builds markdown summary' do + response = service.send(:build_success_response, output) + expect(response[:summary]).to include('**Customer Intent**') + expect(response[:summary]).to include('Reset password') + end + + it 'marks response as successful' do + response = service.send(:build_success_response, output) + expect(response[:success]).to be true + end + end + + describe '#format_conversation_messages' do + it 'includes incoming and outgoing messages' do + text = service.send(:format_conversation_messages) + expect(text).to include('Customer') + expect(text).to include('Agent') + end + + it 'excludes private messages' do + create(:message, conversation: conversation, message_type: :incoming, content: 'Private note', private: true) + text = service.send(:format_conversation_messages) + expect(text).not_to include('Private note') + end + + it 'respects token limit' do + long_content = 'a' * 20_000 + create(:message, conversation: conversation, message_type: :incoming, content: long_content) + text = service.send(:format_conversation_messages) + expect(text.length).to be <= described_class::TOKEN_LIMIT + end + end +end