From 847bb13aa526ec498b4711f7e48362a85849cefa Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 29 Jun 2026 22:04:53 +0530 Subject: [PATCH] feat: add captain assistant overview stats and summary endpoints --- config/routes.rb | 2 + .../captain/assistant_stats_builder.rb | 140 ++++++++++++++++++ .../accounts/captain/assistants_controller.rb | 28 +++- .../app/policies/captain/assistant_policy.rb | 4 + lib/captain/overview_summary_service.rb | 64 ++++++++ .../captain_overview_summary.liquid | 21 +-- 6 files changed, 249 insertions(+), 10 deletions(-) create mode 100644 enterprise/app/builders/captain/assistant_stats_builder.rb create mode 100644 lib/captain/overview_summary_service.rb diff --git a/config/routes.rb b/config/routes.rb index 64e1df440..18431c3c7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -66,6 +66,8 @@ Rails.application.routes.draw do resources :assistants do member do post :playground + get :stats + get :summary end collection do get :tools diff --git a/enterprise/app/builders/captain/assistant_stats_builder.rb b/enterprise/app/builders/captain/assistant_stats_builder.rb new file mode 100644 index 000000000..af5ca9154 --- /dev/null +++ b/enterprise/app/builders/captain/assistant_stats_builder.rb @@ -0,0 +1,140 @@ +# Computes per-assistant overview metrics for the Captain Overview page. +# Each metric is returned for the current window and the previous equal-length +# window, plus a derived trend. See stats.md (repo root) for the metric +# definitions and the source-of-truth queries this mirrors. +class Captain::AssistantStatsBuilder + RESOLVED_EVENT_NAMES = %w[conversation_captain_inference_resolved conversation_bot_resolved].freeze + HANDOFF_EVENT_NAMES = %w[conversation_captain_inference_handoff conversation_bot_handoff].freeze + DEFAULT_RANGE_DAYS = 30 + + attr_reader :assistant, :account, :range_days + + def initialize(assistant, range_days = DEFAULT_RANGE_DAYS) + @assistant = assistant + @account = assistant.account + @range_days = range_days.to_i.positive? ? range_days.to_i : DEFAULT_RANGE_DAYS + end + + def metrics + current = window_metrics(current_range) + previous = window_metrics(previous_range) + + { + conversations_handled: pack(current[:handled], previous[:handled], :percent), + auto_resolution_rate: pack(current[:auto_resolution], previous[:auto_resolution], :point), + handoff_rate: pack(current[:handoff], previous[:handoff], :point), + hours_saved: pack(current[:hours_saved], previous[:hours_saved], :percent), + reopen_rate: pack(current[:reopen], previous[:reopen], :point), + conversation_depth: pack(current[:depth], previous[:depth], :absolute), + knowledge: knowledge + } + end + + private + + def current_range + @current_range ||= (range_days.days.ago)..Time.current + end + + def previous_range + @previous_range ||= ((2 * range_days).days.ago)..range_days.days.ago + end + + # Raw metric values for a single window. + def window_metrics(range) + handled = handled_scope(range).distinct.count(:conversation_id) + public_messages = public_outgoing_scope(range) + public_count = public_messages.count + depth_conversations = public_messages.distinct.count(:conversation_id) + + { + handled: handled, + auto_resolution: rate(resolved_count(range), handled), + handoff: rate(handoff_count(range), handled), + hours_saved: (public_count * avg_reply_time(range) / 3600.0).round, + reopen: reopen_rate(range), + depth: depth_conversations.zero? ? 0 : (public_count.to_f / depth_conversations).round(1) + } + end + + # Conversations the assistant participated in (authored any message). + def handled_scope(range) + account.messages.where(sender_type: 'Captain::Assistant', sender_id: assistant.id, created_at: range) + end + + # Public outgoing replies the assistant sent (excludes private notes / handoff activity). + def public_outgoing_scope(range) + handled_scope(range).where(message_type: :outgoing, private: false) + end + + def resolved_count(range) + distinct_event_conversations(RESOLVED_EVENT_NAMES, handled_scope(range)) + end + + def handoff_count(range) + distinct_event_conversations(HANDOFF_EVENT_NAMES, handled_scope(range)) + end + + def distinct_event_conversations(names, handled) + account.reporting_events + .where(name: names, conversation_id: handled.select(:conversation_id)) + .distinct.count(:conversation_id) + end + + # Of the conversations Captain auto-resolved (inbox-based), the share reopened afterwards. + def reopen_rate(range) + resolved_conversation_ids = account.reporting_events + .where(name: 'conversation_captain_inference_resolved', + inbox_id: assistant_inbox_ids, created_at: range) + .select(:conversation_id) + resolved = account.reporting_events.where(name: 'conversation_captain_inference_resolved', + inbox_id: assistant_inbox_ids, created_at: range) + .distinct.count(:conversation_id) + reopened = account.reporting_events + .where(name: 'conversation_opened', conversation_id: resolved_conversation_ids) + .where('reporting_events.value > 0') + .distinct.count(:conversation_id) + rate(reopened, resolved) + end + + def avg_reply_time(range) + account.reporting_events.where(name: 'reply_time', created_at: range).average(:value).to_f + end + + def assistant_inbox_ids + @assistant_inbox_ids ||= assistant.inboxes.ids + end + + def knowledge + responses = Captain::AssistantResponse.by_assistant(assistant.id) + approved = responses.approved.count + pending = responses.pending.count + total = approved + pending + + { + approved: approved, + pending: pending, + documents: Captain::Document.for_assistant(assistant.id).count, + coverage: total.zero? ? 0 : (approved.to_f / total * 100).round + } + end + + def rate(numerator, denominator) + return 0 if denominator.zero? + + (numerator.to_f / denominator * 100).round(1) + end + + def pack(current, previous, mode) + { current: current, previous: previous, trend: trend(current, previous, mode) } + end + + def trend(current, previous, mode) + case mode + when :percent + previous.zero? ? 0 : ((current - previous).to_f / previous * 100).round(1) + else # :point and :absolute are both current - previous + (current - previous).round(1) + end + end +end diff --git a/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb b/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb index df9dfe5cc..b205edbc2 100644 --- a/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb +++ b/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb @@ -2,7 +2,7 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base before_action :current_account before_action -> { check_authorization(Captain::Assistant) } - before_action :set_assistant, only: [:show, :update, :destroy, :playground] + before_action :set_assistant, only: [:show, :update, :destroy, :playground, :stats, :summary] def index @assistants = account_assistants.ordered @@ -43,8 +43,34 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base @tools = assistant.available_agent_tools end + def stats + render json: Captain::AssistantStatsBuilder.new(@assistant, params[:range]).metrics + end + + def summary + result = Rails.cache.fetch(summary_cache_key, expires_in: 1.day) do + stats = Captain::AssistantStatsBuilder.new(@assistant, params[:range]).metrics + Captain::OverviewSummaryService.new( + account: Current.account, + assistant: @assistant, + first_name: Current.user.name.to_s.split.first, + stats: stats + ).perform + end + + if result[:error] + render json: { error: result[:error] }, status: :unprocessable_content + else + render json: { message: result[:message] } + end + end + private + def summary_cache_key + "captain_overview_summary/#{@assistant.id}/#{params[:range]}/#{Date.current}" + end + def set_assistant @assistant = account_assistants.find(params[:id]) end diff --git a/enterprise/app/policies/captain/assistant_policy.rb b/enterprise/app/policies/captain/assistant_policy.rb index bbde3ffb0..9009a4952 100644 --- a/enterprise/app/policies/captain/assistant_policy.rb +++ b/enterprise/app/policies/captain/assistant_policy.rb @@ -11,6 +11,10 @@ class Captain::AssistantPolicy < ApplicationPolicy true end + def summary? + true + end + def tools? @account_user.administrator? end diff --git a/lib/captain/overview_summary_service.rb b/lib/captain/overview_summary_service.rb new file mode 100644 index 000000000..d7ad76dfb --- /dev/null +++ b/lib/captain/overview_summary_service.rb @@ -0,0 +1,64 @@ +# Generates the LLM welcome summary for the Captain Overview page from the +# assistant's stats hash (see Captain::AssistantStatsBuilder). Renders the +# captain_overview_summary.liquid prompt and returns markdown. +class Captain::OverviewSummaryService < Captain::BaseTaskService + pattr_initialize [:account!, :assistant!, :first_name!, :stats!] + + def perform + api_response = make_api_call( + feature: 'editor', + messages: [ + { role: 'system', content: system_prompt }, + { role: 'user', content: 'Write the summary.' } + ] + ) + + return api_response if api_response[:error] + + { message: api_response[:message] } + end + + private + + def system_prompt + Liquid::Template.parse(prompt_from_file('captain_overview_summary')).render(prompt_variables) + end + + def prompt_variables + { + 'first_name' => first_name.to_s, + 'assistant_name' => assistant.name.to_s, + 'conversations_handled' => current(:conversations_handled), + 'hours_saved' => current(:hours_saved), + 'auto_resolution_rate' => current(:auto_resolution_rate), + 'auto_resolution_trend' => trend(:auto_resolution_rate), + 'handoff_rate' => current(:handoff_rate), + 'handoff_trend' => trend(:handoff_rate), + 'reopen_rate' => current(:reopen_rate), + 'reopen_trend' => trend(:reopen_rate), + 'knowledge_coverage' => stats.dig(:knowledge, :coverage).to_s + } + end + + def current(key) + stats.dig(key, :current).to_s + end + + def trend(key) + stats.dig(key, :trend).to_s + end + + def event_name + 'captain_overview_summary' + end + + def use_account_openai_hook? + true + end + + # The overview summary is an internal analytics readout, not a customer-facing + # response, so it should not consume or be blocked by the captain_responses quota. + def counts_toward_usage? + false + end +end diff --git a/lib/integrations/openai/openai_prompts/captain_overview_summary.liquid b/lib/integrations/openai/openai_prompts/captain_overview_summary.liquid index 22740c35f..54df87d44 100644 --- a/lib/integrations/openai/openai_prompts/captain_overview_summary.liquid +++ b/lib/integrations/openai/openai_prompts/captain_overview_summary.liquid @@ -1,21 +1,24 @@ -You are Captain, the AI support assistant built into Chatwoot. Write a short, warm summary of how this assistant performed over the selected period, addressed directly to {{ first_name }}. +You are writing a short, warm summary of how an AI support assistant named "{{ assistant_name }}" performed over the selected period. The assistant is built into Chatwoot. Address the summary directly to {{ first_name }}, the person who manages it. + +Always refer to the assistant by its name, {{ assistant_name }}. Do not call it "Captain", "the assistant", or "your assistant". For example: "Hey {{ first_name }}, {{ assistant_name }} handled ...". + +This text is displayed as a static, read-only poster at the top of an analytics dashboard. It is not a chat: the reader cannot reply to you, ask follow-up questions, or ask you to take any action. Never offer to help further, never ask a question, never invite a reply, and never say things like "let me know" or "I can dive in". Just state the summary. Here are the assistant's stats for the period: - Conversations handled: {{ conversations_handled }} -- Hours saved for the team: {{ hours_saved }} -- Auto-resolution rate: {{ auto_resolution_rate }} ({{ auto_resolution_trend }} vs previous period) -- Handoff rate: {{ handoff_rate }} ({{ handoff_trend }} vs previous period) -- Reopen-after-resolve rate: {{ reopen_rate }} ({{ reopen_trend }} vs previous period) -- Knowledge base coverage: {{ knowledge_coverage }} -- Flagged response rate: {{ flagged_rate }} -- Credits used: {{ credits_used_pct }} of the monthly allowance +- Hours saved for the team: {{ hours_saved }} hours +- Auto-resolution rate: {{ auto_resolution_rate }}% ({{ auto_resolution_trend }} percentage points vs previous period) +- Handoff rate: {{ handoff_rate }}% ({{ handoff_trend }} percentage points vs previous period) +- Reopen-after-resolve rate: {{ reopen_rate }}% ({{ reopen_trend }} percentage points vs previous period) +- Knowledge base coverage: {{ knowledge_coverage }}% Rules: - Write 2 to 4 sentences in one short paragraph. Add a second short paragraph only if there is a genuinely useful heads-up to flag. - Open with "Hey {{ first_name }},". Be encouraging and conversational, never robotic. +- Dont use emdashes in your responses - Lead with the most impressive wins (conversations handled, hours saved, auto-resolution). Mention a trend only when it is meaningful. -- Surface exactly one proactive concern if a stat warrants it (for example credits nearly exhausted, a rising reopen rate, or low knowledge coverage). Skip it entirely when everything looks healthy. Phrase it as a friendly nudge, not an alarm. +- Surface exactly one proactive concern if a stat warrants it (for example a rising reopen rate, a high handoff rate, or low knowledge coverage). Skip it entirely when everything looks healthy. Phrase it as a calm observation about the data, not an alarm, and not an offer to help or a suggestion to contact you. - Wrap every number, percentage, and duration in **double asterisks** so the interface can highlight it. Bold only the figures, never whole phrases. - Output plain markdown only. No headings, no lists, no preamble, no closing sign-off.