fix: captain hours saved metric (#14948)

Reworks the Captain assistant "hours saved" metric so it produces a
believable number. It previously multiplied assistant reply count by
customer reply-wait time, which inflated the figure into the millions of
hours. It now estimates saved time as replies × a fixed ~2 min per-reply
effort assumption. On the overview card the value renders in days once
it passes 100 hours, and the label/hint copy was updated to match.

## How to test
1. Open the Captain assistant overview page.
2. Check the "Time saved" card shows a sensible value (hours, or days
past 100h).
3. Hover the hint and confirm it reflects the per-reply estimate.
This commit is contained in:
Shivam Mishra
2026-07-07 19:16:24 +05:30
committed by GitHub
parent a5fcecb3f6
commit 29d0b92f1c
4 changed files with 23 additions and 23 deletions
@@ -427,8 +427,8 @@
"HINT": "Share of handled conversations escalated to a human agent."
},
"HOURS_SAVED": {
"LABEL": "Hours saved",
"HINT": "Estimate: Captain replies times the team's average response time. Directional, not measured labor."
"LABEL": "Time saved",
"HINT": "Estimate: Captain replies times ~2 minutes of assumed agent effort per reply. Directional, not measured labor."
},
"REOPEN": {
"LABEL": "Reopen rate",
@@ -49,6 +49,11 @@ const resolveTrendGood = (trendValue, direction) => {
// :point, and a plain number for :absolute counts like conversation depth.
const TREND_SUFFIX = { percent: '%', point: ' pts', absolute: '' };
// Hours-saved is reported in hours, but large values read better as days. Past
// 100h we switch the unit so the card stays legible.
const formatDuration = hours =>
hours >= 100 ? `${Math.round(hours / 24)}d` : `${hours}h`;
const metricFor = (statKey, formatValue, direction, trendKind = 'percent') => {
const data = stats.value?.[statKey];
if (!data) return { value: '—', trend: '', trendGood: null };
@@ -84,7 +89,7 @@ const metrics = computed(() => [
key: 'hoursSaved',
label: t('CAPTAIN.OVERVIEW.METRICS.HOURS_SAVED.LABEL'),
hint: t('CAPTAIN.OVERVIEW.METRICS.HOURS_SAVED.HINT'),
...metricFor('hours_saved', v => `${v}h`, 'up'),
...metricFor('hours_saved', formatDuration, 'up'),
},
{
key: 'reopen',
@@ -121,7 +126,7 @@ const metrics = computed(() => [
<CaptainPaywall />
</template>
<template #body>
<div class="flex flex-col gap-6">
<div class="flex flex-col gap-6 pb-8">
<InboxBanner />
<CoverageBanner :knowledge="stats?.knowledge" />
@@ -3,13 +3,18 @@
# window, plus a derived trend.
#
# Queries are batched to cut round trips: the message-derived counts (handled,
# public replies, depth) and the average reply time are each computed for both
# windows in a single scan via conditional FILTER aggregation.
# public replies, depth) are computed for both windows in a single scan via
# conditional FILTER aggregation.
class Captain::AssistantStatsBuilder
RESOLVED_EVENT_NAMES = %w[conversation_captain_inference_resolved conversation_bot_resolved].freeze
HANDOFF_EVENT_NAMES = %w[conversation_captain_inference_handoff conversation_bot_handoff].freeze
BOT_RESOLVED_EVENT_NAME = 'conversation_bot_resolved'.freeze
# Assumed agent effort displaced by each public assistant reply. Reporting data
# only captures reply latency (customer wait time), not handling effort, so hours
# saved is a count-times-assumed-effort estimate rather than a measured duration.
SECONDS_SAVED_PER_REPLY = 2.minutes.to_i
attr_reader :assistant, :account
delegate :range, :period, to: :window
@@ -26,9 +31,8 @@ class Captain::AssistantStatsBuilder
def metrics
messages = message_window_metrics
reply_times = avg_reply_times
current = window_metrics(current_range, messages[:current], reply_times[:current])
previous = window_metrics(previous_range, messages[:previous], reply_times[:previous])
current = window_metrics(current_range, messages[:current])
previous = window_metrics(previous_range, messages[:previous])
build_metrics(current, previous)
end
@@ -57,8 +61,8 @@ class Captain::AssistantStatsBuilder
}
end
# Combines the per-window message counts and reply time with the reporting-event metrics for one window.
def window_metrics(range, message_counts, avg_reply)
# Combines the per-window message counts with the reporting-event metrics for one window.
def window_metrics(range, message_counts)
handled = message_counts[:handled]
public_count = message_counts[:public_count]
depth_conversations = message_counts[:depth_conversations]
@@ -68,7 +72,7 @@ class Captain::AssistantStatsBuilder
handled: handled,
auto_resolution: rate(resolution[:resolved], handled),
handoff: rate(resolution[:handoff], handled),
hours_saved: (public_count * avg_reply / 3600.0).round,
hours_saved: (public_count * SECONDS_SAVED_PER_REPLY / 3600.0).round,
reopen: reopen_rate(range),
depth: depth_conversations.zero? ? 0 : (public_count.to_f / depth_conversations).round(1)
}
@@ -96,15 +100,6 @@ class Captain::AssistantStatsBuilder
}
end
# Average reply time (seconds) for both windows in one scan.
def avg_reply_times
row = account.reporting_events.where(name: 'reply_time', created_at: full_span).reorder(nil).pick(
Arel.sql("AVG(value) FILTER (WHERE #{window_clause(current_range)})"),
Arel.sql("AVG(value) FILTER (WHERE #{window_clause(previous_range)})")
)
{ current: row[0].to_f, previous: row[1].to_f }
end
# Resolved and handed-off conversation counts for one window, in a single scan
# of the handled set's reporting events.
def resolution_counts(range)
@@ -90,9 +90,9 @@ class Seeders::Reports::AssistantConversationCreator
return unless rand < 0.6
travel(rand((1.minute)..(10.minutes)))
incoming_message(conversation)
follow_up = incoming_message(conversation)
travel(rand((20.seconds)..(5.minutes)))
assistant_reply(conversation, waiting_since: Time.current)
assistant_reply(conversation, waiting_since: follow_up.created_at)
end
def apply_outcome(conversation, created_at, outcome)