fix: captain hours saved metric (#14948)
Reworks the Captain assistant "hours saved" metric so it produces a believable number. It previously multiplied assistant reply count by customer reply-wait time, which inflated the figure into the millions of hours. It now estimates saved time as replies × a fixed ~2 min per-reply effort assumption. On the overview card the value renders in days once it passes 100 hours, and the label/hint copy was updated to match. ## How to test 1. Open the Captain assistant overview page. 2. Check the "Time saved" card shows a sensible value (hours, or days past 100h). 3. Hover the hint and confirm it reflects the per-reply estimate.
This commit is contained in:
@@ -427,8 +427,8 @@
|
||||
"HINT": "Share of handled conversations escalated to a human agent."
|
||||
},
|
||||
"HOURS_SAVED": {
|
||||
"LABEL": "Hours saved",
|
||||
"HINT": "Estimate: Captain replies times the team's average response time. Directional, not measured labor."
|
||||
"LABEL": "Time saved",
|
||||
"HINT": "Estimate: Captain replies times ~2 minutes of assumed agent effort per reply. Directional, not measured labor."
|
||||
},
|
||||
"REOPEN": {
|
||||
"LABEL": "Reopen rate",
|
||||
|
||||
@@ -49,6 +49,11 @@ const resolveTrendGood = (trendValue, direction) => {
|
||||
// :point, and a plain number for :absolute counts like conversation depth.
|
||||
const TREND_SUFFIX = { percent: '%', point: ' pts', absolute: '' };
|
||||
|
||||
// Hours-saved is reported in hours, but large values read better as days. Past
|
||||
// 100h we switch the unit so the card stays legible.
|
||||
const formatDuration = hours =>
|
||||
hours >= 100 ? `${Math.round(hours / 24)}d` : `${hours}h`;
|
||||
|
||||
const metricFor = (statKey, formatValue, direction, trendKind = 'percent') => {
|
||||
const data = stats.value?.[statKey];
|
||||
if (!data) return { value: '—', trend: '', trendGood: null };
|
||||
@@ -84,7 +89,7 @@ const metrics = computed(() => [
|
||||
key: 'hoursSaved',
|
||||
label: t('CAPTAIN.OVERVIEW.METRICS.HOURS_SAVED.LABEL'),
|
||||
hint: t('CAPTAIN.OVERVIEW.METRICS.HOURS_SAVED.HINT'),
|
||||
...metricFor('hours_saved', v => `${v}h`, 'up'),
|
||||
...metricFor('hours_saved', formatDuration, 'up'),
|
||||
},
|
||||
{
|
||||
key: 'reopen',
|
||||
@@ -121,7 +126,7 @@ const metrics = computed(() => [
|
||||
<CaptainPaywall />
|
||||
</template>
|
||||
<template #body>
|
||||
<div class="flex flex-col gap-6">
|
||||
<div class="flex flex-col gap-6 pb-8">
|
||||
<InboxBanner />
|
||||
|
||||
<CoverageBanner :knowledge="stats?.knowledge" />
|
||||
|
||||
@@ -3,13 +3,18 @@
|
||||
# window, plus a derived trend.
|
||||
#
|
||||
# Queries are batched to cut round trips: the message-derived counts (handled,
|
||||
# public replies, depth) and the average reply time are each computed for both
|
||||
# windows in a single scan via conditional FILTER aggregation.
|
||||
# public replies, depth) are computed for both windows in a single scan via
|
||||
# conditional FILTER aggregation.
|
||||
class Captain::AssistantStatsBuilder
|
||||
RESOLVED_EVENT_NAMES = %w[conversation_captain_inference_resolved conversation_bot_resolved].freeze
|
||||
HANDOFF_EVENT_NAMES = %w[conversation_captain_inference_handoff conversation_bot_handoff].freeze
|
||||
BOT_RESOLVED_EVENT_NAME = 'conversation_bot_resolved'.freeze
|
||||
|
||||
# Assumed agent effort displaced by each public assistant reply. Reporting data
|
||||
# only captures reply latency (customer wait time), not handling effort, so hours
|
||||
# saved is a count-times-assumed-effort estimate rather than a measured duration.
|
||||
SECONDS_SAVED_PER_REPLY = 2.minutes.to_i
|
||||
|
||||
attr_reader :assistant, :account
|
||||
|
||||
delegate :range, :period, to: :window
|
||||
@@ -26,9 +31,8 @@ class Captain::AssistantStatsBuilder
|
||||
|
||||
def metrics
|
||||
messages = message_window_metrics
|
||||
reply_times = avg_reply_times
|
||||
current = window_metrics(current_range, messages[:current], reply_times[:current])
|
||||
previous = window_metrics(previous_range, messages[:previous], reply_times[:previous])
|
||||
current = window_metrics(current_range, messages[:current])
|
||||
previous = window_metrics(previous_range, messages[:previous])
|
||||
|
||||
build_metrics(current, previous)
|
||||
end
|
||||
@@ -57,8 +61,8 @@ class Captain::AssistantStatsBuilder
|
||||
}
|
||||
end
|
||||
|
||||
# Combines the per-window message counts and reply time with the reporting-event metrics for one window.
|
||||
def window_metrics(range, message_counts, avg_reply)
|
||||
# Combines the per-window message counts with the reporting-event metrics for one window.
|
||||
def window_metrics(range, message_counts)
|
||||
handled = message_counts[:handled]
|
||||
public_count = message_counts[:public_count]
|
||||
depth_conversations = message_counts[:depth_conversations]
|
||||
@@ -68,7 +72,7 @@ class Captain::AssistantStatsBuilder
|
||||
handled: handled,
|
||||
auto_resolution: rate(resolution[:resolved], handled),
|
||||
handoff: rate(resolution[:handoff], handled),
|
||||
hours_saved: (public_count * avg_reply / 3600.0).round,
|
||||
hours_saved: (public_count * SECONDS_SAVED_PER_REPLY / 3600.0).round,
|
||||
reopen: reopen_rate(range),
|
||||
depth: depth_conversations.zero? ? 0 : (public_count.to_f / depth_conversations).round(1)
|
||||
}
|
||||
@@ -96,15 +100,6 @@ class Captain::AssistantStatsBuilder
|
||||
}
|
||||
end
|
||||
|
||||
# Average reply time (seconds) for both windows in one scan.
|
||||
def avg_reply_times
|
||||
row = account.reporting_events.where(name: 'reply_time', created_at: full_span).reorder(nil).pick(
|
||||
Arel.sql("AVG(value) FILTER (WHERE #{window_clause(current_range)})"),
|
||||
Arel.sql("AVG(value) FILTER (WHERE #{window_clause(previous_range)})")
|
||||
)
|
||||
{ current: row[0].to_f, previous: row[1].to_f }
|
||||
end
|
||||
|
||||
# Resolved and handed-off conversation counts for one window, in a single scan
|
||||
# of the handled set's reporting events.
|
||||
def resolution_counts(range)
|
||||
|
||||
@@ -90,9 +90,9 @@ class Seeders::Reports::AssistantConversationCreator
|
||||
return unless rand < 0.6
|
||||
|
||||
travel(rand((1.minute)..(10.minutes)))
|
||||
incoming_message(conversation)
|
||||
follow_up = incoming_message(conversation)
|
||||
travel(rand((20.seconds)..(5.minutes)))
|
||||
assistant_reply(conversation, waiting_since: Time.current)
|
||||
assistant_reply(conversation, waiting_since: follow_up.created_at)
|
||||
end
|
||||
|
||||
def apply_outcome(conversation, created_at, outcome)
|
||||
|
||||
Reference in New Issue
Block a user