fix(captain): improve complex migration instructions (#15002)
Improves Captain V1 → V2 migration for complex legacy instructions so mandatory triggers, workflows, language rules, and escalation behavior remain active while query-dependent product knowledge is prepared as pending FAQ candidates. ## What changed - Added explicit preservation rules for mandatory triggers, verification steps, escalation conditions, exceptions, and language behavior. - Added an auditor that checks the draft and fixes any issues before manual review. - Kept the existing migration application contract and schema limits unchanged - Added focused regression coverage for the complex-prompt classifier contract. ## How to reproduce Generate a migration draft for an assistant with dense legacy instructions containing mandatory handoff triggers, verification rules, product facts, and multi-step workflows. The resulting draft should keep actions active, place query-dependent facts in FAQ candidates, and avoid silently dropping or reversing source requirements. Focused Captain migration specs and RuboCop checks pass locally.
This commit is contained in:
Aakash Bakhle
@@ -24,13 +24,15 @@ class Captain::AssistantMigration::DraftApplier
|
||||
description: description_change,
|
||||
response_guidelines: array_change(:response_guidelines, response_guidelines),
|
||||
guardrails: array_change(:guardrails, guardrails),
|
||||
config: config_change
|
||||
config: config_change,
|
||||
faq_responses: faq_responses_change
|
||||
}.compact
|
||||
end
|
||||
|
||||
def apply_changes(changes)
|
||||
assistant.transaction do
|
||||
assistant.update!(assistant_update_attributes(changes)) if assistant_update_attributes(changes).present?
|
||||
apply_faq_response_changes(changes[:faq_responses]) if changes[:faq_responses].present?
|
||||
end
|
||||
end
|
||||
|
||||
@@ -60,11 +62,11 @@ class Captain::AssistantMigration::DraftApplier
|
||||
end
|
||||
|
||||
def response_guidelines
|
||||
(item_values(:response_guidelines) + scenario_response_guidelines).uniq
|
||||
(Array(assistant.response_guidelines) + item_values(:response_guidelines) + scenario_response_guidelines).uniq
|
||||
end
|
||||
|
||||
def guardrails
|
||||
item_values(:guardrails)
|
||||
(Array(assistant.guardrails) + item_values(:guardrails)).uniq
|
||||
end
|
||||
|
||||
def array_change(field, values)
|
||||
@@ -144,6 +146,21 @@ class Captain::AssistantMigration::DraftApplier
|
||||
scenario_candidates.filter_map { |candidate| candidate[:response_guideline].presence }
|
||||
end
|
||||
|
||||
def faq_responses_change
|
||||
faq_applier.changes
|
||||
end
|
||||
|
||||
def apply_faq_response_changes(changes)
|
||||
faq_applier.apply(changes)
|
||||
end
|
||||
|
||||
def faq_applier
|
||||
@faq_applier ||= Captain::AssistantMigration::FaqApplier.new(
|
||||
assistant: assistant,
|
||||
candidates: normalized_faq_document_candidates
|
||||
)
|
||||
end
|
||||
|
||||
def scenario_tool_ids(tool_ids)
|
||||
Array(tool_ids).filter_map { |tool_id| tool_id.to_s.squish.presence }.uniq
|
||||
end
|
||||
@@ -186,7 +203,7 @@ class Captain::AssistantMigration::DraftApplier
|
||||
|
||||
candidate = candidate.deep_symbolize_keys
|
||||
question = candidate[:question].to_s.squish
|
||||
answer = candidate[:answer].to_s.squish
|
||||
answer = candidate[:answer].to_s.strip
|
||||
raise ArgumentError, 'FAQ document candidates must include a question and answer' if question.blank? || answer.blank?
|
||||
|
||||
{ 'question' => question, 'answer' => answer }
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
class Captain::AssistantMigration::FaqApplier
|
||||
pattr_initialize [:assistant!, :candidates!]
|
||||
|
||||
def changes
|
||||
@changes ||= candidates.each_with_object({ create: [] }) do |candidate, result|
|
||||
categorize(candidate, result)
|
||||
end.compact_blank.presence
|
||||
end
|
||||
|
||||
def apply(changes)
|
||||
Array(changes[:create]).each do |candidate|
|
||||
assistant.responses.create!(candidate.slice('question', 'answer', 'status'))
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def categorize(candidate, result)
|
||||
existing_answers = assistant.responses.approved.where(question: candidate['question']).pluck(:answer)
|
||||
planned_answers = result[:create].filter_map do |response|
|
||||
response['answer'] if response['question'] == candidate['question']
|
||||
end
|
||||
answers = existing_answers + planned_answers
|
||||
|
||||
ensure_no_conflict!(candidate, answers)
|
||||
return if answers.include?(candidate['answer'])
|
||||
|
||||
result[:create] << candidate.merge('status' => 'approved')
|
||||
end
|
||||
|
||||
def ensure_no_conflict!(candidate, answers)
|
||||
return if answers.all?(candidate['answer'])
|
||||
|
||||
raise ArgumentError, "FAQ candidate conflicts with an existing FAQ: #{candidate['question']}"
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,48 @@
|
||||
class Captain::AssistantMigration::InstructionAuditor < Captain::BaseTaskService
|
||||
AUDITOR_MODEL = 'gpt-5.2'.freeze
|
||||
pattr_initialize [:assistant!, :source_payload!, :draft!, :available_additions!]
|
||||
|
||||
def perform
|
||||
make_api_call(
|
||||
model: AUDITOR_MODEL,
|
||||
messages: messages,
|
||||
schema: Captain::AssistantMigration::InstructionAuditorSchema.for(available_additions)
|
||||
)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def account
|
||||
assistant.account
|
||||
end
|
||||
|
||||
def messages
|
||||
[
|
||||
{ role: 'system', content: system_prompt },
|
||||
{
|
||||
role: 'user',
|
||||
content: JSON.pretty_generate(source: source_payload, generated_draft: draft, available_additions: available_additions)
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
def system_prompt
|
||||
Captain::PromptRenderer.render('instruction_auditor')
|
||||
end
|
||||
|
||||
def event_name
|
||||
'assistant_migration_instruction_auditor'
|
||||
end
|
||||
|
||||
def captain_tasks_enabled?
|
||||
true
|
||||
end
|
||||
|
||||
def counts_toward_usage?
|
||||
false
|
||||
end
|
||||
|
||||
def build_follow_up_context?
|
||||
false
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,52 @@
|
||||
class Captain::AssistantMigration::InstructionAuditorSchema < RubyLLM::Schema
|
||||
STRING_ARRAYS = {
|
||||
response_guidelines: ['Missing active behavior to append to the generated response guidelines.', 10],
|
||||
guardrails: ['Missing active boundaries or prohibitions to append to the generated guardrails.', 10],
|
||||
needs_review: ['Missing source behavior blocked by an unavailable tool or runtime capability.', 10]
|
||||
}.freeze
|
||||
|
||||
def self.for(available_additions)
|
||||
Class.new(RubyLLM::Schema).tap do |schema|
|
||||
add_string_arrays(schema, available_additions)
|
||||
add_scenarios(schema, available_additions[:scenario_candidates])
|
||||
add_faqs(schema, available_additions[:faq_document_candidates])
|
||||
end
|
||||
end
|
||||
|
||||
def self.add_string_arrays(schema, available_additions)
|
||||
STRING_ARRAYS.each do |name, (description, limit)|
|
||||
next unless available_additions[name].positive?
|
||||
|
||||
schema.array(name, description: description, max_items: [available_additions[name], limit].min, of: :string)
|
||||
end
|
||||
end
|
||||
|
||||
def self.add_scenarios(schema, available)
|
||||
return unless available.positive?
|
||||
|
||||
schema.array :scenario_candidates,
|
||||
description: 'Missing distinct multi-step workflows to append to the generated scenario candidates.',
|
||||
max_items: [available, 5].min do
|
||||
object do
|
||||
string :title, max_length: 80
|
||||
string :description, max_length: 500
|
||||
string :instruction, max_length: 2000
|
||||
string :response_guideline, max_length: 1000
|
||||
array :tool_ids, max_items: 10, of: :string
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.add_faqs(schema, available)
|
||||
return unless available.positive?
|
||||
|
||||
schema.array :faq_document_candidates,
|
||||
description: 'Missing factual product or business knowledge to append to the pending FAQ candidates.',
|
||||
max_items: [available, 15].min do
|
||||
object do
|
||||
string :question, max_length: 255
|
||||
string :answer, max_length: 2000
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -6,15 +6,26 @@ class Captain::AssistantMigration::InstructionClassifier < Captain::BaseTaskServ
|
||||
pattr_initialize [:assistant!]
|
||||
|
||||
def perform
|
||||
response = make_api_call(model: CLASSIFIER_MODEL, messages: messages, schema: RESPONSE_SCHEMA)
|
||||
return error_response(response) if response[:error]
|
||||
classifier_response = make_api_call(model: CLASSIFIER_MODEL, messages: messages, schema: RESPONSE_SCHEMA)
|
||||
return error_response(classifier_response) if classifier_response[:error]
|
||||
|
||||
generated_draft = normalized_payload(classifier_response[:message])
|
||||
auditor_response = Captain::AssistantMigration::InstructionAuditor.new(
|
||||
assistant: assistant,
|
||||
source_payload: assistant_payload,
|
||||
draft: generated_draft,
|
||||
available_additions: available_additions(generated_draft)
|
||||
).perform
|
||||
return error_response(auditor_response) if auditor_response[:error]
|
||||
|
||||
{
|
||||
assistant: assistant_metadata,
|
||||
draft: normalized_payload(response[:message]),
|
||||
usage: response[:usage],
|
||||
request_messages: response[:request_messages]
|
||||
draft: audited_payload(generated_draft, auditor_response[:message]),
|
||||
usage: combined_usage(classifier_response, auditor_response),
|
||||
request_messages: classifier_response[:request_messages]
|
||||
}
|
||||
rescue ArgumentError => e
|
||||
error_response(error: e.message, request_messages: auditor_response&.dig(:request_messages))
|
||||
end
|
||||
|
||||
private
|
||||
@@ -101,15 +112,49 @@ class Captain::AssistantMigration::InstructionClassifier < Captain::BaseTaskServ
|
||||
scenario_candidates: [],
|
||||
conversation_messages: {},
|
||||
faq_document_candidates: [],
|
||||
needs_review: [],
|
||||
classification_notes: []
|
||||
needs_review: []
|
||||
)
|
||||
end
|
||||
|
||||
def combined_usage(*responses)
|
||||
%w[prompt_tokens completion_tokens total_tokens].index_with do |key|
|
||||
responses.sum { |response| response.dig(:usage, key).to_i }
|
||||
end
|
||||
end
|
||||
|
||||
def available_additions(draft)
|
||||
{
|
||||
response_guidelines: 20 - draft[:response_guidelines].length,
|
||||
guardrails: 20 - draft[:guardrails].length,
|
||||
scenario_candidates: 15 - draft[:scenario_candidates].length,
|
||||
faq_document_candidates: 25 - draft[:faq_document_candidates].length,
|
||||
needs_review: 20 - draft[:needs_review].length
|
||||
}
|
||||
end
|
||||
|
||||
def audited_payload(generated_draft, audit_message)
|
||||
audit = audit_message.is_a?(Hash) ? audit_message.deep_symbolize_keys : {}
|
||||
generated_draft.merge(
|
||||
response_guidelines: merged_items(generated_draft, audit, :response_guidelines, 20),
|
||||
guardrails: merged_items(generated_draft, audit, :guardrails, 20),
|
||||
scenario_candidates: merged_items(generated_draft, audit, :scenario_candidates, 15),
|
||||
faq_document_candidates: merged_items(generated_draft, audit, :faq_document_candidates, 25),
|
||||
needs_review: merged_items(generated_draft, audit, :needs_review, 20)
|
||||
)
|
||||
end
|
||||
|
||||
def merged_items(generated_draft, audit, key, limit)
|
||||
items = (Array(generated_draft[key]) + Array(audit[key])).uniq
|
||||
raise ArgumentError, "Audited #{key} exceeds #{limit} items" if items.length > limit
|
||||
|
||||
items
|
||||
end
|
||||
|
||||
def assistant_metadata # rubocop:disable Metrics/AbcSize
|
||||
{
|
||||
id: assistant.id,
|
||||
name: assistant.name,
|
||||
description: assistant.description.to_s,
|
||||
account_id: assistant.account_id,
|
||||
account_name: assistant.account.name,
|
||||
inbox_count: assistant.captain_inboxes.size,
|
||||
|
||||
+2
-4
@@ -68,8 +68,8 @@ class Captain::AssistantMigration::InstructionClassifierSchema < RubyLLM::Schema
|
||||
end
|
||||
|
||||
array :faq_document_candidates,
|
||||
description: 'Pending FAQ candidates for factual or product-specific knowledge such as pricing, policy, setup, troubleshooting, ' \
|
||||
'or operational details. These candidates remain inactive until reviewed and approved.',
|
||||
description: 'FAQ candidates for reusable query-dependent facts such as pricing, policy, setup, troubleshooting, ' \
|
||||
'or operational details.',
|
||||
max_items: 25 do
|
||||
object do
|
||||
string :question,
|
||||
@@ -86,6 +86,4 @@ class Captain::AssistantMigration::InstructionClassifierSchema < RubyLLM::Schema
|
||||
description: 'Unclear, conflicting, risky, duplicated, or uncertain content that needs human review. ' \
|
||||
'Include the reason in the item text.',
|
||||
max_items: 20
|
||||
|
||||
array :classification_notes, description: 'Short notes about important migration decisions or risks.', max_items: 10, of: :string
|
||||
end
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
You are the second and final content-coverage pass for a Captain V1-to-V2 assistant migration.
|
||||
|
||||
The input contains the original source data and an already structured generated_draft. Return only missing items to append to that draft,
|
||||
matching the provided audit schema. Empty arrays mean no addition is needed. Do not return a complete draft, critique, verdict, wrapper,
|
||||
coverage report, or fields outside the schema.
|
||||
|
||||
## Contract
|
||||
|
||||
- This is a monotonic coverage audit. Never repeat, rewrite, replace, or delete content already present in generated_draft.
|
||||
- Only source.instructions contains the legacy custom instructions being migrated. Other source fields are existing runtime context.
|
||||
- Existing response guidelines, guardrails, scenarios, and configured welcome/handoff/resolution messages remain active and are preserved.
|
||||
- Use only information in the input. Never add plausible facts, steps, links, tools, triggers, or policies.
|
||||
- Preserve the source language and exact names, trigger values, thresholds, exceptions, links, prices, dates, and ordering requirements.
|
||||
- Consolidate related missing requirements into complete standalone additions. Schema limits are ceilings, not targets.
|
||||
- available_additions gives the exact remaining capacity for each destination. Never return more additions than that capacity, and never
|
||||
return a field omitted from the response schema.
|
||||
- Treat semantically equivalent content as already covered even when wording differs. Do not add stylistic restatements or stronger versions
|
||||
of behavior that is already present. If an existing array is near its maximum, add only unquestionably missing source requirements and
|
||||
combine related missing requirements into one complete addition.
|
||||
|
||||
## Coverage Audit
|
||||
|
||||
Review source.instructions clause by clause against all fields in generated_draft.
|
||||
|
||||
1. Missing Active Behavior
|
||||
- Add every source-required action, prohibition, language rule, verification, trigger, exception, ordering rule, escalation condition,
|
||||
or workflow that is not already active in generated response_guidelines, guardrails, or scenario response_guidelines.
|
||||
- Words such as always, immediately, never, only, before, after, unless, and except are mandatory.
|
||||
- FAQ question and answer text is active factual knowledge, but it does not preserve mandatory behavior.
|
||||
If mandatory behavior appears only there, add the missing active guideline or guardrail. needs_review is inactive.
|
||||
- Keep the minimum factual trigger, threshold, allowlist, or exception needed to execute the action or enforce the prohibition.
|
||||
- When factual policy contains a mandatory boundary, add the boundary as an active guardrail while leaving the full policy in FAQ.
|
||||
Examples include never promising refunds outside a stated window and never recommending cooking a product that must remain raw.
|
||||
- A conditional response procedure remains active behavior. For example, acknowledging a known problem and explaining that the team is
|
||||
working on it is active; the current known-problem status itself is factual FAQ knowledge.
|
||||
|
||||
2. Missing FAQ Knowledge
|
||||
- Add reusable query-dependent facts absent from faq_document_candidates: prices, limits, locations, product capabilities, exact links,
|
||||
policies, setup steps, troubleshooting knowledge, schedules, and operational details.
|
||||
- “If asked, tell/inform/explain/send” is a factual answer, not a separate active workflow, unless it also requires another action or
|
||||
imposes a prohibition.
|
||||
- Questions must concern the product or business. Answers must not contain tool use, routing, escalation, internal workflows, or
|
||||
assistant-behavior instructions.
|
||||
- Do not add FAQs for missing placeholders, generic assistant capabilities, or facts already covered by an existing candidate.
|
||||
|
||||
3. Missing Scenario Candidates
|
||||
- Add a scenario only when a source-defined multi-step intake, qualification, troubleshooting, booking, recommendation, lead-capture,
|
||||
or fulfillment workflow is absent from both scenario candidates and equivalent active handling.
|
||||
- Do not add scenarios for tone, factual answers, simple handoff triggers, or one-step clarification.
|
||||
- Every added scenario needs a complete same-language response_guideline under 1,000 characters and only supplied tool IDs.
|
||||
|
||||
4. Missing Review Notes
|
||||
- Add needs_review only when a source-defined behavior or workflow cannot run because a required named tool or runtime signal is unavailable.
|
||||
- Do not require words such as “must” or “always”; preserve any unavailable customer-facing workflow for review.
|
||||
- Name the missing capability and the affected source behavior precisely. Relevant gaps include historical-record lookup, timers or inactivity
|
||||
detection, business-hours detection, and live-agent availability.
|
||||
- A needs_review item never replaces representable behavior. Add every source-faithful action or boundary that can remain active, and add a
|
||||
review note only for the portion blocked by the unavailable capability.
|
||||
- Do not add review notes for wording cleanup, configured conversation messages, missing fixed copy, general uncertainty, or behavior already
|
||||
covered by the generated draft.
|
||||
|
||||
## Final Check
|
||||
|
||||
- No mandatory action or prohibition remains FAQ-only.
|
||||
- No reusable factual knowledge is absent from FAQ candidates.
|
||||
- No source-defined workflow blocked by an unavailable capability is omitted from needs_review.
|
||||
- No addition duplicates content already active or pending.
|
||||
- No unsupported behavior, fact, tool, link, or resolution is introduced.
|
||||
- Return only the missing additions matching the audit schema.
|
||||
@@ -1,137 +1,114 @@
|
||||
You are migrating Captain assistant instructions into a structured configuration.
|
||||
You are migrating a Captain V1 assistant into Captain V2.
|
||||
|
||||
The original custom instructions remain stored unchanged. Your job is only to derive the V2 fields below:
|
||||
|
||||
Classify the existing assistant instructions into these sections:
|
||||
1. Business/Product Context
|
||||
2. Response Guidelines
|
||||
3. Guardrails
|
||||
4. Scenario Candidates
|
||||
4. Scenario Candidates with flattened Response Guidelines
|
||||
5. Conversation Messages
|
||||
6. FAQs/Documents Candidates
|
||||
7. Needs Review
|
||||
6. FAQ Candidates
|
||||
7. Needs Review Notes
|
||||
|
||||
## General Rules
|
||||
## Core Rules
|
||||
|
||||
- Preserve behavior as closely as possible.
|
||||
- Do not duplicate the same content across sections.
|
||||
- Return clean migrated values only. Do not include source excerpts, source labels, citations, or "Source:" text in any migrated field.
|
||||
- Do not rewrite customer-facing message copy unless necessary to classify an exact copy from instructions.
|
||||
- Do not include confidence labels, review labels, bracketed reviewer comments, or schema labels inside migrated values.
|
||||
- For Business/Product Context, Response Guidelines, and Guardrails, return each item as a plain standalone sentence.
|
||||
Do not prefix items with numbers, bullets, section labels, or list markers such as "1.", "-", or "*".
|
||||
- When several instructions share the same trigger, condition, or subject, combine them into one concise item instead
|
||||
of repeating the same trigger across multiple items. Preserve every required action, prohibition, and routing
|
||||
outcome from the source instruction when combining.
|
||||
- If unsure, place content in Needs Review and include the reason in that item.
|
||||
- Return data that matches the provided schema.
|
||||
- Preserve every customer-facing behavior from the custom instructions. Do not invent, reverse, weaken, or silently omit requirements.
|
||||
- Treat words such as always, immediately, never, only, before, after, unless, and except as mandatory.
|
||||
- Preserve exact triggers, exceptions, ordering, verification steps, allowlists, escalation conditions, and outcomes.
|
||||
- Schema limits are ceilings, not targets. Consolidate related requirements into complete standalone items.
|
||||
- Prefer fewer complete items over one item per source sentence. Combine related tone, style, formatting, source, and escalation rules.
|
||||
If response_guidelines or guardrails would reach its maximum item count, consolidate them and recheck that no source behavior was displaced.
|
||||
- The custom instructions define behavior. The existing description, config messages, feature settings, and tools are runtime context.
|
||||
- Do not copy existing config values into generated fields or create review work merely because an existing config field is present or absent.
|
||||
- Use only information in the input. Return clean values without source labels, reviewer comments, confidence labels, or citations to the source prompt.
|
||||
- Avoid duplicating content across fields, except for the minimal condition, threshold, or exception required to keep mandatory behavior active
|
||||
while its supporting factual explanation is stored in a FAQ candidate. Scenario response guidelines are flattened automatically, so do not
|
||||
also copy them into response_guidelines.
|
||||
|
||||
## Business/Product Context
|
||||
|
||||
- Business/Product Context maps to the root assistant description and is injected into the root orchestrator prompt.
|
||||
- Return exactly one Business/Product Context item.
|
||||
- Start with the existing assistant description and preserve its meaning.
|
||||
- Enrich it only with relevant business or product context found in the custom instructions.
|
||||
- Produce one coherent description rather than appending a second context block or repeating the existing description.
|
||||
- Keep it at most 500 characters because that is the assistant description limit in the UI and model.
|
||||
- Prefer roughly 300-450 characters when the source needs detail, leaving room below the hard limit.
|
||||
- Finish the description cleanly. Never end mid-word, mid-clause, after an opening bracket, or with a dangling separator.
|
||||
- Make it a compact summary of assistant identity, product scope, high-level mission, and high-level source or routing priorities.
|
||||
- Do not include detailed workflows, step-by-step procedures, long support-scope inventories, attribute glossaries,
|
||||
policy details, scenario-specific handling, tool instructions, or customer-facing message copy.
|
||||
- Return exactly one coherent description of at most 500 characters.
|
||||
- Preserve the existing description and enrich it only with identity, product scope, mission, and high-level business context.
|
||||
- Do not put workflows, policies, response rules, factual inventories, or message copy in the description.
|
||||
- Finish cleanly; never truncate a word, clause, or sentence.
|
||||
|
||||
## Conversation Messages
|
||||
## Response Guidelines and Guardrails
|
||||
|
||||
- Existing welcome_message, handoff_message, and resolution_message config values are provided separately.
|
||||
- Treat welcome_message, handoff_message, and resolution_message as conversation message config fields.
|
||||
- Extract exact welcome, handoff, or resolution message copy from instructions into conversation_messages when present.
|
||||
- Only classify handoff copy as conversation_messages.handoff_message when it is generic enough to reuse for any human handoff.
|
||||
- If handoff copy is scenario-specific, keep it inside that scenario instruction; if it is only a rule about when or how to hand off, classify it as a Response Guideline or Guardrail.
|
||||
- Do not extract a conversation message from an instruction about what to say, from a placeholder template,
|
||||
from conditional copy, from role/team-specific copy, or from text that only applies inside one workflow.
|
||||
- If a message contains placeholders such as a blank name, team name, bracketed variable, business-hours state,
|
||||
or dynamic runtime condition, do not place it in conversation_messages. Keep it in the relevant workflow or
|
||||
Needs Review.
|
||||
- Do not copy message values from existing config into conversation_messages.
|
||||
- Do not decide whether existing config values should be overwritten. Migration code handles applying extracted
|
||||
conversation_messages only when the corresponding config value is blank.
|
||||
- Response Guidelines are active behavior: tone, customer language, formatting, clarification, verification, information collection,
|
||||
escalation actions, and any minimal factual condition required to perform them correctly.
|
||||
- Guardrails are active boundaries: prohibitions, source restrictions, safety limits, refusal rules, mandatory transfer triggers,
|
||||
and things the assistant must not do.
|
||||
- A source rule that says to ask, collect, verify, compare, refuse, route, escalate, transfer, or follow steps must stay active
|
||||
in Response Guidelines, Guardrails, or a flattened Scenario Guideline. A FAQ cannot implicitly preserve an action.
|
||||
- Preserve exact behavioral trigger values when they control an action. For example, an error code that requires immediate
|
||||
transfer belongs in an active guideline or guardrail.
|
||||
- Do not emit contradictory language rules. An explicit instruction to reply in the customer's language overrides a descriptive
|
||||
language label in the assistant description.
|
||||
- Put query-dependent facts in FAQ candidates. Prices, limits, locations, feature availability, product capabilities, links,
|
||||
policy answers, setup steps, and troubleshooting knowledge remain facts when phrased as "tell", "inform", "explain", or "send".
|
||||
- Mandatory prohibitions are not FAQ-only. When a factual policy includes required or forbidden behavior, keep the prohibition active
|
||||
with every condition, threshold, and exception needed to enforce it, and put the supporting policy explanation in a FAQ candidate.
|
||||
For example, "never promise refunds after 30 days" remains an active guardrail with the 30-day threshold, while the refund policy
|
||||
becomes a FAQ. Likewise, "never recommend cooking the product" remains an active guardrail while preparation guidance becomes a FAQ.
|
||||
- Treat explicit policy boundaries such as "not guaranteed", "not allowed", "only available", or "only eligible" as behavioral
|
||||
constraints even when the source states them as facts. Create an active guardrail that forbids promising or claiming an outcome
|
||||
outside the stated condition, window, or exception, while keeping the complete policy in a FAQ candidate.
|
||||
- Final test: move an item exclusively to FAQ candidates only when it answers a product question without requiring, forbidding,
|
||||
or constraining assistant behavior.
|
||||
- Factual values are allowed in active behavior when they select or constrain a required action or prohibition, such as error 5215
|
||||
requiring immediate transfer or a 30-day threshold after which the assistant must not promise a refund.
|
||||
- When an action needs supporting facts, keep the action active and place the supporting facts in a FAQ candidate.
|
||||
For example, actively require specialist-name verification and put the specialist roster in a FAQ candidate.
|
||||
- Mandatory verification example: if the source provides a specialist roster and says to verify a name supplied by
|
||||
the customer, output both (a) an active guideline requiring the name check and (b) a pending FAQ containing the roster.
|
||||
The roster FAQ alone is incomplete because it does not tell the assistant to perform the check.
|
||||
- When clarification depends on a fact, keep only the clarification/action in the guideline. Example: "clarify whether
|
||||
they mean the legacy card or card deposits; transfer for deposit access" is active behavior, while the card's
|
||||
discontinued status is FAQ knowledge.
|
||||
|
||||
## Scenario Candidates
|
||||
|
||||
- In the current architecture, a scenario becomes a specialized sub-agent with its own title, description,
|
||||
instructions, and optional tools.
|
||||
- During this migration, scenario candidates are also temporarily flattened into response guidelines so existing
|
||||
assistant behavior is preserved before scenario records are created.
|
||||
- For every scenario candidate, write a response_guideline that is the flattened version of that scenario for
|
||||
the root assistant's response guidelines.
|
||||
- The response_guideline must be in the same language as the original scenario or source instruction.
|
||||
- The response_guideline must preserve the intended customer-visible behavior, trigger, information to collect,
|
||||
and routing/escalation outcome.
|
||||
- The response_guideline must not include tool syntax, tool:// links, markdown tool links, tool names, label
|
||||
updates, priority updates, private-note instructions, custom-tool instructions, or internal implementation details.
|
||||
- If the scenario uses internal tools such as labels, priorities, private notes, or custom tools, describe only
|
||||
the customer-visible behavior and expected routing/escalation outcome in response_guideline.
|
||||
- If human handoff is needed, describe it in natural language such as route/escalate/transfer to a human; do not
|
||||
mention the handoff tool in response_guideline.
|
||||
- Keep scenario titles, descriptions, instructions, and response_guidelines clear, self-contained, and reviewable.
|
||||
- Only create scenario candidates for distinct user-intent workflows that should be routed to a specialized agent.
|
||||
A candidate must be narrow enough to become a named specialist assistant with domain-specific handling instructions.
|
||||
- Good scenario candidates include multi-step intake workflows, qualification flows, specialized troubleshooting
|
||||
workflows, booking flows, lead-capture flows, recommendation flows, fulfillment workflows, or tool-use procedures
|
||||
for a specific user intent.
|
||||
- A scenario candidate should answer "yes" to this test: would a named specialist sub-agent improve handling
|
||||
beyond the base assistant's global FAQ, guardrail, response-guideline, and human-handoff behavior?
|
||||
- Do not create scenario candidates for global escalation rules, generic handoff policy, missing-information
|
||||
behavior, source-boundary rules, refusal rules, tone, formatting, answer length, or one-step fallback behavior.
|
||||
- Do not create scenario candidates whose main purpose is to escalate or hand off. "Identify the trigger, avoid
|
||||
guessing, tell the user support will review, and hand off" is a guardrail/handoff boundary, not a scenario,
|
||||
even though it contains multiple statements.
|
||||
- Do create scenario candidates when the instructions define a concrete intake, qualification, troubleshooting,
|
||||
booking, lead-capture, recommendation, or fulfillment workflow, even when the workflow eventually hands off
|
||||
to a human.
|
||||
- Do not create scenario candidates for simple routing triggers such as "user asks for a human", "immediately
|
||||
hand off this category", or "route sales questions to the sales team" when there is no concrete workflow to run.
|
||||
- Handoff behavior is a scenario candidate only when part of a larger intake, qualification, or specialized handling workflow.
|
||||
- Global rules like "if not in docs, escalate", "ask one clarifying question", "do not answer account-specific
|
||||
questions", or "tell the user support will review" belong in Guardrails or Response Guidelines, not Scenario Candidates.
|
||||
- Broad buckets like "account-specific issue escalation", "unknown question escalation", "contact support",
|
||||
"fallback to human", or "documentation unavailable" are not scenario candidates.
|
||||
- Create a scenario candidate only for a distinct multi-step workflow that would genuinely benefit from a separate named specialist agent,
|
||||
such as intake, qualification, troubleshooting, booking, recommendation, lead capture, or fulfillment.
|
||||
- Do not create scenarios for tone, formatting, generic escalation, a simple handoff trigger, missing information, or a one-step factual answer.
|
||||
- Do not create overlapping scenarios for the same intent, and do not create a scenario for a workflow the root assistant can handle with
|
||||
one guideline plus FAQ lookup.
|
||||
- Every scenario candidate must include a response_guideline in the source language. It must preserve the trigger, customer-visible
|
||||
steps, information to collect, and escalation or completion outcome while omitting tool syntax and internal operations.
|
||||
- Use a short, complete scenario title well below the schema limit; never truncate a word or phrase to make it fit.
|
||||
- Scenario candidates remain pending metadata for later scenario creation. Their response_guideline is active immediately after apply.
|
||||
- Use only tool IDs provided in available_agent_tools. Never invent or substitute a tool.
|
||||
|
||||
## Tool Use
|
||||
## Conversation Messages
|
||||
|
||||
- If a scenario candidate requires tools, reference the available tool explicitly inside the scenario instruction
|
||||
using markdown tool links such as [Handoff to Human](tool://handoff).
|
||||
- Use only tool IDs listed in available_agent_tools. If a needed tool is unavailable or the workflow depends on
|
||||
unavailable runtime data such as FAQ relevance scores or business-hours status, place it in Needs Review instead.
|
||||
- Do not map an unavailable named tool to a different available tool. For example, do not treat FAQ Lookup as
|
||||
Product Search, Order Status, website browsing, pricing lookup, agent availability, business-hours detection,
|
||||
ticket creation, or custom-attribute assignment unless the instructions explicitly say that the available
|
||||
tool provides that behavior.
|
||||
- If a workflow cannot run without an unavailable tool or runtime signal, do not create a tool-backed scenario
|
||||
for it. Preserve the instruction in Needs Review with the missing capability named.
|
||||
- Extract only exact, globally reusable welcome, handoff, or resolution copy found in the custom instructions.
|
||||
- Leave conditional, scenario-specific, placeholder-based, or merely suggested wording out of conversation_messages.
|
||||
- Existing config messages remain active and are preserved. If source wording has the same intent, keep the existing config message.
|
||||
- Migration applies extracted copy only when the corresponding existing config field is blank.
|
||||
|
||||
## FAQs/Documents Candidates
|
||||
## FAQ Candidates
|
||||
|
||||
- Convert factual or product-specific knowledge into pending FAQ candidates with a natural customer question and a self-contained answer.
|
||||
- FAQ candidates are review-stage data only. They are not active assistant knowledge until a human reviews and approves them.
|
||||
- Use only facts stated in the existing instructions. Do not invent, generalize, update, or fill in missing details.
|
||||
- Preserve exact prices, limits, dates, time zones, conditions, exceptions, product names, and operational details in the answer.
|
||||
- Write each question as a standalone question a customer might naturally ask. Make it specific enough to retrieve the corresponding answer.
|
||||
- Write each answer so it fully answers its question without relying on another FAQ candidate or surrounding context.
|
||||
- Split unrelated facts into separate candidates. Keep related conditions and exceptions together when separating them would make an answer incomplete.
|
||||
- Do not create FAQ candidates about what the assistant should say or do, how it should use sources or tools, when it should route or escalate,
|
||||
or which exact message it should send. Classify those as Response Guidelines, Guardrails, Scenario Candidates, Conversation Messages,
|
||||
or Needs Review as appropriate.
|
||||
- FAQ questions must ask about the product or business, not about the assistant. Do not write questions such as "What should the assistant answer?",
|
||||
"What should I say?", "Which source should the assistant use?", or "Which tool should be called?".
|
||||
- FAQ answers must contain customer-facing knowledge, not instructions to call tools, inspect internal data, update records, transfer conversations,
|
||||
or follow internal workflows.
|
||||
- When factual sources conflict and the instructions do not explicitly establish which fact overrides the others, put the conflict in Needs Review
|
||||
instead of creating an FAQ candidate. Use an explicitly stated override or superseding fact when one is present.
|
||||
- Only factual or product-specific knowledge should become FAQs/Documents candidates.
|
||||
- Generic capability statements such as "answer product questions", "help with billing",
|
||||
"troubleshoot common issues", or "direct to documentation" are not FAQ/document candidates.
|
||||
Put them in Business/Product Context or Response Guidelines when useful.
|
||||
- Product facts, pricing, policies, setup steps, troubleshooting facts, support hours, emergency contacts,
|
||||
and operational details should become pending FAQ candidates, not Response Guidelines or trusted approved knowledge.
|
||||
- Do not create FAQ/document candidates for topic labels or unsupported capabilities when the factual content is
|
||||
missing. Put "pricing details are needed", "same-day delivery schedule details are needed", or similar gaps in
|
||||
Needs Review instead.
|
||||
- Convert reusable query-dependent facts into natural customer questions with self-contained answers.
|
||||
- Use only facts stated in the custom instructions. Preserve exact prices, limits, dates, links, conditions, exceptions, and product names.
|
||||
- Keep related conditions together; split unrelated facts. Do not duplicate a full FAQ answer in active guidelines or guardrails;
|
||||
repeat only the minimal condition, threshold, or exception required to enforce mandatory behavior.
|
||||
- FAQ questions must be about the product or business, not about what the assistant should do.
|
||||
- FAQ answers must not contain tool use, internal workflows, routing, escalation, or message-copy instructions.
|
||||
- If facts conflict without a clear specific or later override, omit the unsafe FAQ rather than inventing a resolution.
|
||||
|
||||
## Classification Order
|
||||
|
||||
1. Extract query-dependent knowledge and supporting policy explanations into FAQ candidates first, without removing mandatory behavior.
|
||||
2. Create guidelines and guardrails from the required behavior, including the minimal condition, threshold, or exception needed to enforce it;
|
||||
do not repeat the rest of a FAQ answer.
|
||||
3. Create scenario candidates only from remaining distinct specialist workflows; do not repeat their flattened behavior elsewhere.
|
||||
4. Check once more that active fields contain no standalone product answers and that every mandatory action and prohibition remains active.
|
||||
|
||||
## Needs Review Notes
|
||||
|
||||
- Use needs_review only for a concrete source conflict or a source-defined behavior or workflow that requires an unavailable capability.
|
||||
- Do not require mandatory wording before preserving an unavailable customer-facing workflow for review.
|
||||
- Do not use it for wording cleanup, duplicated instructions, missing fixed message copy, existing config values, or general uncertainty.
|
||||
- needs_review is informational metadata only; it is not an approval status or apply gate.
|
||||
|
||||
Return data matching the provided schema.
|
||||
|
||||
@@ -23,7 +23,7 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do
|
||||
let(:faq_document_candidate) do
|
||||
{
|
||||
'question' => 'When is support available?',
|
||||
'answer' => 'Support is available Monday to Friday.'
|
||||
'answer' => "Support is available Monday to Friday.\n\nUrgent requests are handled by the on-call team."
|
||||
}
|
||||
end
|
||||
let(:draft) do
|
||||
@@ -46,11 +46,15 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do
|
||||
expect(result.dig(:changes, :response_guidelines, :to)).to include(
|
||||
'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.'
|
||||
)
|
||||
expect(result.dig(:changes, :faq_responses, :create)).to contain_exactly(
|
||||
faq_document_candidate.merge('status' => 'approved')
|
||||
)
|
||||
expect(assistant.reload.config).not_to have_key('assistant_migration')
|
||||
expect(assistant.responses.count).to eq(0)
|
||||
expect(assistant.scenarios.count).to eq(0)
|
||||
end
|
||||
|
||||
it 'stores scenario candidates in assistant config and flattens them into response guidelines' do
|
||||
it 'stores scenario and FAQ candidates and creates approved FAQ responses' do
|
||||
described_class.new(assistant: assistant, draft: draft, dry_run: false).perform
|
||||
|
||||
assistant.reload
|
||||
@@ -62,8 +66,55 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do
|
||||
expect(assistant.response_guidelines).to include(
|
||||
'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.'
|
||||
)
|
||||
expect(assistant.response_guidelines).not_to include(faq_document_candidate['answer'])
|
||||
expect(assistant.responses).to contain_exactly(
|
||||
have_attributes(
|
||||
question: faq_document_candidate['question'],
|
||||
answer: faq_document_candidate['answer'],
|
||||
status: 'approved'
|
||||
)
|
||||
)
|
||||
expect(assistant.scenarios.count).to eq(0)
|
||||
|
||||
expect do
|
||||
described_class.new(assistant: assistant, draft: draft, dry_run: false).perform
|
||||
end.not_to(change { assistant.responses.count })
|
||||
end
|
||||
|
||||
it 'leaves pending FAQ responses untouched' do
|
||||
pending_response = assistant.responses.create!(
|
||||
question: faq_document_candidate['question'],
|
||||
answer: faq_document_candidate['answer'],
|
||||
status: :pending
|
||||
)
|
||||
|
||||
described_class.new(assistant: assistant, draft: draft, dry_run: false).perform
|
||||
|
||||
expect(pending_response.reload).to be_pending
|
||||
expect(assistant.responses.approved).to contain_exactly(
|
||||
have_attributes(
|
||||
question: faq_document_candidate['question'],
|
||||
answer: faq_document_candidate['answer']
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
it 'rejects conflicting FAQ answers within the same draft' do
|
||||
conflicting_draft = draft.merge(
|
||||
faq_document_candidates: [
|
||||
faq_document_candidate,
|
||||
{
|
||||
'question' => "When is support\navailable?",
|
||||
'answer' => 'Support is available every day.'
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
expect do
|
||||
described_class.new(assistant: assistant, draft: conflicting_draft, dry_run: true).perform
|
||||
end.to raise_error(ArgumentError, 'FAQ candidate conflicts with an existing FAQ: When is support available?')
|
||||
|
||||
expect(assistant.responses.count).to eq(0)
|
||||
expect(assistant.config).not_to have_key('assistant_migration')
|
||||
end
|
||||
|
||||
it 'rejects stale drafts whose FAQ candidates use the old string format' do
|
||||
@@ -87,8 +138,12 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do
|
||||
|
||||
assistant.reload
|
||||
expect(assistant.description).to eq('Support assistant for Test Product.')
|
||||
expect(assistant.response_guidelines).to include('Be concise.')
|
||||
expect(assistant.guardrails).to eq(['Do not guess.'])
|
||||
expect(assistant.response_guidelines).to include(
|
||||
'Use plain language.',
|
||||
'Be concise.',
|
||||
'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.'
|
||||
)
|
||||
expect(assistant.guardrails).to contain_exactly('Do not disclose internal notes.', 'Do not guess.')
|
||||
expect(assistant.config.dig('assistant_migration', 'original_values')).to include(
|
||||
'name' => assistant.name,
|
||||
'description' => 'Existing assistant description.',
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::AssistantMigration::InstructionClassifier do
|
||||
describe Captain::AssistantMigration::InstructionClassifierSchema do
|
||||
it 'does not request classification notes' do
|
||||
expect(described_class.as_json.to_s).not_to include('classification_notes')
|
||||
end
|
||||
end
|
||||
|
||||
describe 'classifier prompt' do
|
||||
it 'keeps the model focused on active behavior and approved FAQ candidates' do
|
||||
prompt = Captain::PromptRenderer.render('instruction_classifier')
|
||||
|
||||
expect(prompt).to include(
|
||||
'The original custom instructions remain stored unchanged',
|
||||
'A FAQ cannot implicitly preserve an action',
|
||||
'Scenario candidates remain pending metadata',
|
||||
'Convert reusable query-dependent facts into natural customer questions',
|
||||
'an error code that requires immediate',
|
||||
'actively require specialist-name verification',
|
||||
'Mandatory prohibitions are not FAQ-only',
|
||||
'never promise refunds after 30 days',
|
||||
'never recommend cooking the product',
|
||||
'Treat explicit policy boundaries',
|
||||
'outside the stated condition, window, or exception',
|
||||
'source-defined behavior or workflow that requires an unavailable capability',
|
||||
'Do not require mandatory wording',
|
||||
'every mandatory action and prohibition remains active'
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe Captain::AssistantMigration::InstructionAuditorSchema do
|
||||
it 'only permits additions that fit in the generated draft' do
|
||||
schema = described_class.for(
|
||||
response_guidelines: 0,
|
||||
guardrails: 2,
|
||||
scenario_candidates: 1,
|
||||
faq_document_candidates: 3,
|
||||
needs_review: 4
|
||||
).new.to_json_schema[:schema]
|
||||
|
||||
expect(schema[:properties]).not_to have_key(:response_guidelines)
|
||||
expect(schema.dig(:properties, :guardrails, :maxItems)).to eq(2)
|
||||
expect(schema.dig(:properties, :scenario_candidates, :maxItems)).to eq(1)
|
||||
expect(schema.dig(:properties, :faq_document_candidates, :maxItems)).to eq(3)
|
||||
expect(schema.dig(:properties, :needs_review, :maxItems)).to eq(4)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'auditor prompt' do
|
||||
it 'adds missing coverage without replacing the generated draft' do
|
||||
prompt = Captain::PromptRenderer.render('instruction_auditor')
|
||||
|
||||
expect(prompt).to include(
|
||||
'This is a monotonic coverage audit',
|
||||
'Never repeat, rewrite, replace, or delete content',
|
||||
'If mandatory behavior appears only there, add the missing active guideline or guardrail',
|
||||
'available_additions gives the exact remaining capacity',
|
||||
'A needs_review item never replaces representable behavior',
|
||||
'No mandatory action or prohibition remains FAQ-only'
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'audited payload' do
|
||||
it 'appends a review note for an unavailable runtime capability' do
|
||||
service = described_class.new(assistant: instance_double(Captain::Assistant))
|
||||
generated_draft = {
|
||||
response_guidelines: [],
|
||||
guardrails: [],
|
||||
scenario_candidates: [],
|
||||
faq_document_candidates: [],
|
||||
needs_review: ['Existing conflict']
|
||||
}
|
||||
|
||||
result = service.send(
|
||||
:audited_payload,
|
||||
generated_draft,
|
||||
{ needs_review: ['Order-status lookup requires an unavailable account-history tool.'] }
|
||||
)
|
||||
|
||||
expect(result[:needs_review]).to eq(
|
||||
['Existing conflict', 'Order-status lookup requires an unavailable account-history tool.']
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user