diff --git a/enterprise/app/services/captain/assistant_migration/draft_applier.rb b/enterprise/app/services/captain/assistant_migration/draft_applier.rb index df03624b4..e59acc205 100644 --- a/enterprise/app/services/captain/assistant_migration/draft_applier.rb +++ b/enterprise/app/services/captain/assistant_migration/draft_applier.rb @@ -24,13 +24,15 @@ class Captain::AssistantMigration::DraftApplier description: description_change, response_guidelines: array_change(:response_guidelines, response_guidelines), guardrails: array_change(:guardrails, guardrails), - config: config_change + config: config_change, + faq_responses: faq_responses_change }.compact end def apply_changes(changes) assistant.transaction do assistant.update!(assistant_update_attributes(changes)) if assistant_update_attributes(changes).present? + apply_faq_response_changes(changes[:faq_responses]) if changes[:faq_responses].present? end end @@ -60,11 +62,11 @@ class Captain::AssistantMigration::DraftApplier end def response_guidelines - (item_values(:response_guidelines) + scenario_response_guidelines).uniq + (Array(assistant.response_guidelines) + item_values(:response_guidelines) + scenario_response_guidelines).uniq end def guardrails - item_values(:guardrails) + (Array(assistant.guardrails) + item_values(:guardrails)).uniq end def array_change(field, values) @@ -144,6 +146,21 @@ class Captain::AssistantMigration::DraftApplier scenario_candidates.filter_map { |candidate| candidate[:response_guideline].presence } end + def faq_responses_change + faq_applier.changes + end + + def apply_faq_response_changes(changes) + faq_applier.apply(changes) + end + + def faq_applier + @faq_applier ||= Captain::AssistantMigration::FaqApplier.new( + assistant: assistant, + candidates: normalized_faq_document_candidates + ) + end + def scenario_tool_ids(tool_ids) Array(tool_ids).filter_map { |tool_id| tool_id.to_s.squish.presence }.uniq end @@ -186,7 +203,7 @@ class Captain::AssistantMigration::DraftApplier candidate = candidate.deep_symbolize_keys question = candidate[:question].to_s.squish - answer = candidate[:answer].to_s.squish + answer = candidate[:answer].to_s.strip raise ArgumentError, 'FAQ document candidates must include a question and answer' if question.blank? || answer.blank? { 'question' => question, 'answer' => answer } diff --git a/enterprise/app/services/captain/assistant_migration/faq_applier.rb b/enterprise/app/services/captain/assistant_migration/faq_applier.rb new file mode 100644 index 000000000..dcd526a5a --- /dev/null +++ b/enterprise/app/services/captain/assistant_migration/faq_applier.rb @@ -0,0 +1,36 @@ +class Captain::AssistantMigration::FaqApplier + pattr_initialize [:assistant!, :candidates!] + + def changes + @changes ||= candidates.each_with_object({ create: [] }) do |candidate, result| + categorize(candidate, result) + end.compact_blank.presence + end + + def apply(changes) + Array(changes[:create]).each do |candidate| + assistant.responses.create!(candidate.slice('question', 'answer', 'status')) + end + end + + private + + def categorize(candidate, result) + existing_answers = assistant.responses.approved.where(question: candidate['question']).pluck(:answer) + planned_answers = result[:create].filter_map do |response| + response['answer'] if response['question'] == candidate['question'] + end + answers = existing_answers + planned_answers + + ensure_no_conflict!(candidate, answers) + return if answers.include?(candidate['answer']) + + result[:create] << candidate.merge('status' => 'approved') + end + + def ensure_no_conflict!(candidate, answers) + return if answers.all?(candidate['answer']) + + raise ArgumentError, "FAQ candidate conflicts with an existing FAQ: #{candidate['question']}" + end +end diff --git a/enterprise/app/services/captain/assistant_migration/instruction_auditor.rb b/enterprise/app/services/captain/assistant_migration/instruction_auditor.rb new file mode 100644 index 000000000..023ab741f --- /dev/null +++ b/enterprise/app/services/captain/assistant_migration/instruction_auditor.rb @@ -0,0 +1,48 @@ +class Captain::AssistantMigration::InstructionAuditor < Captain::BaseTaskService + AUDITOR_MODEL = 'gpt-5.2'.freeze + pattr_initialize [:assistant!, :source_payload!, :draft!, :available_additions!] + + def perform + make_api_call( + model: AUDITOR_MODEL, + messages: messages, + schema: Captain::AssistantMigration::InstructionAuditorSchema.for(available_additions) + ) + end + + private + + def account + assistant.account + end + + def messages + [ + { role: 'system', content: system_prompt }, + { + role: 'user', + content: JSON.pretty_generate(source: source_payload, generated_draft: draft, available_additions: available_additions) + } + ] + end + + def system_prompt + Captain::PromptRenderer.render('instruction_auditor') + end + + def event_name + 'assistant_migration_instruction_auditor' + end + + def captain_tasks_enabled? + true + end + + def counts_toward_usage? + false + end + + def build_follow_up_context? + false + end +end diff --git a/enterprise/app/services/captain/assistant_migration/instruction_auditor_schema.rb b/enterprise/app/services/captain/assistant_migration/instruction_auditor_schema.rb new file mode 100644 index 000000000..78230ba29 --- /dev/null +++ b/enterprise/app/services/captain/assistant_migration/instruction_auditor_schema.rb @@ -0,0 +1,52 @@ +class Captain::AssistantMigration::InstructionAuditorSchema < RubyLLM::Schema + STRING_ARRAYS = { + response_guidelines: ['Missing active behavior to append to the generated response guidelines.', 10], + guardrails: ['Missing active boundaries or prohibitions to append to the generated guardrails.', 10], + needs_review: ['Missing source behavior blocked by an unavailable tool or runtime capability.', 10] + }.freeze + + def self.for(available_additions) + Class.new(RubyLLM::Schema).tap do |schema| + add_string_arrays(schema, available_additions) + add_scenarios(schema, available_additions[:scenario_candidates]) + add_faqs(schema, available_additions[:faq_document_candidates]) + end + end + + def self.add_string_arrays(schema, available_additions) + STRING_ARRAYS.each do |name, (description, limit)| + next unless available_additions[name].positive? + + schema.array(name, description: description, max_items: [available_additions[name], limit].min, of: :string) + end + end + + def self.add_scenarios(schema, available) + return unless available.positive? + + schema.array :scenario_candidates, + description: 'Missing distinct multi-step workflows to append to the generated scenario candidates.', + max_items: [available, 5].min do + object do + string :title, max_length: 80 + string :description, max_length: 500 + string :instruction, max_length: 2000 + string :response_guideline, max_length: 1000 + array :tool_ids, max_items: 10, of: :string + end + end + end + + def self.add_faqs(schema, available) + return unless available.positive? + + schema.array :faq_document_candidates, + description: 'Missing factual product or business knowledge to append to the pending FAQ candidates.', + max_items: [available, 15].min do + object do + string :question, max_length: 255 + string :answer, max_length: 2000 + end + end + end +end diff --git a/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb b/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb index 989989855..6efaf54cd 100644 --- a/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb +++ b/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb @@ -6,15 +6,26 @@ class Captain::AssistantMigration::InstructionClassifier < Captain::BaseTaskServ pattr_initialize [:assistant!] def perform - response = make_api_call(model: CLASSIFIER_MODEL, messages: messages, schema: RESPONSE_SCHEMA) - return error_response(response) if response[:error] + classifier_response = make_api_call(model: CLASSIFIER_MODEL, messages: messages, schema: RESPONSE_SCHEMA) + return error_response(classifier_response) if classifier_response[:error] + + generated_draft = normalized_payload(classifier_response[:message]) + auditor_response = Captain::AssistantMigration::InstructionAuditor.new( + assistant: assistant, + source_payload: assistant_payload, + draft: generated_draft, + available_additions: available_additions(generated_draft) + ).perform + return error_response(auditor_response) if auditor_response[:error] { assistant: assistant_metadata, - draft: normalized_payload(response[:message]), - usage: response[:usage], - request_messages: response[:request_messages] + draft: audited_payload(generated_draft, auditor_response[:message]), + usage: combined_usage(classifier_response, auditor_response), + request_messages: classifier_response[:request_messages] } + rescue ArgumentError => e + error_response(error: e.message, request_messages: auditor_response&.dig(:request_messages)) end private @@ -101,15 +112,49 @@ class Captain::AssistantMigration::InstructionClassifier < Captain::BaseTaskServ scenario_candidates: [], conversation_messages: {}, faq_document_candidates: [], - needs_review: [], - classification_notes: [] + needs_review: [] ) end + def combined_usage(*responses) + %w[prompt_tokens completion_tokens total_tokens].index_with do |key| + responses.sum { |response| response.dig(:usage, key).to_i } + end + end + + def available_additions(draft) + { + response_guidelines: 20 - draft[:response_guidelines].length, + guardrails: 20 - draft[:guardrails].length, + scenario_candidates: 15 - draft[:scenario_candidates].length, + faq_document_candidates: 25 - draft[:faq_document_candidates].length, + needs_review: 20 - draft[:needs_review].length + } + end + + def audited_payload(generated_draft, audit_message) + audit = audit_message.is_a?(Hash) ? audit_message.deep_symbolize_keys : {} + generated_draft.merge( + response_guidelines: merged_items(generated_draft, audit, :response_guidelines, 20), + guardrails: merged_items(generated_draft, audit, :guardrails, 20), + scenario_candidates: merged_items(generated_draft, audit, :scenario_candidates, 15), + faq_document_candidates: merged_items(generated_draft, audit, :faq_document_candidates, 25), + needs_review: merged_items(generated_draft, audit, :needs_review, 20) + ) + end + + def merged_items(generated_draft, audit, key, limit) + items = (Array(generated_draft[key]) + Array(audit[key])).uniq + raise ArgumentError, "Audited #{key} exceeds #{limit} items" if items.length > limit + + items + end + def assistant_metadata # rubocop:disable Metrics/AbcSize { id: assistant.id, name: assistant.name, + description: assistant.description.to_s, account_id: assistant.account_id, account_name: assistant.account.name, inbox_count: assistant.captain_inboxes.size, diff --git a/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb b/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb index 3e42bb49d..abbee3779 100644 --- a/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb +++ b/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb @@ -68,8 +68,8 @@ class Captain::AssistantMigration::InstructionClassifierSchema < RubyLLM::Schema end array :faq_document_candidates, - description: 'Pending FAQ candidates for factual or product-specific knowledge such as pricing, policy, setup, troubleshooting, ' \ - 'or operational details. These candidates remain inactive until reviewed and approved.', + description: 'FAQ candidates for reusable query-dependent facts such as pricing, policy, setup, troubleshooting, ' \ + 'or operational details.', max_items: 25 do object do string :question, @@ -86,6 +86,4 @@ class Captain::AssistantMigration::InstructionClassifierSchema < RubyLLM::Schema description: 'Unclear, conflicting, risky, duplicated, or uncertain content that needs human review. ' \ 'Include the reason in the item text.', max_items: 20 - - array :classification_notes, description: 'Short notes about important migration decisions or risks.', max_items: 10, of: :string end diff --git a/enterprise/lib/captain/prompts/instruction_auditor.liquid b/enterprise/lib/captain/prompts/instruction_auditor.liquid new file mode 100644 index 000000000..027b0a978 --- /dev/null +++ b/enterprise/lib/captain/prompts/instruction_auditor.liquid @@ -0,0 +1,69 @@ +You are the second and final content-coverage pass for a Captain V1-to-V2 assistant migration. + +The input contains the original source data and an already structured generated_draft. Return only missing items to append to that draft, +matching the provided audit schema. Empty arrays mean no addition is needed. Do not return a complete draft, critique, verdict, wrapper, +coverage report, or fields outside the schema. + +## Contract + +- This is a monotonic coverage audit. Never repeat, rewrite, replace, or delete content already present in generated_draft. +- Only source.instructions contains the legacy custom instructions being migrated. Other source fields are existing runtime context. +- Existing response guidelines, guardrails, scenarios, and configured welcome/handoff/resolution messages remain active and are preserved. +- Use only information in the input. Never add plausible facts, steps, links, tools, triggers, or policies. +- Preserve the source language and exact names, trigger values, thresholds, exceptions, links, prices, dates, and ordering requirements. +- Consolidate related missing requirements into complete standalone additions. Schema limits are ceilings, not targets. +- available_additions gives the exact remaining capacity for each destination. Never return more additions than that capacity, and never + return a field omitted from the response schema. +- Treat semantically equivalent content as already covered even when wording differs. Do not add stylistic restatements or stronger versions + of behavior that is already present. If an existing array is near its maximum, add only unquestionably missing source requirements and + combine related missing requirements into one complete addition. + +## Coverage Audit + +Review source.instructions clause by clause against all fields in generated_draft. + +1. Missing Active Behavior + - Add every source-required action, prohibition, language rule, verification, trigger, exception, ordering rule, escalation condition, + or workflow that is not already active in generated response_guidelines, guardrails, or scenario response_guidelines. + - Words such as always, immediately, never, only, before, after, unless, and except are mandatory. + - FAQ question and answer text is active factual knowledge, but it does not preserve mandatory behavior. + If mandatory behavior appears only there, add the missing active guideline or guardrail. needs_review is inactive. + - Keep the minimum factual trigger, threshold, allowlist, or exception needed to execute the action or enforce the prohibition. + - When factual policy contains a mandatory boundary, add the boundary as an active guardrail while leaving the full policy in FAQ. + Examples include never promising refunds outside a stated window and never recommending cooking a product that must remain raw. + - A conditional response procedure remains active behavior. For example, acknowledging a known problem and explaining that the team is + working on it is active; the current known-problem status itself is factual FAQ knowledge. + +2. Missing FAQ Knowledge + - Add reusable query-dependent facts absent from faq_document_candidates: prices, limits, locations, product capabilities, exact links, + policies, setup steps, troubleshooting knowledge, schedules, and operational details. + - “If asked, tell/inform/explain/send” is a factual answer, not a separate active workflow, unless it also requires another action or + imposes a prohibition. + - Questions must concern the product or business. Answers must not contain tool use, routing, escalation, internal workflows, or + assistant-behavior instructions. + - Do not add FAQs for missing placeholders, generic assistant capabilities, or facts already covered by an existing candidate. + +3. Missing Scenario Candidates + - Add a scenario only when a source-defined multi-step intake, qualification, troubleshooting, booking, recommendation, lead-capture, + or fulfillment workflow is absent from both scenario candidates and equivalent active handling. + - Do not add scenarios for tone, factual answers, simple handoff triggers, or one-step clarification. + - Every added scenario needs a complete same-language response_guideline under 1,000 characters and only supplied tool IDs. + +4. Missing Review Notes + - Add needs_review only when a source-defined behavior or workflow cannot run because a required named tool or runtime signal is unavailable. + - Do not require words such as “must” or “always”; preserve any unavailable customer-facing workflow for review. + - Name the missing capability and the affected source behavior precisely. Relevant gaps include historical-record lookup, timers or inactivity + detection, business-hours detection, and live-agent availability. + - A needs_review item never replaces representable behavior. Add every source-faithful action or boundary that can remain active, and add a + review note only for the portion blocked by the unavailable capability. + - Do not add review notes for wording cleanup, configured conversation messages, missing fixed copy, general uncertainty, or behavior already + covered by the generated draft. + +## Final Check + +- No mandatory action or prohibition remains FAQ-only. +- No reusable factual knowledge is absent from FAQ candidates. +- No source-defined workflow blocked by an unavailable capability is omitted from needs_review. +- No addition duplicates content already active or pending. +- No unsupported behavior, fact, tool, link, or resolution is introduced. +- Return only the missing additions matching the audit schema. diff --git a/enterprise/lib/captain/prompts/instruction_classifier.liquid b/enterprise/lib/captain/prompts/instruction_classifier.liquid index abc58ff60..a1d66a28f 100644 --- a/enterprise/lib/captain/prompts/instruction_classifier.liquid +++ b/enterprise/lib/captain/prompts/instruction_classifier.liquid @@ -1,137 +1,114 @@ -You are migrating Captain assistant instructions into a structured configuration. +You are migrating a Captain V1 assistant into Captain V2. + +The original custom instructions remain stored unchanged. Your job is only to derive the V2 fields below: -Classify the existing assistant instructions into these sections: 1. Business/Product Context 2. Response Guidelines 3. Guardrails -4. Scenario Candidates +4. Scenario Candidates with flattened Response Guidelines 5. Conversation Messages -6. FAQs/Documents Candidates -7. Needs Review +6. FAQ Candidates +7. Needs Review Notes -## General Rules +## Core Rules -- Preserve behavior as closely as possible. -- Do not duplicate the same content across sections. -- Return clean migrated values only. Do not include source excerpts, source labels, citations, or "Source:" text in any migrated field. -- Do not rewrite customer-facing message copy unless necessary to classify an exact copy from instructions. -- Do not include confidence labels, review labels, bracketed reviewer comments, or schema labels inside migrated values. -- For Business/Product Context, Response Guidelines, and Guardrails, return each item as a plain standalone sentence. - Do not prefix items with numbers, bullets, section labels, or list markers such as "1.", "-", or "*". -- When several instructions share the same trigger, condition, or subject, combine them into one concise item instead - of repeating the same trigger across multiple items. Preserve every required action, prohibition, and routing - outcome from the source instruction when combining. -- If unsure, place content in Needs Review and include the reason in that item. -- Return data that matches the provided schema. +- Preserve every customer-facing behavior from the custom instructions. Do not invent, reverse, weaken, or silently omit requirements. +- Treat words such as always, immediately, never, only, before, after, unless, and except as mandatory. +- Preserve exact triggers, exceptions, ordering, verification steps, allowlists, escalation conditions, and outcomes. +- Schema limits are ceilings, not targets. Consolidate related requirements into complete standalone items. +- Prefer fewer complete items over one item per source sentence. Combine related tone, style, formatting, source, and escalation rules. + If response_guidelines or guardrails would reach its maximum item count, consolidate them and recheck that no source behavior was displaced. +- The custom instructions define behavior. The existing description, config messages, feature settings, and tools are runtime context. +- Do not copy existing config values into generated fields or create review work merely because an existing config field is present or absent. +- Use only information in the input. Return clean values without source labels, reviewer comments, confidence labels, or citations to the source prompt. +- Avoid duplicating content across fields, except for the minimal condition, threshold, or exception required to keep mandatory behavior active + while its supporting factual explanation is stored in a FAQ candidate. Scenario response guidelines are flattened automatically, so do not + also copy them into response_guidelines. ## Business/Product Context -- Business/Product Context maps to the root assistant description and is injected into the root orchestrator prompt. -- Return exactly one Business/Product Context item. -- Start with the existing assistant description and preserve its meaning. -- Enrich it only with relevant business or product context found in the custom instructions. -- Produce one coherent description rather than appending a second context block or repeating the existing description. -- Keep it at most 500 characters because that is the assistant description limit in the UI and model. -- Prefer roughly 300-450 characters when the source needs detail, leaving room below the hard limit. -- Finish the description cleanly. Never end mid-word, mid-clause, after an opening bracket, or with a dangling separator. -- Make it a compact summary of assistant identity, product scope, high-level mission, and high-level source or routing priorities. -- Do not include detailed workflows, step-by-step procedures, long support-scope inventories, attribute glossaries, - policy details, scenario-specific handling, tool instructions, or customer-facing message copy. +- Return exactly one coherent description of at most 500 characters. +- Preserve the existing description and enrich it only with identity, product scope, mission, and high-level business context. +- Do not put workflows, policies, response rules, factual inventories, or message copy in the description. +- Finish cleanly; never truncate a word, clause, or sentence. -## Conversation Messages +## Response Guidelines and Guardrails -- Existing welcome_message, handoff_message, and resolution_message config values are provided separately. -- Treat welcome_message, handoff_message, and resolution_message as conversation message config fields. -- Extract exact welcome, handoff, or resolution message copy from instructions into conversation_messages when present. -- Only classify handoff copy as conversation_messages.handoff_message when it is generic enough to reuse for any human handoff. -- If handoff copy is scenario-specific, keep it inside that scenario instruction; if it is only a rule about when or how to hand off, classify it as a Response Guideline or Guardrail. -- Do not extract a conversation message from an instruction about what to say, from a placeholder template, - from conditional copy, from role/team-specific copy, or from text that only applies inside one workflow. -- If a message contains placeholders such as a blank name, team name, bracketed variable, business-hours state, - or dynamic runtime condition, do not place it in conversation_messages. Keep it in the relevant workflow or - Needs Review. -- Do not copy message values from existing config into conversation_messages. -- Do not decide whether existing config values should be overwritten. Migration code handles applying extracted - conversation_messages only when the corresponding config value is blank. +- Response Guidelines are active behavior: tone, customer language, formatting, clarification, verification, information collection, + escalation actions, and any minimal factual condition required to perform them correctly. +- Guardrails are active boundaries: prohibitions, source restrictions, safety limits, refusal rules, mandatory transfer triggers, + and things the assistant must not do. +- A source rule that says to ask, collect, verify, compare, refuse, route, escalate, transfer, or follow steps must stay active + in Response Guidelines, Guardrails, or a flattened Scenario Guideline. A FAQ cannot implicitly preserve an action. +- Preserve exact behavioral trigger values when they control an action. For example, an error code that requires immediate + transfer belongs in an active guideline or guardrail. +- Do not emit contradictory language rules. An explicit instruction to reply in the customer's language overrides a descriptive + language label in the assistant description. +- Put query-dependent facts in FAQ candidates. Prices, limits, locations, feature availability, product capabilities, links, + policy answers, setup steps, and troubleshooting knowledge remain facts when phrased as "tell", "inform", "explain", or "send". +- Mandatory prohibitions are not FAQ-only. When a factual policy includes required or forbidden behavior, keep the prohibition active + with every condition, threshold, and exception needed to enforce it, and put the supporting policy explanation in a FAQ candidate. + For example, "never promise refunds after 30 days" remains an active guardrail with the 30-day threshold, while the refund policy + becomes a FAQ. Likewise, "never recommend cooking the product" remains an active guardrail while preparation guidance becomes a FAQ. +- Treat explicit policy boundaries such as "not guaranteed", "not allowed", "only available", or "only eligible" as behavioral + constraints even when the source states them as facts. Create an active guardrail that forbids promising or claiming an outcome + outside the stated condition, window, or exception, while keeping the complete policy in a FAQ candidate. +- Final test: move an item exclusively to FAQ candidates only when it answers a product question without requiring, forbidding, + or constraining assistant behavior. +- Factual values are allowed in active behavior when they select or constrain a required action or prohibition, such as error 5215 + requiring immediate transfer or a 30-day threshold after which the assistant must not promise a refund. +- When an action needs supporting facts, keep the action active and place the supporting facts in a FAQ candidate. + For example, actively require specialist-name verification and put the specialist roster in a FAQ candidate. +- Mandatory verification example: if the source provides a specialist roster and says to verify a name supplied by + the customer, output both (a) an active guideline requiring the name check and (b) a pending FAQ containing the roster. + The roster FAQ alone is incomplete because it does not tell the assistant to perform the check. +- When clarification depends on a fact, keep only the clarification/action in the guideline. Example: "clarify whether + they mean the legacy card or card deposits; transfer for deposit access" is active behavior, while the card's + discontinued status is FAQ knowledge. ## Scenario Candidates -- In the current architecture, a scenario becomes a specialized sub-agent with its own title, description, - instructions, and optional tools. -- During this migration, scenario candidates are also temporarily flattened into response guidelines so existing - assistant behavior is preserved before scenario records are created. -- For every scenario candidate, write a response_guideline that is the flattened version of that scenario for - the root assistant's response guidelines. -- The response_guideline must be in the same language as the original scenario or source instruction. -- The response_guideline must preserve the intended customer-visible behavior, trigger, information to collect, - and routing/escalation outcome. -- The response_guideline must not include tool syntax, tool:// links, markdown tool links, tool names, label - updates, priority updates, private-note instructions, custom-tool instructions, or internal implementation details. -- If the scenario uses internal tools such as labels, priorities, private notes, or custom tools, describe only - the customer-visible behavior and expected routing/escalation outcome in response_guideline. -- If human handoff is needed, describe it in natural language such as route/escalate/transfer to a human; do not - mention the handoff tool in response_guideline. -- Keep scenario titles, descriptions, instructions, and response_guidelines clear, self-contained, and reviewable. -- Only create scenario candidates for distinct user-intent workflows that should be routed to a specialized agent. - A candidate must be narrow enough to become a named specialist assistant with domain-specific handling instructions. -- Good scenario candidates include multi-step intake workflows, qualification flows, specialized troubleshooting - workflows, booking flows, lead-capture flows, recommendation flows, fulfillment workflows, or tool-use procedures - for a specific user intent. -- A scenario candidate should answer "yes" to this test: would a named specialist sub-agent improve handling - beyond the base assistant's global FAQ, guardrail, response-guideline, and human-handoff behavior? -- Do not create scenario candidates for global escalation rules, generic handoff policy, missing-information - behavior, source-boundary rules, refusal rules, tone, formatting, answer length, or one-step fallback behavior. -- Do not create scenario candidates whose main purpose is to escalate or hand off. "Identify the trigger, avoid - guessing, tell the user support will review, and hand off" is a guardrail/handoff boundary, not a scenario, - even though it contains multiple statements. -- Do create scenario candidates when the instructions define a concrete intake, qualification, troubleshooting, - booking, lead-capture, recommendation, or fulfillment workflow, even when the workflow eventually hands off - to a human. -- Do not create scenario candidates for simple routing triggers such as "user asks for a human", "immediately - hand off this category", or "route sales questions to the sales team" when there is no concrete workflow to run. -- Handoff behavior is a scenario candidate only when part of a larger intake, qualification, or specialized handling workflow. -- Global rules like "if not in docs, escalate", "ask one clarifying question", "do not answer account-specific - questions", or "tell the user support will review" belong in Guardrails or Response Guidelines, not Scenario Candidates. -- Broad buckets like "account-specific issue escalation", "unknown question escalation", "contact support", - "fallback to human", or "documentation unavailable" are not scenario candidates. +- Create a scenario candidate only for a distinct multi-step workflow that would genuinely benefit from a separate named specialist agent, + such as intake, qualification, troubleshooting, booking, recommendation, lead capture, or fulfillment. +- Do not create scenarios for tone, formatting, generic escalation, a simple handoff trigger, missing information, or a one-step factual answer. +- Do not create overlapping scenarios for the same intent, and do not create a scenario for a workflow the root assistant can handle with + one guideline plus FAQ lookup. +- Every scenario candidate must include a response_guideline in the source language. It must preserve the trigger, customer-visible + steps, information to collect, and escalation or completion outcome while omitting tool syntax and internal operations. +- Use a short, complete scenario title well below the schema limit; never truncate a word or phrase to make it fit. +- Scenario candidates remain pending metadata for later scenario creation. Their response_guideline is active immediately after apply. +- Use only tool IDs provided in available_agent_tools. Never invent or substitute a tool. -## Tool Use +## Conversation Messages -- If a scenario candidate requires tools, reference the available tool explicitly inside the scenario instruction - using markdown tool links such as [Handoff to Human](tool://handoff). -- Use only tool IDs listed in available_agent_tools. If a needed tool is unavailable or the workflow depends on - unavailable runtime data such as FAQ relevance scores or business-hours status, place it in Needs Review instead. -- Do not map an unavailable named tool to a different available tool. For example, do not treat FAQ Lookup as - Product Search, Order Status, website browsing, pricing lookup, agent availability, business-hours detection, - ticket creation, or custom-attribute assignment unless the instructions explicitly say that the available - tool provides that behavior. -- If a workflow cannot run without an unavailable tool or runtime signal, do not create a tool-backed scenario - for it. Preserve the instruction in Needs Review with the missing capability named. +- Extract only exact, globally reusable welcome, handoff, or resolution copy found in the custom instructions. +- Leave conditional, scenario-specific, placeholder-based, or merely suggested wording out of conversation_messages. +- Existing config messages remain active and are preserved. If source wording has the same intent, keep the existing config message. +- Migration applies extracted copy only when the corresponding existing config field is blank. -## FAQs/Documents Candidates +## FAQ Candidates -- Convert factual or product-specific knowledge into pending FAQ candidates with a natural customer question and a self-contained answer. -- FAQ candidates are review-stage data only. They are not active assistant knowledge until a human reviews and approves them. -- Use only facts stated in the existing instructions. Do not invent, generalize, update, or fill in missing details. -- Preserve exact prices, limits, dates, time zones, conditions, exceptions, product names, and operational details in the answer. -- Write each question as a standalone question a customer might naturally ask. Make it specific enough to retrieve the corresponding answer. -- Write each answer so it fully answers its question without relying on another FAQ candidate or surrounding context. -- Split unrelated facts into separate candidates. Keep related conditions and exceptions together when separating them would make an answer incomplete. -- Do not create FAQ candidates about what the assistant should say or do, how it should use sources or tools, when it should route or escalate, - or which exact message it should send. Classify those as Response Guidelines, Guardrails, Scenario Candidates, Conversation Messages, - or Needs Review as appropriate. -- FAQ questions must ask about the product or business, not about the assistant. Do not write questions such as "What should the assistant answer?", - "What should I say?", "Which source should the assistant use?", or "Which tool should be called?". -- FAQ answers must contain customer-facing knowledge, not instructions to call tools, inspect internal data, update records, transfer conversations, - or follow internal workflows. -- When factual sources conflict and the instructions do not explicitly establish which fact overrides the others, put the conflict in Needs Review - instead of creating an FAQ candidate. Use an explicitly stated override or superseding fact when one is present. -- Only factual or product-specific knowledge should become FAQs/Documents candidates. -- Generic capability statements such as "answer product questions", "help with billing", - "troubleshoot common issues", or "direct to documentation" are not FAQ/document candidates. - Put them in Business/Product Context or Response Guidelines when useful. -- Product facts, pricing, policies, setup steps, troubleshooting facts, support hours, emergency contacts, - and operational details should become pending FAQ candidates, not Response Guidelines or trusted approved knowledge. -- Do not create FAQ/document candidates for topic labels or unsupported capabilities when the factual content is - missing. Put "pricing details are needed", "same-day delivery schedule details are needed", or similar gaps in - Needs Review instead. +- Convert reusable query-dependent facts into natural customer questions with self-contained answers. +- Use only facts stated in the custom instructions. Preserve exact prices, limits, dates, links, conditions, exceptions, and product names. +- Keep related conditions together; split unrelated facts. Do not duplicate a full FAQ answer in active guidelines or guardrails; + repeat only the minimal condition, threshold, or exception required to enforce mandatory behavior. +- FAQ questions must be about the product or business, not about what the assistant should do. +- FAQ answers must not contain tool use, internal workflows, routing, escalation, or message-copy instructions. +- If facts conflict without a clear specific or later override, omit the unsafe FAQ rather than inventing a resolution. + +## Classification Order + +1. Extract query-dependent knowledge and supporting policy explanations into FAQ candidates first, without removing mandatory behavior. +2. Create guidelines and guardrails from the required behavior, including the minimal condition, threshold, or exception needed to enforce it; + do not repeat the rest of a FAQ answer. +3. Create scenario candidates only from remaining distinct specialist workflows; do not repeat their flattened behavior elsewhere. +4. Check once more that active fields contain no standalone product answers and that every mandatory action and prohibition remains active. + +## Needs Review Notes + +- Use needs_review only for a concrete source conflict or a source-defined behavior or workflow that requires an unavailable capability. +- Do not require mandatory wording before preserving an unavailable customer-facing workflow for review. +- Do not use it for wording cleanup, duplicated instructions, missing fixed message copy, existing config values, or general uncertainty. +- needs_review is informational metadata only; it is not an approval status or apply gate. + +Return data matching the provided schema. diff --git a/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb b/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb index 0e2f420ac..b92a1a6a6 100644 --- a/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb +++ b/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb @@ -23,7 +23,7 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do let(:faq_document_candidate) do { 'question' => 'When is support available?', - 'answer' => 'Support is available Monday to Friday.' + 'answer' => "Support is available Monday to Friday.\n\nUrgent requests are handled by the on-call team." } end let(:draft) do @@ -46,11 +46,15 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do expect(result.dig(:changes, :response_guidelines, :to)).to include( 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.' ) + expect(result.dig(:changes, :faq_responses, :create)).to contain_exactly( + faq_document_candidate.merge('status' => 'approved') + ) expect(assistant.reload.config).not_to have_key('assistant_migration') + expect(assistant.responses.count).to eq(0) expect(assistant.scenarios.count).to eq(0) end - it 'stores scenario candidates in assistant config and flattens them into response guidelines' do + it 'stores scenario and FAQ candidates and creates approved FAQ responses' do described_class.new(assistant: assistant, draft: draft, dry_run: false).perform assistant.reload @@ -62,8 +66,55 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do expect(assistant.response_guidelines).to include( 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.' ) - expect(assistant.response_guidelines).not_to include(faq_document_candidate['answer']) + expect(assistant.responses).to contain_exactly( + have_attributes( + question: faq_document_candidate['question'], + answer: faq_document_candidate['answer'], + status: 'approved' + ) + ) expect(assistant.scenarios.count).to eq(0) + + expect do + described_class.new(assistant: assistant, draft: draft, dry_run: false).perform + end.not_to(change { assistant.responses.count }) + end + + it 'leaves pending FAQ responses untouched' do + pending_response = assistant.responses.create!( + question: faq_document_candidate['question'], + answer: faq_document_candidate['answer'], + status: :pending + ) + + described_class.new(assistant: assistant, draft: draft, dry_run: false).perform + + expect(pending_response.reload).to be_pending + expect(assistant.responses.approved).to contain_exactly( + have_attributes( + question: faq_document_candidate['question'], + answer: faq_document_candidate['answer'] + ) + ) + end + + it 'rejects conflicting FAQ answers within the same draft' do + conflicting_draft = draft.merge( + faq_document_candidates: [ + faq_document_candidate, + { + 'question' => "When is support\navailable?", + 'answer' => 'Support is available every day.' + } + ] + ) + + expect do + described_class.new(assistant: assistant, draft: conflicting_draft, dry_run: true).perform + end.to raise_error(ArgumentError, 'FAQ candidate conflicts with an existing FAQ: When is support available?') + + expect(assistant.responses.count).to eq(0) + expect(assistant.config).not_to have_key('assistant_migration') end it 'rejects stale drafts whose FAQ candidates use the old string format' do @@ -87,8 +138,12 @@ RSpec.describe Captain::AssistantMigration::DraftApplier do assistant.reload expect(assistant.description).to eq('Support assistant for Test Product.') - expect(assistant.response_guidelines).to include('Be concise.') - expect(assistant.guardrails).to eq(['Do not guess.']) + expect(assistant.response_guidelines).to include( + 'Use plain language.', + 'Be concise.', + 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.' + ) + expect(assistant.guardrails).to contain_exactly('Do not disclose internal notes.', 'Do not guess.') expect(assistant.config.dig('assistant_migration', 'original_values')).to include( 'name' => assistant.name, 'description' => 'Existing assistant description.', diff --git a/spec/enterprise/services/captain/assistant_migration/instruction_classifier_spec.rb b/spec/enterprise/services/captain/assistant_migration/instruction_classifier_spec.rb new file mode 100644 index 000000000..448433f17 --- /dev/null +++ b/spec/enterprise/services/captain/assistant_migration/instruction_classifier_spec.rb @@ -0,0 +1,88 @@ +require 'rails_helper' + +RSpec.describe Captain::AssistantMigration::InstructionClassifier do + describe Captain::AssistantMigration::InstructionClassifierSchema do + it 'does not request classification notes' do + expect(described_class.as_json.to_s).not_to include('classification_notes') + end + end + + describe 'classifier prompt' do + it 'keeps the model focused on active behavior and approved FAQ candidates' do + prompt = Captain::PromptRenderer.render('instruction_classifier') + + expect(prompt).to include( + 'The original custom instructions remain stored unchanged', + 'A FAQ cannot implicitly preserve an action', + 'Scenario candidates remain pending metadata', + 'Convert reusable query-dependent facts into natural customer questions', + 'an error code that requires immediate', + 'actively require specialist-name verification', + 'Mandatory prohibitions are not FAQ-only', + 'never promise refunds after 30 days', + 'never recommend cooking the product', + 'Treat explicit policy boundaries', + 'outside the stated condition, window, or exception', + 'source-defined behavior or workflow that requires an unavailable capability', + 'Do not require mandatory wording', + 'every mandatory action and prohibition remains active' + ) + end + end + + describe Captain::AssistantMigration::InstructionAuditorSchema do + it 'only permits additions that fit in the generated draft' do + schema = described_class.for( + response_guidelines: 0, + guardrails: 2, + scenario_candidates: 1, + faq_document_candidates: 3, + needs_review: 4 + ).new.to_json_schema[:schema] + + expect(schema[:properties]).not_to have_key(:response_guidelines) + expect(schema.dig(:properties, :guardrails, :maxItems)).to eq(2) + expect(schema.dig(:properties, :scenario_candidates, :maxItems)).to eq(1) + expect(schema.dig(:properties, :faq_document_candidates, :maxItems)).to eq(3) + expect(schema.dig(:properties, :needs_review, :maxItems)).to eq(4) + end + end + + describe 'auditor prompt' do + it 'adds missing coverage without replacing the generated draft' do + prompt = Captain::PromptRenderer.render('instruction_auditor') + + expect(prompt).to include( + 'This is a monotonic coverage audit', + 'Never repeat, rewrite, replace, or delete content', + 'If mandatory behavior appears only there, add the missing active guideline or guardrail', + 'available_additions gives the exact remaining capacity', + 'A needs_review item never replaces representable behavior', + 'No mandatory action or prohibition remains FAQ-only' + ) + end + end + + describe 'audited payload' do + it 'appends a review note for an unavailable runtime capability' do + service = described_class.new(assistant: instance_double(Captain::Assistant)) + generated_draft = { + response_guidelines: [], + guardrails: [], + scenario_candidates: [], + faq_document_candidates: [], + needs_review: ['Existing conflict'] + } + + result = service.send( + :audited_payload, + generated_draft, + { needs_review: ['Order-status lookup requires an unavailable account-history tool.'] } + ) + + expect(result[:needs_review]).to eq( + ['Existing conflict', 'Order-status lookup requires an unavailable account-history tool.'] + ) + end + end +end