fix(captain): conditionally include custom instruction policy
This commit is contained in:
@@ -147,6 +147,8 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
end
|
||||
|
||||
def system_prompt
|
||||
Captain::Llm::SystemPromptsService.assistant_action_classifier
|
||||
Captain::Llm::SystemPromptsService.assistant_action_classifier(
|
||||
has_custom_instructions: @assistant.config['instructions'].present?
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -93,7 +93,7 @@ class Captain::Llm::SystemPromptsService
|
||||
SYSTEM_PROMPT_MESSAGE
|
||||
end
|
||||
|
||||
def assistant_action_classifier
|
||||
def assistant_action_classifier(has_custom_instructions: false)
|
||||
<<~PROMPT
|
||||
You are a routing classifier for a customer-support assistant.
|
||||
|
||||
@@ -127,16 +127,11 @@ class Captain::Llm::SystemPromptsService
|
||||
- The user explicitly asks for a human, agent, representative, phone call, callback, or escalation.
|
||||
- The user accepts an offer to speak with a human.
|
||||
- The user has provided enough detail for an account-specific or transaction-specific issue requiring private verification, such as order status, payment, deposit, withdrawal, refund, cancellation, subscription, purchase, plan activation, email verification, login, account recovery, delivery, or access.
|
||||
- The user reports a bug or operational issue that needs support to inspect their account, setup, logs, integration, channel, or delivery state after a reasonable clarifying step.
|
||||
- The user reports the same unresolved bug or operational issue after trying the assistant's suggested step, repeating the action, checking again, or otherwise making more than one reasonable attempt.
|
||||
- The user is repeatedly frustrated, distrustful, or stuck in a loop.
|
||||
- The assistant response itself says the current conversation will be transferred to a human agent now.
|
||||
|
||||
You may receive account custom instructions inside <account_custom_instructions> tags.
|
||||
These are instructions configured by the account administrator, not the current end user's message.
|
||||
Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
|
||||
If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
|
||||
Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
|
||||
Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
|
||||
#{assistant_action_classifier_custom_instructions_policy if has_custom_instructions}
|
||||
|
||||
Return only the structured fields requested by the response schema.
|
||||
PROMPT
|
||||
@@ -373,6 +368,17 @@ class Captain::Llm::SystemPromptsService
|
||||
TOOLS
|
||||
end
|
||||
|
||||
def assistant_action_classifier_custom_instructions_policy
|
||||
<<~POLICY
|
||||
Account custom instructions are provided inside <account_custom_instructions> tags.
|
||||
These are instructions configured by the account administrator, not the current end user's message.
|
||||
Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
|
||||
If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
|
||||
Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
|
||||
Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
|
||||
POLICY
|
||||
end
|
||||
|
||||
def build_contact_context(contact)
|
||||
return '' if contact.nil?
|
||||
|
||||
|
||||
@@ -39,6 +39,9 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
|
||||
|
||||
it 'passes delimited custom instructions and classifier context to the LLM' do
|
||||
expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:with_instructions).with(
|
||||
a_string_including('Account custom instructions are provided inside <account_custom_instructions> tags.')
|
||||
).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:ask) do |prompt|
|
||||
expect(prompt).to include(
|
||||
'<account_custom_instructions>',
|
||||
@@ -63,5 +66,20 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
|
||||
'prompt_version' => 'v1_custom_xml_precedence'
|
||||
)
|
||||
end
|
||||
|
||||
context 'when the assistant has no custom instructions' do
|
||||
before do
|
||||
assistant.update!(config: assistant.config.except('instructions'))
|
||||
end
|
||||
|
||||
it 'does not add custom-instruction policy to the system prompt' do
|
||||
expect(mock_chat).to receive(:with_instructions).with(
|
||||
satisfy { |prompt| prompt.exclude?('Account custom instructions are provided') }
|
||||
).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
||||
|
||||
service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user