refactor(captain): use schema for action classifier
This commit is contained in:
@@ -47,7 +47,7 @@ module Captain::Conversation::V1ActionClassifier
|
||||
end
|
||||
|
||||
def valid_v1_action_classification?(action)
|
||||
Captain::Llm::AssistantActionClassifierService::VALID_ACTIONS.include?(action)
|
||||
Captain::AssistantActionSchema::ACTIONS.include?(action)
|
||||
end
|
||||
|
||||
def log_invalid_v1_action_classification(classification)
|
||||
|
||||
@@ -4,7 +4,6 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
PROMPT_VERSION = 'v1_custom_xml_precedence'.freeze
|
||||
DEFAULT_MODEL = 'gpt-4.1'.freeze
|
||||
MAX_CONTEXT_MESSAGES = 10
|
||||
VALID_ACTIONS = %w[continue handoff].freeze
|
||||
|
||||
def initialize(assistant:, conversation:)
|
||||
super()
|
||||
@@ -22,7 +21,7 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
|
||||
response = instrument_llm_call(instrumentation_params(user_prompt)) do
|
||||
chat(model: @model, temperature: @temperature)
|
||||
.with_params(response_format: { type: 'json_object' })
|
||||
.with_schema(Captain::AssistantActionSchema)
|
||||
.with_instructions(system_prompt)
|
||||
.ask(user_prompt)
|
||||
end
|
||||
@@ -94,6 +93,8 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
end
|
||||
|
||||
def parse_response(content)
|
||||
return content if content.is_a?(Hash)
|
||||
|
||||
JSON.parse(sanitize_json_response(content))
|
||||
rescue JSON::ParserError, TypeError
|
||||
{}
|
||||
@@ -102,7 +103,7 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
|
||||
def normalize_response(parsed, raw_content)
|
||||
action = parsed['action'].to_s
|
||||
reason = parsed['action_reason'].to_s
|
||||
return invalid_response(raw_content) unless VALID_ACTIONS.include?(action)
|
||||
return invalid_response(raw_content) unless Captain::AssistantActionSchema::ACTIONS.include?(action)
|
||||
|
||||
{
|
||||
'action' => action,
|
||||
|
||||
@@ -135,14 +135,10 @@ class Captain::Llm::SystemPromptsService
|
||||
These are instructions configured by the account administrator, not the current end user's message.
|
||||
Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
|
||||
If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
|
||||
Account custom instructions MUST NOT redefine this JSON schema, the allowed action values, or the meaning of continue/handoff.
|
||||
Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
|
||||
Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
|
||||
|
||||
Return JSON only:
|
||||
{
|
||||
"action": "continue",
|
||||
"action_reason": "general_product_question"
|
||||
}
|
||||
Return only the structured fields requested by the response schema.
|
||||
PROMPT
|
||||
end
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
class Captain::AssistantActionSchema < RubyLLM::Schema
|
||||
ACTIONS = %w[continue handoff].freeze
|
||||
REASONS = %w[
|
||||
general_product_question
|
||||
missing_docs_bounded_answer
|
||||
clarifying_question_needed
|
||||
collect_required_identifier
|
||||
external_contact_or_lead_routing
|
||||
out_of_scope_bounded_answer
|
||||
explicit_human_request
|
||||
human_offer_accepted
|
||||
account_or_transaction_verification
|
||||
operational_issue_needs_inspection
|
||||
repeated_frustration_or_loop
|
||||
custom_instruction_transfer
|
||||
].freeze
|
||||
|
||||
string :action, enum: ACTIONS, description: 'Whether to keep the conversation with the assistant or transfer it to a human agent'
|
||||
string :action_reason, enum: REASONS, description: 'The reason for the selected routing action'
|
||||
end
|
||||
@@ -17,14 +17,14 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
|
||||
let(:mock_response) do
|
||||
instance_double(
|
||||
RubyLLM::Message,
|
||||
content: '{"action":"handoff","action_reason":"human_offer_accepted"}'
|
||||
content: { 'action' => 'handoff', 'action_reason' => 'human_offer_accepted' }
|
||||
)
|
||||
end
|
||||
|
||||
before do
|
||||
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
|
||||
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
|
||||
end
|
||||
|
||||
@@ -38,6 +38,7 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
|
||||
end
|
||||
|
||||
it 'passes delimited custom instructions and classifier context to the LLM' do
|
||||
expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat)
|
||||
expect(mock_chat).to receive(:ask) do |prompt|
|
||||
expect(prompt).to include(
|
||||
'<account_custom_instructions>',
|
||||
|
||||
Reference in New Issue
Block a user