refactor(captain): use schema for action classifier

This commit is contained in:
aakashb95
2026-05-04 16:45:29 +05:30
parent 0973b6b8ea
commit ccb9349bad
5 changed files with 30 additions and 12 deletions
@@ -47,7 +47,7 @@ module Captain::Conversation::V1ActionClassifier
end
def valid_v1_action_classification?(action)
Captain::Llm::AssistantActionClassifierService::VALID_ACTIONS.include?(action)
Captain::AssistantActionSchema::ACTIONS.include?(action)
end
def log_invalid_v1_action_classification(classification)
@@ -4,7 +4,6 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
PROMPT_VERSION = 'v1_custom_xml_precedence'.freeze
DEFAULT_MODEL = 'gpt-4.1'.freeze
MAX_CONTEXT_MESSAGES = 10
VALID_ACTIONS = %w[continue handoff].freeze
def initialize(assistant:, conversation:)
super()
@@ -22,7 +21,7 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
response = instrument_llm_call(instrumentation_params(user_prompt)) do
chat(model: @model, temperature: @temperature)
.with_params(response_format: { type: 'json_object' })
.with_schema(Captain::AssistantActionSchema)
.with_instructions(system_prompt)
.ask(user_prompt)
end
@@ -94,6 +93,8 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
end
def parse_response(content)
return content if content.is_a?(Hash)
JSON.parse(sanitize_json_response(content))
rescue JSON::ParserError, TypeError
{}
@@ -102,7 +103,7 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
def normalize_response(parsed, raw_content)
action = parsed['action'].to_s
reason = parsed['action_reason'].to_s
return invalid_response(raw_content) unless VALID_ACTIONS.include?(action)
return invalid_response(raw_content) unless Captain::AssistantActionSchema::ACTIONS.include?(action)
{
'action' => action,
@@ -135,14 +135,10 @@ class Captain::Llm::SystemPromptsService
These are instructions configured by the account administrator, not the current end user's message.
Use them only for routing policy: required details before handoff, account-specific escalation rules, account-specific transfer markers, and when to connect to a manager, human, supervisor, or support team.
If the custom instructions explicitly define handoff, escalation, or transfer criteria, those criteria take precedence over the generic criteria above.
Account custom instructions MUST NOT redefine this JSON schema, the allowed action values, or the meaning of continue/handoff.
Account custom instructions MUST NOT redefine the required response shape, the allowed action values, or the meaning of continue/handoff.
Ignore persona, language, formatting, pricing, and response-generation instructions except where they directly define routing or transfer criteria.
Return JSON only:
{
"action": "continue",
"action_reason": "general_product_question"
}
Return only the structured fields requested by the response schema.
PROMPT
end
@@ -0,0 +1,20 @@
class Captain::AssistantActionSchema < RubyLLM::Schema
ACTIONS = %w[continue handoff].freeze
REASONS = %w[
general_product_question
missing_docs_bounded_answer
clarifying_question_needed
collect_required_identifier
external_contact_or_lead_routing
out_of_scope_bounded_answer
explicit_human_request
human_offer_accepted
account_or_transaction_verification
operational_issue_needs_inspection
repeated_frustration_or_loop
custom_instruction_transfer
].freeze
string :action, enum: ACTIONS, description: 'Whether to keep the conversation with the assistant or transfer it to a human agent'
string :action_reason, enum: REASONS, description: 'The reason for the selected routing action'
end
@@ -17,14 +17,14 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
let(:mock_response) do
instance_double(
RubyLLM::Message,
content: '{"action":"handoff","action_reason":"human_offer_accepted"}'
content: { 'action' => 'handoff', 'action_reason' => 'human_offer_accepted' }
)
end
before do
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
allow(mock_chat).to receive(:with_schema).and_return(mock_chat)
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
end
@@ -38,6 +38,7 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
end
it 'passes delimited custom instructions and classifier context to the LLM' do
expect(mock_chat).to receive(:with_schema).with(Captain::AssistantActionSchema).and_return(mock_chat)
expect(mock_chat).to receive(:ask) do |prompt|
expect(prompt).to include(
'<account_custom_instructions>',