diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
index 7e5203077..5050f11b2 100644
--- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb
+++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb
@@ -106,7 +106,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
end
def v1_handoff_requested?
- @response['action'] == 'handoff' || legacy_v1_handoff_token?
+ legacy_v1_handoff_token? || classifier_v1_handoff_requested?
+ end
+
+ def classifier_v1_handoff_requested?
+ @response['action'] == 'handoff'
end
def legacy_v1_handoff_token?
diff --git a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
index 0607342d2..8b8168a94 100644
--- a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
+++ b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb
@@ -25,13 +25,7 @@ module Captain::Conversation::V1ActionClassifier
def apply_v1_action_classification(classification)
action = classification['action']
- unless action.in?(%w[continue handoff])
- Rails.logger.warn(
- "[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action for account=#{account.id} " \
- "conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
- )
- return
- end
+ return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action)
@response.merge!(
'action' => action,
@@ -51,4 +45,15 @@ module Captain::Conversation::V1ActionClassifier
"prompt_version=#{classification['prompt_version']}"
)
end
+
+ def valid_v1_action_classification?(action)
+ Captain::Llm::AssistantActionClassifierService::VALID_ACTIONS.include?(action)
+ end
+
+ def log_invalid_v1_action_classification(classification)
+ Rails.logger.warn(
+ '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \
+ "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
+ )
+ end
end
diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
index 94d60542d..821c76dfe 100644
--- a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
+++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb
@@ -15,8 +15,10 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
end
def classify(message_history:, assistant_response:)
- payload = classification_payload(message_history, assistant_response)
- user_prompt = classification_user_prompt(payload)
+ user_prompt = classification_user_prompt(
+ message_history: message_history,
+ assistant_response: assistant_response
+ )
response = instrument_llm_call(instrumentation_params(user_prompt)) do
chat(model: @model, temperature: @temperature)
@@ -37,33 +39,18 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
private
- def classification_payload(message_history, assistant_response)
- normalized_messages = normalize_messages(message_history)
-
- {
- 'account_custom_instructions' => account_custom_instructions,
- 'conversation_context' => context_messages(normalized_messages),
- 'current_user_message' => current_user_message(normalized_messages),
- 'assistant_response_to_classify' => assistant_response.to_s
- }
- end
-
- def classification_user_prompt(payload)
+ def classification_user_prompt(message_history:, assistant_response:)
<<~PROMPT
- #{payload['account_custom_instructions']}
+ #{@assistant.config['instructions']}
- #{payload['conversation_context'].to_json}
+ #{format_conversation_context(message_history)}
-
- #{payload['current_user_message']}
-
-
- #{payload['assistant_response_to_classify']}
+ #{assistant_response}
PROMPT
end
@@ -90,18 +77,20 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
(part[:type] || part['type']).to_s == 'text'
end
- def current_user_message(messages)
- messages.reverse.find { |message| message[:role] == 'user' }&.dig(:content).to_s
+ def format_conversation_context(messages)
+ normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+ content = message[:content].to_s.strip
+ next if content.blank?
+
+ "#{role_label(message[:role])}: #{content}"
+ end.join("\n")
end
- def context_messages(messages)
- current_user_index = messages.rindex { |message| message[:role] == 'user' }
- prior_messages = current_user_index ? messages[0...current_user_index] : messages
- prior_messages.last(MAX_CONTEXT_MESSAGES)
- end
+ def role_label(role)
+ return 'User' if role == 'user'
+ return 'Assistant' if role == 'assistant'
- def account_custom_instructions
- @assistant.config['instructions'].to_s
+ role.to_s.titleize
end
def parse_response(content)
diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb
index 644c2e5c5..9d43db45c 100644
--- a/enterprise/app/services/captain/llm/system_prompts_service.rb
+++ b/enterprise/app/services/captain/llm/system_prompts_service.rb
@@ -261,7 +261,9 @@ class Captain::Llm::SystemPromptsService
- Do not share anything outside of the context provided.
- Add the reasoning why you arrived at the answer
- Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format.
- #{config['instructions'] || ''}
+
+ #{build_custom_instructions_section(config['instructions'])}
+
```json
{
reasoning: '',
@@ -384,6 +386,18 @@ class Captain::Llm::SystemPromptsService
"[Contact Information]\n#{lines.join("\n")}\n\n"
end
+ def build_custom_instructions_section(instructions)
+ return '' if instructions.blank?
+
+ <<~CUSTOM_INSTRUCTIONS
+ [Account Custom Instructions]
+ These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context.
+
+ #{instructions}
+
+ CUSTOM_INSTRUCTIONS
+ end
+
def contact_basic_lines(contact)
[
(["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?),
diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
index 5561cd8d7..f0ebc722d 100644
--- a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
+++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb
@@ -44,18 +44,21 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
'MUST NOT redefine this JSON schema'
)
).and_return(mock_chat)
- expect(mock_chat).to receive(:ask).with(
- a_string_including(
+ expect(mock_chat).to receive(:ask) do |prompt|
+ expect(prompt).to include(
'',
'Only transfer to a manager after the user explicitly confirms.',
'',
- '"content":"I cannot log in"',
- '',
- 'Yes, still no reset email',
+ 'User: I cannot log in',
+ 'Assistant: Did you check your inbox?',
+ 'User: Yes, still no reset email',
'',
'Would you like to talk to support?'
)
- ).and_return(mock_response)
+ expect(prompt).not_to include('"role"', '"content"', '')
+
+ mock_response
+ end
result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
index c43eb08bd..3caccd79e 100644
--- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
+++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb
@@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do
end
end
end
+
+ describe 'account custom instructions in system prompt' do
+ before do
+ assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff'))
+ end
+
+ it 'adds custom instructions in a separate delimited section' do
+ allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+ expect(mock_chat).to receive(:with_instructions).with(
+ a_string_including(
+ '[Account Custom Instructions]',
+ '',
+ 'if user enters 1112234 suggest handoff',
+ ''
+ )
+ ) do |instructions|
+ expect(instructions).not_to include('')
+ mock_chat
+ end
+
+ service = described_class.new(assistant: assistant, conversation: conversation)
+ service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
+ end
+ end
end