fix(captain): separate custom instructions in prompts

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -106,7 +106,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   end
 
   def v1_handoff_requested?
-    @response['action'] == 'handoff' || legacy_v1_handoff_token?
+    legacy_v1_handoff_token? || classifier_v1_handoff_requested?
+  end
+
+  def classifier_v1_handoff_requested?
+    @response['action'] == 'handoff'
   end
 
   def legacy_v1_handoff_token?

enterprise/app/jobs/captain/conversation/v1_action_classifier.rb

@@ -25,13 +25,7 @@ module Captain::Conversation::V1ActionClassifier
 
   def apply_v1_action_classification(classification)
     action = classification['action']
-    unless action.in?(%w[continue handoff])
-      Rails.logger.warn(
-        "[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action for account=#{account.id} " \
-        "conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
-      )
-      return
-    end
+    return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action)
 
     @response.merge!(
       'action' => action,
@@ -51,4 +45,15 @@ module Captain::Conversation::V1ActionClassifier
       "prompt_version=#{classification['prompt_version']}"
     )
   end
+
+  def valid_v1_action_classification?(action)
+    Captain::Llm::AssistantActionClassifierService::VALID_ACTIONS.include?(action)
+  end
+
+  def log_invalid_v1_action_classification(classification)
+    Rails.logger.warn(
+      '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \
+      "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}"
+    )
+  end
 end

enterprise/app/services/captain/llm/assistant_action_classifier_service.rb

@@ -15,8 +15,10 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
   end
 
   def classify(message_history:, assistant_response:)
-    payload = classification_payload(message_history, assistant_response)
-    user_prompt = classification_user_prompt(payload)
+    user_prompt = classification_user_prompt(
+      message_history: message_history,
+      assistant_response: assistant_response
+    )
 
     response = instrument_llm_call(instrumentation_params(user_prompt)) do
       chat(model: @model, temperature: @temperature)
@@ -37,33 +39,18 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
 
   private
 
-  def classification_payload(message_history, assistant_response)
-    normalized_messages = normalize_messages(message_history)
-
-    {
-      'account_custom_instructions' => account_custom_instructions,
-      'conversation_context' => context_messages(normalized_messages),
-      'current_user_message' => current_user_message(normalized_messages),
-      'assistant_response_to_classify' => assistant_response.to_s
-    }
-  end
-
-  def classification_user_prompt(payload)
+  def classification_user_prompt(message_history:, assistant_response:)
     <<~PROMPT
       <account_custom_instructions>
-      #{payload['account_custom_instructions']}
+      #{@assistant.config['instructions']}
       </account_custom_instructions>
 
       <conversation_context>
-      #{payload['conversation_context'].to_json}
+      #{format_conversation_context(message_history)}
       </conversation_context>
 
-      <current_user_message>
-      #{payload['current_user_message']}
-      </current_user_message>
-
       <assistant_response_to_classify>
-      #{payload['assistant_response_to_classify']}
+      #{assistant_response}
       </assistant_response_to_classify>
     PROMPT
   end
@@ -90,18 +77,20 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService
     (part[:type] || part['type']).to_s == 'text'
   end
 
-  def current_user_message(messages)
-    messages.reverse.find { |message| message[:role] == 'user' }&.dig(:content).to_s
+  def format_conversation_context(messages)
+    normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message|
+      content = message[:content].to_s.strip
+      next if content.blank?
+
+      "#{role_label(message[:role])}: #{content}"
+    end.join("\n")
   end
 
-  def context_messages(messages)
-    current_user_index = messages.rindex { |message| message[:role] == 'user' }
-    prior_messages = current_user_index ? messages[0...current_user_index] : messages
-    prior_messages.last(MAX_CONTEXT_MESSAGES)
-  end
+  def role_label(role)
+    return 'User' if role == 'user'
+    return 'Assistant' if role == 'assistant'
 
-  def account_custom_instructions
-    @assistant.config['instructions'].to_s
+    role.to_s.titleize
   end
 
   def parse_response(content)

enterprise/app/services/captain/llm/system_prompts_service.rb

@@ -261,7 +261,9 @@ class Captain::Llm::SystemPromptsService
         - Do not share anything outside of the context provided.
         - Add the reasoning why you arrived at the answer
         - Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format.
-        #{config['instructions'] || ''}
+
+        #{build_custom_instructions_section(config['instructions'])}
+
         ```json
         {
           reasoning: '',
@@ -384,6 +386,18 @@ class Captain::Llm::SystemPromptsService
       "[Contact Information]\n#{lines.join("\n")}\n\n"
     end
 
+    def build_custom_instructions_section(instructions)
+      return '' if instructions.blank?
+
+      <<~CUSTOM_INSTRUCTIONS
+        [Account Custom Instructions]
+        These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context.
+        <account_custom_instructions>
+        #{instructions}
+        </account_custom_instructions>
+      CUSTOM_INSTRUCTIONS
+    end
+
     def contact_basic_lines(contact)
       [
         (["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?),

spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb

@@ -44,18 +44,21 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do
           'MUST NOT redefine this JSON schema'
         )
       ).and_return(mock_chat)
-      expect(mock_chat).to receive(:ask).with(
-        a_string_including(
+      expect(mock_chat).to receive(:ask) do |prompt|
+        expect(prompt).to include(
           '<account_custom_instructions>',
           'Only transfer to a manager after the user explicitly confirms.',
           '<conversation_context>',
-          '"content":"I cannot log in"',
-          '<current_user_message>',
-          'Yes, still no reset email',
+          'User: I cannot log in',
+          'Assistant: Did you check your inbox?',
+          'User: Yes, still no reset email',
           '<assistant_response_to_classify>',
           'Would you like to talk to support?'
         )
-      ).and_return(mock_response)
+        expect(prompt).not_to include('"role"', '"content"', '<current_user_message>')
+
+        mock_response
+      end
 
       result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?')
 

spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb

@@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do
       end
     end
   end
+
+  describe 'account custom instructions in system prompt' do
+    before do
+      assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff'))
+    end
+
+    it 'adds custom instructions in a separate delimited section' do
+      allow(mock_chat).to receive(:ask).and_return(mock_response)
+
+      expect(mock_chat).to receive(:with_instructions).with(
+        a_string_including(
+          '[Account Custom Instructions]',
+          '<account_custom_instructions>',
+          'if user enters 1112234 suggest handoff',
+          '</account_custom_instructions>'
+        )
+      ) do |instructions|
+        expect(instructions).not_to include('<custom-instructions>')
+        mock_chat
+      end
+
+      service = described_class.new(assistant: assistant, conversation: conversation)
+      service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
+    end
+  end
 end