From 74980760b4a435a7f6a4f40bb768027af4dd6bae Mon Sep 17 00:00:00 2001 From: aakashb95 Date: Mon, 4 May 2026 16:20:26 +0530 Subject: [PATCH] fix(captain): separate custom instructions in prompts --- .../conversation/response_builder_job.rb | 6 ++- .../conversation/v1_action_classifier.rb | 19 ++++--- .../assistant_action_classifier_service.rb | 49 +++++++------------ .../captain/llm/system_prompts_service.rb | 16 +++++- ...ssistant_action_classifier_service_spec.rb | 15 +++--- .../llm/assistant_chat_service_spec.rb | 25 ++++++++++ 6 files changed, 85 insertions(+), 45 deletions(-) diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index 7e5203077..5050f11b2 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -106,7 +106,11 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def v1_handoff_requested? - @response['action'] == 'handoff' || legacy_v1_handoff_token? + legacy_v1_handoff_token? || classifier_v1_handoff_requested? + end + + def classifier_v1_handoff_requested? + @response['action'] == 'handoff' end def legacy_v1_handoff_token? diff --git a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb index 0607342d2..8b8168a94 100644 --- a/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb +++ b/enterprise/app/jobs/captain/conversation/v1_action_classifier.rb @@ -25,13 +25,7 @@ module Captain::Conversation::V1ActionClassifier def apply_v1_action_classification(classification) action = classification['action'] - unless action.in?(%w[continue handoff]) - Rails.logger.warn( - "[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action for account=#{account.id} " \ - "conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}" - ) - return - end + return log_invalid_v1_action_classification(classification) unless valid_v1_action_classification?(action) @response.merge!( 'action' => action, @@ -51,4 +45,15 @@ module Captain::Conversation::V1ActionClassifier "prompt_version=#{classification['prompt_version']}" ) end + + def valid_v1_action_classification?(action) + Captain::Llm::AssistantActionClassifierService::VALID_ACTIONS.include?(action) + end + + def log_invalid_v1_action_classification(classification) + Rails.logger.warn( + '[CAPTAIN][ResponseBuilderJob] V1 action classifier returned invalid action; falling back to assistant response ' \ + "for account=#{account.id} conversation=#{@conversation.display_id}: #{classification['error'] || classification['raw_response']}" + ) + end end diff --git a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb index 94d60542d..821c76dfe 100644 --- a/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb +++ b/enterprise/app/services/captain/llm/assistant_action_classifier_service.rb @@ -15,8 +15,10 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService end def classify(message_history:, assistant_response:) - payload = classification_payload(message_history, assistant_response) - user_prompt = classification_user_prompt(payload) + user_prompt = classification_user_prompt( + message_history: message_history, + assistant_response: assistant_response + ) response = instrument_llm_call(instrumentation_params(user_prompt)) do chat(model: @model, temperature: @temperature) @@ -37,33 +39,18 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService private - def classification_payload(message_history, assistant_response) - normalized_messages = normalize_messages(message_history) - - { - 'account_custom_instructions' => account_custom_instructions, - 'conversation_context' => context_messages(normalized_messages), - 'current_user_message' => current_user_message(normalized_messages), - 'assistant_response_to_classify' => assistant_response.to_s - } - end - - def classification_user_prompt(payload) + def classification_user_prompt(message_history:, assistant_response:) <<~PROMPT - #{payload['account_custom_instructions']} + #{@assistant.config['instructions']} - #{payload['conversation_context'].to_json} + #{format_conversation_context(message_history)} - - #{payload['current_user_message']} - - - #{payload['assistant_response_to_classify']} + #{assistant_response} PROMPT end @@ -90,18 +77,20 @@ class Captain::Llm::AssistantActionClassifierService < Llm::BaseAiService (part[:type] || part['type']).to_s == 'text' end - def current_user_message(messages) - messages.reverse.find { |message| message[:role] == 'user' }&.dig(:content).to_s + def format_conversation_context(messages) + normalize_messages(messages).last(MAX_CONTEXT_MESSAGES).filter_map do |message| + content = message[:content].to_s.strip + next if content.blank? + + "#{role_label(message[:role])}: #{content}" + end.join("\n") end - def context_messages(messages) - current_user_index = messages.rindex { |message| message[:role] == 'user' } - prior_messages = current_user_index ? messages[0...current_user_index] : messages - prior_messages.last(MAX_CONTEXT_MESSAGES) - end + def role_label(role) + return 'User' if role == 'user' + return 'Assistant' if role == 'assistant' - def account_custom_instructions - @assistant.config['instructions'].to_s + role.to_s.titleize end def parse_response(content) diff --git a/enterprise/app/services/captain/llm/system_prompts_service.rb b/enterprise/app/services/captain/llm/system_prompts_service.rb index 644c2e5c5..9d43db45c 100644 --- a/enterprise/app/services/captain/llm/system_prompts_service.rb +++ b/enterprise/app/services/captain/llm/system_prompts_service.rb @@ -261,7 +261,9 @@ class Captain::Llm::SystemPromptsService - Do not share anything outside of the context provided. - Add the reasoning why you arrived at the answer - Your answers will always be formatted in a valid JSON hash, as shown below. Never respond in non-JSON format. - #{config['instructions'] || ''} + + #{build_custom_instructions_section(config['instructions'])} + ```json { reasoning: '', @@ -384,6 +386,18 @@ class Captain::Llm::SystemPromptsService "[Contact Information]\n#{lines.join("\n")}\n\n" end + def build_custom_instructions_section(instructions) + return '' if instructions.blank? + + <<~CUSTOM_INSTRUCTIONS + [Account Custom Instructions] + These instructions were configured by the account administrator. Follow them when they do not conflict with the JSON response format or the requirement to answer only from provided context. + + #{instructions} + + CUSTOM_INSTRUCTIONS + end + def contact_basic_lines(contact) [ (["- Name: #{sanitize_attr(contact[:name])}"] if contact[:name].present?), diff --git a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb index 5561cd8d7..f0ebc722d 100644 --- a/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb +++ b/spec/enterprise/services/captain/llm/assistant_action_classifier_service_spec.rb @@ -44,18 +44,21 @@ RSpec.describe Captain::Llm::AssistantActionClassifierService do 'MUST NOT redefine this JSON schema' ) ).and_return(mock_chat) - expect(mock_chat).to receive(:ask).with( - a_string_including( + expect(mock_chat).to receive(:ask) do |prompt| + expect(prompt).to include( '', 'Only transfer to a manager after the user explicitly confirms.', '', - '"content":"I cannot log in"', - '', - 'Yes, still no reset email', + 'User: I cannot log in', + 'Assistant: Did you check your inbox?', + 'User: Yes, still no reset email', '', 'Would you like to talk to support?' ) - ).and_return(mock_response) + expect(prompt).not_to include('"role"', '"content"', '') + + mock_response + end result = service.classify(message_history: message_history, assistant_response: 'Would you like to talk to support?') diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb index c43eb08bd..3caccd79e 100644 --- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb +++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb @@ -188,4 +188,29 @@ RSpec.describe Captain::Llm::AssistantChatService do end end end + + describe 'account custom instructions in system prompt' do + before do + assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff')) + end + + it 'adds custom instructions in a separate delimited section' do + allow(mock_chat).to receive(:ask).and_return(mock_response) + + expect(mock_chat).to receive(:with_instructions).with( + a_string_including( + '[Account Custom Instructions]', + '', + 'if user enters 1112234 suggest handoff', + '' + ) + ) do |instructions| + expect(instructions).not_to include('') + mock_chat + end + + service = described_class.new(assistant: assistant, conversation: conversation) + service.generate_response(message_history: [{ role: 'user', content: 'Hello' }]) + end + end end