Aakash BakhleandGitHub
926a9d8a69
# Pull Request Template ## Description - Default temperature to 0.5 and remove UI control - No migrations needed for existing accounts, their current settings are preserved ## Type of change - [x] New feature (non-breaking change which adds functionality) ## How Has This Been Tested? Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration. locally and spec ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules
269 lines
10 KiB
Ruby
269 lines
10 KiB
Ruby
require 'rails_helper'
|
|
|
|
RSpec.describe Captain::Llm::AssistantChatService do
|
|
let(:account) { create(:account) }
|
|
let(:assistant) { create(:captain_assistant, account: account) }
|
|
let(:conversation) { create(:conversation, account: account) }
|
|
|
|
let(:mock_chat) { instance_double(RubyLLM::Chat) }
|
|
let(:mock_response) do
|
|
instance_double(
|
|
RubyLLM::Message,
|
|
content: '{"response": "I can see the image shows a pricing table", "reasoning": "Analyzed the image"}'
|
|
)
|
|
end
|
|
|
|
before do
|
|
create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
|
|
|
|
allow(RubyLLM).to receive(:chat).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:with_temperature).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:with_params).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:with_tool).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:with_instructions).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:add_message).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:on_end_message).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:on_tool_call).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:on_tool_result).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:messages).and_return([])
|
|
end
|
|
|
|
describe 'instrumentation metadata' do
|
|
it 'uses the assistant feature model' do
|
|
account.update!(captain_models: { 'assistant' => 'gpt-5.2' })
|
|
|
|
expect(RubyLLM).to receive(:chat).with(model: 'gpt-5.2').and_return(mock_chat)
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
|
|
it 'uses default temperature when assistant config does not include temperature' do
|
|
expect(mock_chat).to receive(:with_temperature).with(0.5).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
|
|
it 'preserves explicit assistant config temperature' do
|
|
assistant.update!(config: assistant.config.merge('temperature' => 1.0))
|
|
|
|
expect(mock_chat).to receive(:with_temperature).with(1.0).and_return(mock_chat)
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
|
|
it 'passes channel_type to the agent session instrumentation' do
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
|
|
expect(service).to receive(:instrument_agent_session).with(
|
|
hash_including(metadata: hash_including(channel_type: conversation.inbox.channel_type))
|
|
).and_yield
|
|
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
|
|
it 'marks final response generations for observation-level evaluators' do
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
message = instance_double(RubyLLM::Message, content: 'Final answer', input_tokens: 10, output_tokens: 20, tool_calls: {})
|
|
|
|
attributes = service.send(:generation_attributes, mock_chat, message)
|
|
|
|
expect(attributes['langfuse.observation.metadata.generation_stage']).to eq('final_response')
|
|
end
|
|
|
|
it 'marks tool call generations separately from final responses' do
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
message = instance_double(
|
|
RubyLLM::Message,
|
|
content: '',
|
|
input_tokens: 10,
|
|
output_tokens: 20,
|
|
tool_calls: { 'call_1' => instance_double(RubyLLM::ToolCall) }
|
|
)
|
|
|
|
attributes = service.send(:generation_attributes, mock_chat, message)
|
|
|
|
expect(attributes['langfuse.observation.metadata.generation_stage']).to eq('tool_call')
|
|
end
|
|
end
|
|
|
|
describe 'image analysis' do
|
|
context 'when user sends a message with an image attachment' do
|
|
let(:message_history) do
|
|
[
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: 'What do you see in this image?' },
|
|
{ type: 'image_url', image_url: { url: 'https://example.com/screenshot.png' } }
|
|
]
|
|
}
|
|
]
|
|
end
|
|
|
|
it 'sends the image to the LLM for analysis' do
|
|
expect(mock_chat).to receive(:ask).with(
|
|
'What do you see in this image?',
|
|
with: ['https://example.com/screenshot.png']
|
|
).and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: message_history)
|
|
end
|
|
end
|
|
|
|
context 'when user sends only an image without text' do
|
|
let(:message_history) do
|
|
[
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'image_url', image_url: { url: 'https://example.com/photo.jpg' } }
|
|
]
|
|
}
|
|
]
|
|
end
|
|
|
|
it 'sends the image to the LLM with nil text' do
|
|
expect(mock_chat).to receive(:ask).with(
|
|
nil,
|
|
with: ['https://example.com/photo.jpg']
|
|
).and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: message_history)
|
|
end
|
|
end
|
|
|
|
context 'when user sends a plain text message' do
|
|
let(:message_history) do
|
|
[
|
|
{ role: 'user', content: 'Hello, how can you help me?' }
|
|
]
|
|
end
|
|
|
|
it 'sends the text without attachments' do
|
|
expect(mock_chat).to receive(:ask).with('Hello, how can you help me?').and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: message_history)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'conversation history with images' do
|
|
context 'when previous messages contain images' do
|
|
let(:message_history) do
|
|
[
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: 'Here is my error screenshot' },
|
|
{ type: 'image_url', image_url: { url: 'https://example.com/error.png' } }
|
|
]
|
|
},
|
|
{ role: 'assistant', content: 'I see the error. Try restarting.' },
|
|
{ role: 'user', content: 'It still does not work' }
|
|
]
|
|
end
|
|
|
|
it 'includes images from conversation history in context' do
|
|
# First historical message should include the image via RubyLLM::Content
|
|
expect(mock_chat).to receive(:add_message) do |args|
|
|
expect(args[:role]).to eq(:user)
|
|
expect(args[:content]).to be_a(RubyLLM::Content)
|
|
expect(args[:content].text).to eq('Here is my error screenshot')
|
|
expect(args[:content].attachments.first.source.to_s).to eq('https://example.com/error.png')
|
|
end.ordered
|
|
|
|
# Second historical message is plain text
|
|
expect(mock_chat).to receive(:add_message).with(
|
|
role: :assistant,
|
|
content: 'I see the error. Try restarting.'
|
|
).ordered
|
|
|
|
# Current message asked via chat.ask
|
|
expect(mock_chat).to receive(:ask).with('It still does not work').and_return(mock_response)
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: message_history)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'contact attributes in system prompt' do
|
|
let(:contact) { create(:contact, account: account, name: 'Diep Bui', email: 'diep@example.com', custom_attributes: { 'plan' => 'pro' }) }
|
|
let(:conversation) { create(:conversation, account: account, contact: contact) }
|
|
|
|
context 'when feature_contact_attributes is enabled' do
|
|
before { assistant.update!(config: assistant.config.merge('feature_contact_attributes' => true)) }
|
|
|
|
it 'includes contact information in the system prompt' do
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
expect(mock_chat).to receive(:with_instructions).with(a_string_including('[Contact Information]')) do |_instructions|
|
|
mock_chat
|
|
end
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
|
|
it 'includes custom attributes in the system prompt' do
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
expect(mock_chat).to receive(:with_instructions).with(a_string_including('plan: pro')) do |_instructions|
|
|
mock_chat
|
|
end
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
end
|
|
|
|
context 'when feature_contact_attributes is disabled' do
|
|
it 'does not include contact information in the system prompt' do
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
expect(mock_chat).to receive(:with_instructions).with(satisfy { |s| s.exclude?('[Contact Information]') }) do |_instructions|
|
|
mock_chat
|
|
end
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
end
|
|
end
|
|
|
|
describe 'account custom instructions in system prompt' do
|
|
before do
|
|
assistant.update!(config: assistant.config.merge('instructions' => 'if user enters 1112234 suggest handoff'))
|
|
end
|
|
|
|
it 'adds custom instructions in a separate delimited section' do
|
|
allow(mock_chat).to receive(:ask).and_return(mock_response)
|
|
|
|
expect(mock_chat).to receive(:with_instructions).with(
|
|
a_string_including(
|
|
'<account_custom_instructions>',
|
|
'if user enters 1112234 suggest handoff',
|
|
'</account_custom_instructions>'
|
|
)
|
|
) do |instructions|
|
|
expect(instructions).not_to include('<custom-instructions>')
|
|
expect(instructions.index('<account_custom_instructions>')).to be < instructions.index('```json')
|
|
mock_chat
|
|
end
|
|
|
|
service = described_class.new(assistant: assistant, conversation: conversation)
|
|
service.generate_response(message_history: [{ role: 'user', content: 'Hello' }])
|
|
end
|
|
end
|
|
end
|