From 368853c46fd90cbe298cdd67f90afe7e6b30afbd Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 4 Apr 2025 13:46:53 +0530 Subject: [PATCH] test: update specs --- .../llm/assistant_chat_service_spec.rb | 232 ++++++++---------- 1 file changed, 102 insertions(+), 130 deletions(-) diff --git a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb index 195da219b..a66126a29 100644 --- a/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb +++ b/spec/enterprise/services/captain/llm/assistant_chat_service_spec.rb @@ -5,34 +5,29 @@ RSpec.describe Captain::Llm::AssistantChatService do let(:service) { described_class.new(assistant: captain_assistant) } let(:client) { instance_double(OpenAI::Client) } let(:input) { 'How can I help you?' } + let(:openai_response) do + { + 'choices' => [ + { + 'message' => { + 'content' => { + reasoning: 'This is a helpful response', + response: 'I can assist you with your questions.' + }.to_json + } + } + ] + } + end before do create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key') allow(OpenAI::Client).to receive(:new).and_return(client) + allow(client).to receive(:chat).and_return(openai_response) end describe '#generate_response' do - let(:previous_messages) { [{ role: 'user', content: 'Previous message' }] } - let(:openai_response) do - { - 'choices' => [ - { - 'message' => { - 'content' => { - reasoning: 'This is a helpful response', - response: 'I can assist you with your questions.' - }.to_json - } - } - ] - } - end - context 'when successful' do - before do - allow(client).to receive(:chat).and_return(openai_response) - end - it 'generates a response with input' do response = service.generate_response(input) expect(response).to eq({ @@ -42,6 +37,7 @@ RSpec.describe Captain::Llm::AssistantChatService do end it 'generates a response with input and previous messages' do + previous_messages = [{ role: 'user', content: 'Previous message' }] response = service.generate_response(input, previous_messages) expect(response).to eq({ 'reasoning' => 'This is a helpful response', @@ -50,6 +46,7 @@ RSpec.describe Captain::Llm::AssistantChatService do end it 'includes previous messages in the chat parameters' do + previous_messages = [{ role: 'user', content: 'Previous message' }] service.generate_response(input, previous_messages) expect(client).to have_received(:chat) do |params| messages = params[:parameters][:messages] @@ -67,55 +64,9 @@ RSpec.describe Captain::Llm::AssistantChatService do expect(messages.first[:content]).to include(captain_assistant.config['product_name']) end end - - it 'includes documentation search tool in chat parameters' do - service.generate_response(input) - expect(client).to have_received(:chat) do |params| - tools = params[:parameters][:tools] - expect(tools).to contain_exactly( - hash_including( - type: 'function', - function: hash_including( - name: 'search_documentation', - parameters: hash_including( - properties: hash_including( - search_query: hash_including(type: 'string') - ) - ) - ) - ) - ) - end - end - - it 'includes json response format in chat parameters' do - service.generate_response(input) - expect(client).to have_received(:chat) do |params| - expect(params[:parameters][:response_format]).to eq({ type: 'json_object' }) - end - end end - context 'when input is empty' do - let(:input) { '' } - - before do - allow(client).to receive(:chat).and_return(openai_response) - end - - it 'does not add empty input to messages' do - service.generate_response(input) - expect(client).to have_received(:chat) do |params| - messages = params[:parameters][:messages] - expect(messages.pluck(:role)).not_to include('user') - end - end - end - - context 'with documentation search' do - let(:tool_call_id) { 'call_123' } - let(:search_query) { 'test query' } - let(:response_content) { 'Documentation content' } + context 'when search fails' do let(:openai_response_with_tool) do { 'choices' => [ @@ -123,10 +74,10 @@ RSpec.describe Captain::Llm::AssistantChatService do 'message' => { 'tool_calls' => [ { - 'id' => tool_call_id, + 'id' => 'call_123', 'function' => { 'name' => 'search_documentation', - 'arguments' => { 'search_query' => search_query }.to_json + 'arguments' => { 'search_query' => 'test query' }.to_json } } ] @@ -138,85 +89,106 @@ RSpec.describe Captain::Llm::AssistantChatService do before do allow(client).to receive(:chat) - .and_return(openai_response_with_tool, openai_response) + .and_return(openai_response_with_tool) allow(captain_assistant.responses).to receive(:approved).and_return(captain_assistant.responses) - allow(captain_assistant.responses).to receive(:search).with(search_query) - .and_return([ - instance_double(Captain::AssistantResponse, - question: 'Test Q?', - answer: 'Test A', - documentable: nil) - ]) + allow(captain_assistant.responses).to receive(:search) + .and_raise(StandardError, 'Search failed') end - it 'processes tool calls and fetches documentation' do - response = service.generate_response(input) - expect(client).to have_received(:chat).at_least(:once) - expect(response).to eq({ - 'reasoning' => 'This is a helpful response', - 'response' => 'I can assist you with your questions.' - }) + it 'raises the error' do + expect { service.generate_response(input) }.to raise_error(StandardError, 'Search failed') + end + end + + context 'with invalid tool parameters' do + let(:openai_response_with_invalid_tool) do + { + 'choices' => [ + { + 'message' => { + 'tool_calls' => [ + { + 'id' => 'call_123', + 'function' => { + 'name' => 'search_documentation', + 'arguments' => 'invalid_json' + } + } + ] + } + } + ] + } end - it 'appends tool calls and responses to messages' do + before do + allow(client).to receive(:chat) + .and_return(openai_response_with_invalid_tool) + end + + it 'raises JSON::ParserError' do + expect { service.generate_response(input) }.to raise_error(JSON::ParserError) + end + end + + context 'with model configuration' do + it 'uses the configured model' do service.generate_response(input) - expect(client).to have_received(:chat).at_least(:once) do |params| - messages = params[:parameters][:messages] - assistant_message = messages.find { |m| m[:role] == 'assistant' } - expect(assistant_message).to include( - tool_calls: array_including( - hash_including( - 'id' => tool_call_id, - 'function' => hash_including( - 'name' => 'search_documentation', - 'arguments' => { 'search_query' => search_query }.to_json - ) - ) - ) - ) + expect(client).to have_received(:chat) do |params| + expect(params[:parameters][:model]).to eq('gpt-4o-mini') end end - context 'with external link' do - before do - allow(captain_assistant.responses).to receive(:search).with(search_query) - .and_return([ - instance_double(Captain::AssistantResponse, - question: 'Test Q?', - answer: 'Test A', - documentable: instance_double(Captain::Document, - external_link: 'https://example.com')) - ]) - end - - it 'includes source in formatted response' do - service.generate_response(input) - expect(client).to have_received(:chat).at_least(:once) do |params| - messages = params[:parameters][:messages] - tool_response = messages.find { |m| m[:role] == 'tool' } - expect(tool_response[:content]).to include('Source: https://example.com') - end + it 'includes json response format' do + service.generate_response(input) + expect(client).to have_received(:chat) do |params| + expect(params[:parameters][:response_format]).to eq({ type: 'json_object' }) end end end end - describe '#chat_parameters' do - before do - allow(client).to receive(:chat).and_return({ - 'choices' => [ - { - 'message' => { - 'content' => { reasoning: '', response: '' }.to_json - } - } - ] - }) + describe '#initialize' do + it 'configures search_documentation tool' do + service.generate_response(input) + expect(client).to have_received(:chat) do |params| + tools = params[:parameters][:tools] + expect(tools).to contain_exactly( + hash_including( + type: 'function', + function: hash_including( + name: 'search_documentation', + description: match(/documentation/), + parameters: hash_including( + type: 'object', + properties: hash_including( + search_query: hash_including( + type: 'string', + description: match(/search query/) + ) + ), + required: ['search_query'] + ) + ) + ) + ) + end end + it 'provides access to assistant configuration' do + service.generate_response(input) + expect(client).to have_received(:chat) do |params| + messages = params[:parameters][:messages] + system_message = messages.find { |m| m[:role] == 'system' } + expect(system_message[:content]).to include(captain_assistant.config['product_name']) + end + end + end + + describe '#chat_parameters' do it 'includes correct model and response format' do service.generate_response(input) - expect(client).to have_received(:chat).at_least(:once) do |params| + expect(client).to have_received(:chat) do |params| parameters = params[:parameters] expect(parameters[:model]).to eq('gpt-4o-mini') expect(parameters[:response_format]).to eq({ type: 'json_object' })