From c3619f51a1348a2fe6322082e742d15b0bc9be1d Mon Sep 17 00:00:00 2001 From: Tanmay Deep Sharma Date: Fri, 13 Jun 2025 00:15:46 +0530 Subject: [PATCH] feat: add multimodal message content support --- .../accounts/captain/assistants_controller.rb | 2 +- enterprise/app/helpers/captain/chat_helper.rb | 78 +++++++++++++++++++ .../conversation/response_builder_job.rb | 45 ++++------- .../conversation/response_builder_job_spec.rb | 22 ++++++ 4 files changed, 115 insertions(+), 32 deletions(-) diff --git a/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb b/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb index e5a055836..17ce78fba 100644 --- a/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb +++ b/enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb @@ -25,7 +25,7 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base def playground response = Captain::Llm::AssistantChatService.new(assistant: @assistant).generate_response( - params[:message_content], + message_content_multimodal(params[:message_content]), message_history ) diff --git a/enterprise/app/helpers/captain/chat_helper.rb b/enterprise/app/helpers/captain/chat_helper.rb index f90b8d07e..fecb7d6e7 100644 --- a/enterprise/app/helpers/captain/chat_helper.rb +++ b/enterprise/app/helpers/captain/chat_helper.rb @@ -18,6 +18,84 @@ module Captain::ChatHelper raise e end + def message_content_multimodal(message) + # If message has text content, start with that + content_parts = [] + + if message.content.present? + content_parts << { + type: 'text', + text: message.content + } + end + + # Add image content if available + if message.attachments.any? + image_attachments = message.attachments.where(file_type: :image) + + image_attachments.each do |attachment| + image_url = get_attachment_url(attachment) + next unless image_url.present? + + content_parts << { + type: 'image_url', + image_url: { + url: image_url + } + } + end + + # Handle audio transcriptions + audio_transcriptions = extract_audio_transcriptions(message.attachments) + if audio_transcriptions.present? + content_parts << { + type: 'text', + text: audio_transcriptions + } + end + + # Handle other attachment types + other_attachments = message.attachments.where.not(file_type: [:image, :audio]) + if other_attachments.any? + content_parts << { + type: 'text', + text: 'User has shared an attachment' + } + end + end + + # Return just text if no special content, otherwise return array for multimodal + if content_parts.length == 1 && content_parts.first[:type] == 'text' + content_parts.first[:text] + elsif content_parts.any? + content_parts + else + 'Message without content' + end + end + + def get_attachment_url(attachment) + if attachment.external_url.present? + attachment.external_url + elsif attachment.file.attached? + # For uploaded files, we need to generate a public URL + # This will work if the file is stored in a public cloud storage + attachment.file.url if attachment.file.respond_to?(:url) + end + end + + def extract_audio_transcriptions(attachments) + audio_attachments = attachments.where(file_type: :audio) + return '' if audio_attachments.blank? + + transcriptions = '' + audio_attachments.each do |attachment| + result = Messages::AudioTranscriptionService.new(attachment).perform + transcriptions += result[:transcriptions] if result[:success] + end + transcriptions + end + private def handle_response(response) diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index eb62a9a38..308e5fd68 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -1,4 +1,6 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob + include Captain::ChatHelper + MAX_MESSAGE_LENGTH = 10_000 retry_on ActiveStorage::FileNotFoundError, attempts: 3 @@ -25,8 +27,9 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob delegate :account, :inbox, to: :@conversation def generate_and_process_response + latest_message = @conversation.messages.incoming.last @response = Captain::Llm::AssistantChatService.new(assistant: @assistant).generate_response( - @conversation.messages.incoming.last.content, + message_content_multimodal(latest_message), collect_previous_messages ) @@ -37,39 +40,19 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob account.increment_response_usage end - def collect_previous_messages - @conversation - .messages - .where(message_type: [:incoming, :outgoing]) - .where(private: false) - .map do |message| - { - content: message_content(message), - role: determine_role(message) - } - end - end + def collect_previous_messages(include_all: false) + messages_query = @conversation + .messages + .where(message_type: [:incoming, :outgoing]) - def message_content(message) - return message.content if message.content.present? - return 'User has shared a message without content' unless message.attachments.any? + messages_query = messages_query.where(private: false) unless include_all - audio_transcriptions = extract_audio_transcriptions(message.attachments) - return audio_transcriptions if audio_transcriptions.present? - - 'User has shared an attachment' - end - - def extract_audio_transcriptions(attachments) - audio_attachments = attachments.where(file_type: :audio) - return '' if audio_attachments.blank? - - transcriptions = '' - audio_attachments.each do |attachment| - result = Messages::AudioTranscriptionService.new(attachment).perform - transcriptions += result[:transcriptions] if result[:success] + messages_query.map do |message| + { + content: message_content_multimodal(message), + role: determine_role(message) + } end - transcriptions end def determine_role(message) diff --git a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb index 1e4a6e824..e2765fbdc 100644 --- a/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb +++ b/spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb @@ -30,5 +30,27 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do account.reload expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1) end + + context 'when message contains an image' do + let(:message_with_image) { create(:message, conversation: conversation, message_type: :incoming, content: 'Can you help with this error?') } + let(:image_attachment) { message_with_image.attachments.create!(account: account, file_type: :image, external_url: 'https://example.com/error.jpg') } + + before do + image_attachment + end + + it 'includes image URL directly in the message content for OpenAI vision analysis' do + # Expect the generate_response to receive multimodal content with image URL + expect(mock_llm_chat_service).to receive(:generate_response) do |content, _history| + # Content should be an array for multimodal + expect(content).to be_an(Array) + expect(content.any? { |part| part[:type] == 'text' && part[:text] == 'Can you help with this error?' }).to be true + expect(content.any? { |part| part[:type] == 'image_url' && part[:image_url][:url] == 'https://example.com/error.jpg' }).to be true + { 'response' => 'I can see the error in your image. It appears to be a database connection issue.' } + end + + described_class.perform_now(conversation, assistant) + end + end end end