feat: add multimodal message content support

enterprise/app/controllers/api/v1/accounts/captain/assistants_controller.rb

@@ -25,7 +25,7 @@ class Api::V1::Accounts::Captain::AssistantsController < Api::V1::Accounts::Base
 
   def playground
     response = Captain::Llm::AssistantChatService.new(assistant: @assistant).generate_response(
-      params[:message_content],
+      message_content_multimodal(params[:message_content]),
       message_history
     )
 

enterprise/app/helpers/captain/chat_helper.rb

@@ -18,6 +18,84 @@ module Captain::ChatHelper
     raise e
   end
 
+  def message_content_multimodal(message)
+    # If message has text content, start with that
+    content_parts = []
+
+    if message.content.present?
+      content_parts << {
+        type: 'text',
+        text: message.content
+      }
+    end
+
+    # Add image content if available
+    if message.attachments.any?
+      image_attachments = message.attachments.where(file_type: :image)
+
+      image_attachments.each do |attachment|
+        image_url = get_attachment_url(attachment)
+        next unless image_url.present?
+
+        content_parts << {
+          type: 'image_url',
+          image_url: {
+            url: image_url
+          }
+        }
+      end
+
+      # Handle audio transcriptions
+      audio_transcriptions = extract_audio_transcriptions(message.attachments)
+      if audio_transcriptions.present?
+        content_parts << {
+          type: 'text',
+          text: audio_transcriptions
+        }
+      end
+
+      # Handle other attachment types
+      other_attachments = message.attachments.where.not(file_type: [:image, :audio])
+      if other_attachments.any?
+        content_parts << {
+          type: 'text',
+          text: 'User has shared an attachment'
+        }
+      end
+    end
+
+    # Return just text if no special content, otherwise return array for multimodal
+    if content_parts.length == 1 && content_parts.first[:type] == 'text'
+      content_parts.first[:text]
+    elsif content_parts.any?
+      content_parts
+    else
+      'Message without content'
+    end
+  end
+
+  def get_attachment_url(attachment)
+    if attachment.external_url.present?
+      attachment.external_url
+    elsif attachment.file.attached?
+      # For uploaded files, we need to generate a public URL
+      # This will work if the file is stored in a public cloud storage
+      attachment.file.url if attachment.file.respond_to?(:url)
+    end
+  end
+
+  def extract_audio_transcriptions(attachments)
+    audio_attachments = attachments.where(file_type: :audio)
+    return '' if audio_attachments.blank?
+
+    transcriptions = ''
+    audio_attachments.each do |attachment|
+      result = Messages::AudioTranscriptionService.new(attachment).perform
+      transcriptions += result[:transcriptions] if result[:success]
+    end
+    transcriptions
+  end
+
   private
 
   def handle_response(response)

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -1,4 +1,6 @@
 class Captain::Conversation::ResponseBuilderJob < ApplicationJob
+  include Captain::ChatHelper
+
   MAX_MESSAGE_LENGTH = 10_000
   retry_on ActiveStorage::FileNotFoundError, attempts: 3
 
@@ -25,8 +27,9 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   delegate :account, :inbox, to: :@conversation
 
   def generate_and_process_response
+    latest_message = @conversation.messages.incoming.last
     @response = Captain::Llm::AssistantChatService.new(assistant: @assistant).generate_response(
-      @conversation.messages.incoming.last.content,
+      message_content_multimodal(latest_message),
       collect_previous_messages
     )
 
@@ -37,39 +40,19 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
     account.increment_response_usage
   end
 
-  def collect_previous_messages
-    @conversation
-      .messages
-      .where(message_type: [:incoming, :outgoing])
-      .where(private: false)
-      .map do |message|
-        {
-          content: message_content(message),
-          role: determine_role(message)
-        }
-      end
-  end
+  def collect_previous_messages(include_all: false)
+    messages_query = @conversation
+                     .messages
+                     .where(message_type: [:incoming, :outgoing])
 
-  def message_content(message)
-    return message.content if message.content.present?
-    return 'User has shared a message without content' unless message.attachments.any?
+    messages_query = messages_query.where(private: false) unless include_all
 
-    audio_transcriptions = extract_audio_transcriptions(message.attachments)
-    return audio_transcriptions if audio_transcriptions.present?
-
-    'User has shared an attachment'
-  end
-
-  def extract_audio_transcriptions(attachments)
-    audio_attachments = attachments.where(file_type: :audio)
-    return '' if audio_attachments.blank?
-
-    transcriptions = ''
-    audio_attachments.each do |attachment|
-      result = Messages::AudioTranscriptionService.new(attachment).perform
-      transcriptions += result[:transcriptions] if result[:success]
+    messages_query.map do |message|
+      {
+        content: message_content_multimodal(message),
+        role: determine_role(message)
+      }
     end
-    transcriptions
   end
 
   def determine_role(message)

spec/enterprise/jobs/captain/conversation/response_builder_job_spec.rb

@@ -30,5 +30,27 @@ RSpec.describe Captain::Conversation::ResponseBuilderJob, type: :job do
       account.reload
       expect(account.usage_limits[:captain][:responses][:consumed]).to eq(1)
     end
+
+    context 'when message contains an image' do
+      let(:message_with_image) { create(:message, conversation: conversation, message_type: :incoming, content: 'Can you help with this error?') }
+      let(:image_attachment) { message_with_image.attachments.create!(account: account, file_type: :image, external_url: 'https://example.com/error.jpg') }
+
+      before do
+        image_attachment
+      end
+
+      it 'includes image URL directly in the message content for OpenAI vision analysis' do
+        # Expect the generate_response to receive multimodal content with image URL
+        expect(mock_llm_chat_service).to receive(:generate_response) do |content, _history|
+          # Content should be an array for multimodal
+          expect(content).to be_an(Array)
+          expect(content.any? { |part| part[:type] == 'text' && part[:text] == 'Can you help with this error?' }).to be true
+          expect(content.any? { |part| part[:type] == 'image_url' && part[:image_url][:url] == 'https://example.com/error.jpg' }).to be true
+          { 'response' => 'I can see the error in your image. It appears to be a database connection issue.' }
+        end
+
+        described_class.perform_now(conversation, assistant)
+      end
+    end
   end
 end