From 3fd054fcc8a5fe9ba1febfccb5e8004ac15dc150 Mon Sep 17 00:00:00 2001 From: Tanmay Deep Sharma Date: Tue, 17 Jun 2025 11:50:26 +0530 Subject: [PATCH] feat: add OpenaiMultimodalContentService for content generation --- .../conversation_llm_formatter.rb | 13 +++- .../openai_multimodal_content_service.rb | 74 +++++++++++++++++++ enterprise/app/helpers/captain/chat_helper.rb | 12 --- .../conversation/response_builder_job.rb | 71 ++---------------- .../tools/copilot/get_conversation_service.rb | 2 +- enterprise/lib/chat_gpt.rb | 6 +- 6 files changed, 94 insertions(+), 84 deletions(-) create mode 100644 app/services/openai_multimodal_content_service.rb diff --git a/app/services/llm_formatter/conversation_llm_formatter.rb b/app/services/llm_formatter/conversation_llm_formatter.rb index d9f184659..d7e7a1cdb 100644 --- a/app/services/llm_formatter/conversation_llm_formatter.rb +++ b/app/services/llm_formatter/conversation_llm_formatter.rb @@ -5,7 +5,7 @@ class LlmFormatter::ConversationLlmFormatter < LlmFormatter::DefaultLlmFormatter sections << "Channel: #{@record.inbox.channel.name}" sections << 'Message History:' sections << if @record.messages.any? - build_messages + build_messages(config) else 'No messages in this conversation' end @@ -18,11 +18,16 @@ class LlmFormatter::ConversationLlmFormatter < LlmFormatter::DefaultLlmFormatter private - def build_messages + def build_messages(config = {}) return "No messages in this conversation\n" if @record.messages.empty? message_text = '' - @record.messages.where.not(message_type: :activity).order(created_at: :asc).each do |message| + messages = @record.messages.where.not(message_type: :activity).order(created_at: :asc) + + messages.each do |message| + # Skip private messages unless explicitly included in config + next if message.private? && !config[:include_private_messages] + message_text << format_message(message) end message_text @@ -30,7 +35,7 @@ class LlmFormatter::ConversationLlmFormatter < LlmFormatter::DefaultLlmFormatter def format_message(message) sender = message.message_type == 'incoming' ? 'User' : 'Support agent' - sender = "[Private] #{sender}" if message.private? + sender = "[Private Note] #{sender}" if message.private? "#{sender}: #{message.content}\n" end diff --git a/app/services/openai_multimodal_content_service.rb b/app/services/openai_multimodal_content_service.rb new file mode 100644 index 000000000..11bf3c98e --- /dev/null +++ b/app/services/openai_multimodal_content_service.rb @@ -0,0 +1,74 @@ +class OpenaiMultimodalContentService + def initialize(message) + @message = message + end + + def generate_content + parts = [] + + parts << text_part(@message.content) if @message.content.present? + parts.concat(attachment_parts(@message.attachments)) if @message.attachments.any? + + finalize_content_parts(parts) + end + + private + + def text_part(text) + { type: 'text', text: text } + end + + def attachment_parts(attachments) + [].tap do |parts| + parts.concat(image_parts(attachments.where(file_type: :image))) + + transcription = extract_audio_transcriptions(attachments) + parts << text_part(transcription) if transcription.present? + + parts << text_part('User has shared an attachment') if attachments.where.not(file_type: %i[image audio]).exists? + end + end + + def image_parts(image_attachments) + image_attachments.each_with_object([]) do |attachment, parts| + url = get_attachment_url(attachment) + next if url.blank? + + parts << { + type: 'image_url', + image_url: { url: url } + } + end + end + + def finalize_content_parts(parts) + return 'Message without content' if parts.blank? + return parts.first[:text] if single_text_part?(parts) + + parts + end + + def single_text_part?(parts) + parts.one? && parts.first[:type] == 'text' + end + + def get_attachment_url(attachment) + return attachment.external_url if attachment.external_url.present? + + return unless attachment.file.attached? + + attachment.file_url + end + + def extract_audio_transcriptions(attachments) + audio_attachments = attachments.where(file_type: :audio) + return '' if audio_attachments.blank? + + transcriptions = '' + audio_attachments.each do |attachment| + result = Messages::AudioTranscriptionService.new(attachment).perform + transcriptions += result[:transcriptions] if result[:success] + end + transcriptions + end +end \ No newline at end of file diff --git a/enterprise/app/helpers/captain/chat_helper.rb b/enterprise/app/helpers/captain/chat_helper.rb index baff0a209..f90b8d07e 100644 --- a/enterprise/app/helpers/captain/chat_helper.rb +++ b/enterprise/app/helpers/captain/chat_helper.rb @@ -18,18 +18,6 @@ module Captain::ChatHelper raise e end - def extract_audio_transcriptions(attachments) - audio_attachments = attachments.where(file_type: :audio) - return '' if audio_attachments.blank? - - transcriptions = '' - audio_attachments.each do |attachment| - result = Messages::AudioTranscriptionService.new(attachment).perform - transcriptions += result[:transcriptions] if result[:success] - end - transcriptions - end - private def handle_response(response) diff --git a/enterprise/app/jobs/captain/conversation/response_builder_job.rb b/enterprise/app/jobs/captain/conversation/response_builder_job.rb index eea8b934e..976743f74 100644 --- a/enterprise/app/jobs/captain/conversation/response_builder_job.rb +++ b/enterprise/app/jobs/captain/conversation/response_builder_job.rb @@ -38,14 +38,12 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob account.increment_response_usage end - def collect_previous_messages(include_all: false) - messages_query = @conversation - .messages - .where(message_type: [:incoming, :outgoing]) - - messages_query = messages_query.where(private: false) unless include_all - - messages_query.map do |message| + def collect_previous_messages + @conversation + .messages + .where(message_type: [:incoming, :outgoing]) + .where(private: false) + .map do |message| { content: message_content_multimodal(message), role: determine_role(message) @@ -60,62 +58,7 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob end def message_content_multimodal(message) - parts = [] - - parts << text_part(message.content) if message.content.present? - parts.concat(attachment_parts(message.attachments)) if message.attachments.any? - - finalize_content_parts(parts) - end - - def text_part(text) - { type: 'text', text: text } - end - - def attachment_parts(attachments) - [].tap do |parts| - parts.concat(image_parts(attachments.where(file_type: :image))) - - transcription = extract_audio_transcriptions(attachments) - parts << text_part(transcription) if transcription.present? - - parts << text_part('User has shared an attachment') if attachments.where.not(file_type: %i[image audio]).exists? - end - end - - def image_parts(image_attachments) - image_attachments.each_with_object([]) do |attachment, parts| - url = get_attachment_url(attachment) - next if url.blank? - - parts << { - type: 'image_url', - image_url: { url: url } - } - end - end - - def finalize_content_parts(parts) - return 'Message without content' if parts.blank? - return parts.first[:text] if single_text_part?(parts) - - parts - end - - def single_text_part?(parts) - parts.one? && parts.first[:type] == 'text' - end - - def get_attachment_url(attachment) - return attachment.external_url if attachment.external_url.present? - - return unless attachment.file.attached? - - begin - attachment.file_url - rescue ActiveStorage::FileNotFoundError - nil - end + OpenaiMultimodalContentService.new(message).generate_content end def handoff_requested? diff --git a/enterprise/app/services/captain/tools/copilot/get_conversation_service.rb b/enterprise/app/services/captain/tools/copilot/get_conversation_service.rb index 64b52d012..6f942ee8e 100644 --- a/enterprise/app/services/captain/tools/copilot/get_conversation_service.rb +++ b/enterprise/app/services/captain/tools/copilot/get_conversation_service.rb @@ -30,7 +30,7 @@ class Captain::Tools::Copilot::GetConversationService < Captain::Tools::BaseServ conversation = Conversation.find_by(display_id: conversation_id, account_id: @assistant.account_id) return 'Conversation not found' if conversation.blank? - conversation.to_llm_text + conversation.to_llm_text(include_private_messages: true) end def active? diff --git a/enterprise/lib/chat_gpt.rb b/enterprise/lib/chat_gpt.rb index ad77159cf..44afbd641 100644 --- a/enterprise/lib/chat_gpt.rb +++ b/enterprise/lib/chat_gpt.rb @@ -8,9 +8,9 @@ class ChatGpt @messages = [system_message(context_sections)] end - def generate_response(additional_message: nil, message_history: [], role: 'user') - @messages += message_history - @messages << { 'role': role, 'content': additional_message } if additional_message.present? + def generate_response(input, previous_messages = [], role = 'user') + @messages += previous_messages + @messages << { 'role': role, 'content': input } if input.present? response = request_gpt JSON.parse(response['choices'][0]['message']['content'].strip)