From e06525181b69a42f95e9786714329a4d7cb9925e Mon Sep 17 00:00:00 2001 From: Sojan Date: Tue, 13 May 2025 03:50:41 -0700 Subject: [PATCH] chore: Clean up and add transcriptions --- Gemfile | 2 +- Gemfile.lock | 22 +-- app/builders/messages/message_builder.rb | 2 +- .../api/v1/accounts/voice_controller.rb | 17 +- .../twilio/transcription_controller.rb | 62 +++++++ app/controllers/twilio/voice_controller.rb | 22 ++- .../widgets/conversation/MessagePreview.vue | 152 +++++++++++------- .../i18n/locale/en/conversation.json | 6 +- .../conversations/event_data_presenter.rb | 3 +- app/services/voice/call_status/manager.rb | 27 ++-- app/services/voice/incoming_call_service.rb | 30 +++- app/services/voice/outgoing_call_service.rb | 10 -- config/routes.rb | 3 + 13 files changed, 251 insertions(+), 107 deletions(-) create mode 100644 app/controllers/twilio/transcription_controller.rb diff --git a/Gemfile b/Gemfile index 1e8605379..bf53bd58c 100644 --- a/Gemfile +++ b/Gemfile @@ -87,7 +87,7 @@ gem 'wisper', '2.0.0' ##--- gems for channels ---## gem 'facebook-messenger' gem 'line-bot-api' -gem 'twilio-ruby', '~> 5.66' +gem 'twilio-ruby' # twitty will handle subscription of twitter account events # gem 'twitty', git: 'https://github.com/chatwoot/twitty' gem 'twitty', '~> 0.1.5' diff --git a/Gemfile.lock b/Gemfile.lock index da19dbb72..a686c8d5b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -235,8 +235,10 @@ GEM railties (>= 5.0.0) faker (3.2.0) i18n (>= 1.8.11, < 2) - faraday (2.9.0) - faraday-net_http (>= 2.0, < 3.2) + faraday (2.13.1) + faraday-net_http (>= 2.0, < 3.5) + json + logger faraday-follow_redirects (0.3.0) faraday (>= 1, < 3) faraday-mashify (0.1.1) @@ -244,8 +246,8 @@ GEM hashie faraday-multipart (1.0.4) multipart-post (~> 2) - faraday-net_http (3.1.0) - net-http + faraday-net_http (3.4.0) + net-http (>= 0.5.0) faraday-net_http_persistent (2.1.0) faraday (~> 2.5) net-http-persistent (~> 4.0) @@ -388,7 +390,7 @@ GEM rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) - json (2.6.3) + json (2.11.3) json_refs (0.1.8) hana json_schemer (0.2.24) @@ -403,7 +405,7 @@ GEM judoscale-sidekiq (1.8.2) judoscale-ruby (= 1.8.2) sidekiq (>= 5.0) - jwt (2.8.1) + jwt (2.10.1) base64 kaminari (1.2.2) activesupport (>= 4.1.0) @@ -445,7 +447,7 @@ GEM llhttp-ffi (0.4.0) ffi-compiler (~> 1.0) rake (~> 13.0) - logger (1.6.0) + logger (1.7.0) lograge (0.14.0) actionpack (>= 4) activesupport (>= 4) @@ -481,7 +483,7 @@ GEM mutex_m (0.3.0) neighbor (0.2.3) activerecord (>= 5.2) - net-http (0.4.1) + net-http (0.6.0) uri net-http-persistent (4.0.2) connection_pool (~> 2.2) @@ -800,7 +802,7 @@ GEM i18n timeout (0.4.3) trailblazer-option (0.1.2) - twilio-ruby (5.77.0) + twilio-ruby (7.6.0) faraday (>= 0.9, < 3.0) jwt (>= 1.5, < 3.0) nokogiri (>= 1.6, < 2.0) @@ -984,7 +986,7 @@ DEPENDENCIES telephone_number test-prof time_diff - twilio-ruby (~> 5.66) + twilio-ruby twitty (~> 0.1.5) tzinfo-data uglifier diff --git a/app/builders/messages/message_builder.rb b/app/builders/messages/message_builder.rb index b16bd3bb5..67df74863 100644 --- a/app/builders/messages/message_builder.rb +++ b/app/builders/messages/message_builder.rb @@ -7,7 +7,7 @@ class Messages::MessageBuilder @private = params[:private] || false @conversation = conversation @user = user - @message_type = params[:message_type] || 'outgoing' + @message_type = params[:message_type].to_s || 'outgoing' @attachments = params[:attachments] @automation_rule = content_attributes&.dig(:automation_rule_id) return unless params.instance_of?(ActionController::Parameters) diff --git a/app/controllers/api/v1/accounts/voice_controller.rb b/app/controllers/api/v1/accounts/voice_controller.rb index 5127b96e3..e9d7159ac 100644 --- a/app/controllers/api/v1/accounts/voice_controller.rb +++ b/app/controllers/api/v1/accounts/voice_controller.rb @@ -174,7 +174,22 @@ class Api::V1::Accounts::VoiceController < Api::V1::Accounts::BaseController # ---- TwiML ----------------------------------------------------------------- def build_twiml(conference_name) + # For agent legs, we need to add transcription too + account_id = params[:account_id] || Current.account&.id + agent_id = params[:agent_id] || current_user&.id + transcription_url = "#{base_url}/twilio/transcription_callback?account_id=#{account_id}&conference_sid=#{conference_name}&speaker_type=agent&agent_id=#{agent_id}" + Twilio::TwiML::VoiceResponse.new do |r| + # Add transcription for the agent leg too + r.start do |start| + start.transcription( + status_callback_url: transcription_url, + status_callback_method: 'POST', + track: 'inbound_track', # Use inbound_track consistently for conference calls + language_code: 'en-US' + ) + end + r.dial do |dial| dial.conference( conference_name, @@ -187,7 +202,7 @@ class Api::V1::Accounts::VoiceController < Api::V1::Accounts::BaseController statusCallback: conference_callback_url, statusCallbackEvent: 'start end join leave', statusCallbackMethod: 'POST', - participantLabel: "agent-#{params[:agent_id] || current_user.id}" + participantLabel: "agent-#{params[:agent_id] || current_user&.id}" ) end end.to_s diff --git a/app/controllers/twilio/transcription_controller.rb b/app/controllers/twilio/transcription_controller.rb new file mode 100644 index 000000000..78dc47f26 --- /dev/null +++ b/app/controllers/twilio/transcription_controller.rb @@ -0,0 +1,62 @@ +class Twilio::TranscriptionController < ActionController::Base + skip_forgery_protection + + # Receives real-time transcription updates from Twilio + def transcription_callback + # Set Current.account + Current.account = Account.find_by(id: params[:account_id]) + + # Only process transcription content events + if params['TranscriptionEvent'] == 'transcription-content' + process_transcription_content + end + + head :ok + end + + private + + def process_transcription_content + # Extract transcript content from JSON + data = JSON.parse(params['TranscriptionData']) + transcript_content = data['transcript'] + confidence = data['confidence'] + + + # Find conversation by conference_sid from our standard format + display_id = params[:conference_sid].match(/^conf_account_\d+_conv_(\d+)$/)[1] + conversation = Current.account.conversations.find_by(display_id: display_id) + + # Create message based on speaker_type + create_message(conversation, transcript_content, confidence) + end + + def create_message(conversation, content, confidence) + if params[:speaker_type] == 'contact' + # Contact message (incoming) + sender = conversation.contact + message_type = :incoming + else + # Agent message (outgoing) + sender = User.find_by(id: params[:agent_id]) + message_type = :outgoing + end + + # Create the message + Messages::MessageBuilder.new( + sender, + conversation, + content: content, + message_type: message_type, + private: false, + additional_attributes: { + transcription: true, + call_sid: params['CallSid'], + conference_sid: params[:conference_sid], + speaker_type: params[:speaker_type], + confidence: confidence, + track: params['Track'] + } + ).perform + end +end \ No newline at end of file diff --git a/app/controllers/twilio/voice_controller.rb b/app/controllers/twilio/voice_controller.rb index 95b96bb0a..419f0130c 100644 --- a/app/controllers/twilio/voice_controller.rb +++ b/app/controllers/twilio/voice_controller.rb @@ -54,6 +54,26 @@ class Twilio::VoiceController < ActionController::Base render_twiml do |r| r.say(message: 'Please wait while we connect you to an agent') + + # Enable real-time transcription for this call leg + # For outbound calls, we're connecting to the contact, so this track is for the contact + contact_id = conversation.contact_id + callback_url = "#{base_url}/twilio/transcription_callback?account_id=#{@inbox.account_id}&conference_sid=#{conference_name}&speaker_type=contact&contact_id=#{contact_id}" + Rails.logger.info("📞 VoiceController: Setting transcription callback to: #{callback_url}") + + r.start do |start| + start.transcription( + status_callback_url: callback_url, + status_callback_method: 'POST', + track: 'inbound_track', + language_code: 'en-US' + ) + end + + # Set up the conference + conference_callback_url = "#{base_url}/api/v1/accounts/#{@inbox.account_id}/channels/voice/webhooks/conference_status" + Rails.logger.info("📞 VoiceController: Setting conference callback to: #{conference_callback_url}") + r.dial do |d| d.conference( conference_name, @@ -63,7 +83,7 @@ class Twilio::VoiceController < ActionController::Base muted: false, waitUrl: '', earlyMedia: true, - statusCallback: "#{base_url}/api/v1/accounts/#{@inbox.account_id}/channels/voice/webhooks/conference_status", + statusCallback: conference_callback_url, statusCallbackMethod: 'POST', statusCallbackEvent: 'start end join leave', participantLabel: "caller-#{@call_sid.last(8)}" diff --git a/app/javascript/dashboard/components/widgets/conversation/MessagePreview.vue b/app/javascript/dashboard/components/widgets/conversation/MessagePreview.vue index 05d930b51..8cf24cd51 100644 --- a/app/javascript/dashboard/components/widgets/conversation/MessagePreview.vue +++ b/app/javascript/dashboard/components/widgets/conversation/MessagePreview.vue @@ -30,6 +30,13 @@ export default { }; }, computed: { + shouldShowCallStatus() { + // Always show call status for voice channels if present + return ( + this.conversation?.meta?.channel === 'Channel::Voice' && + !!this.conversation?.additional_attributes?.call_status + ); + }, messageByAgent() { const { message_type: messageType } = this.message; return messageType === MESSAGE_TYPE.OUTGOING; @@ -44,33 +51,27 @@ export default { }, // Simple check: Is this a voice channel conversation? isVoiceChannel() { - return this.conversation?.meta?.inbox?.channel_type === 'Channel::Voice'; + return this.conversation?.meta?.channel === 'Channel::Voice'; }, // Check if this is a voice call message isVoiceCall() { return ( - this.message?.content_type === 'voice_call' || - this.message?.content_attributes?.type === 'voice_call' || - this.message?.content_attributes?.data?.callType === 'voice_call' || - this.isVoiceChannel + this.message?.content_type === 'voice_call' ); }, // Get call direction for voice calls isIncomingCall() { - if (!this.isVoiceCall) return false; + if (!this.isVoiceChannel) return false; // First check conversation attributes const direction = this.conversation?.additional_attributes?.call_direction; if (direction) { return direction === 'inbound'; } - - // Then fall back to message type - return this.message.message_type === MESSAGE_TYPE.INCOMING; }, // Get normalized call status callStatus() { - if (!this.isVoiceCall) return null; + if (!this.isVoiceChannel) return null; // Get raw status from conversation const status = this.conversation?.additional_attributes?.call_status; @@ -90,11 +91,11 @@ export default { if (status === 'ringing') return 'ringing'; // Default status - return 'ended'; + return 'active'; }, // Voice call icon based on status voiceCallIcon() { - if (!this.isVoiceCall) return null; + if (!this.isVoiceChannel) return null; const status = this.callStatus; const isIncoming = this.isIncomingCall; @@ -120,13 +121,17 @@ export default { }, parsedLastMessage() { // For voice calls, return status text - if (this.isVoiceCall) { + if (this.isVoiceChannel) { // Get status-based text const status = this.callStatus; const isIncoming = this.isIncomingCall; // Return appropriate status text based on call status and direction if (status === 'active') { + // return last message content if message is not activity and not voice call + if (!this.isMessageAnActivity && !this.isVoiceCall) { + return this.getPlainText(this.message.content); + } return this.$t('CONVERSATION.VOICE_CALL.CALL_IN_PROGRESS'); } @@ -186,17 +191,9 @@ export default {