akadmin/chatwoot
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

chore: Clean up and add transcriptions

Browse Source
This commit is contained in:
Sojan
2025-05-13 03:50:41 -07:00
parent 3d29962969
commit e06525181b
13 changed files with 251 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Gemfile
+1 -1
View File
@@ -87,7 +87,7 @@ gem 'wisper', '2.0.0'
##--- gems for channels ---##
gem 'facebook-messenger'
gem 'line-bot-api'
gem 'twilio-ruby', '~> 5.66'
gem 'twilio-ruby'
# twitty will handle subscription of twitter account events
# gem 'twitty', git: 'https://github.com/chatwoot/twitty'
gem 'twitty', '~> 0.1.5'
Gemfile.lock
+12 -10
View File
@@ -235,8 +235,10 @@ GEM
railties (>= 5.0.0)
faker (3.2.0)
i18n (>= 1.8.11, < 2)
faraday (2.9.0)
faraday-net_http (>= 2.0, < 3.2)
faraday (2.13.1)
faraday-net_http (>= 2.0, < 3.5)
json
logger
faraday-follow_redirects (0.3.0)
faraday (>= 1, < 3)
faraday-mashify (0.1.1)
@@ -244,8 +246,8 @@ GEM
hashie
faraday-multipart (1.0.4)
multipart-post (~> 2)
faraday-net_http (3.1.0)
net-http
faraday-net_http (3.4.0)
net-http (>= 0.5.0)
faraday-net_http_persistent (2.1.0)
faraday (~> 2.5)
net-http-persistent (~> 4.0)
@@ -388,7 +390,7 @@ GEM
rails-dom-testing (>= 1, < 3)
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
json (2.6.3)
json (2.11.3)
json_refs (0.1.8)
hana
json_schemer (0.2.24)
@@ -403,7 +405,7 @@ GEM
judoscale-sidekiq (1.8.2)
judoscale-ruby (= 1.8.2)
sidekiq (>= 5.0)
jwt (2.8.1)
jwt (2.10.1)
base64
kaminari (1.2.2)
activesupport (>= 4.1.0)
@@ -445,7 +447,7 @@ GEM
llhttp-ffi (0.4.0)
ffi-compiler (~> 1.0)
rake (~> 13.0)
logger (1.6.0)
logger (1.7.0)
lograge (0.14.0)
actionpack (>= 4)
activesupport (>= 4)
@@ -481,7 +483,7 @@ GEM
mutex_m (0.3.0)
neighbor (0.2.3)
activerecord (>= 5.2)
net-http (0.4.1)
net-http (0.6.0)
uri
net-http-persistent (4.0.2)
connection_pool (~> 2.2)
@@ -800,7 +802,7 @@ GEM
i18n
timeout (0.4.3)
trailblazer-option (0.1.2)
twilio-ruby (5.77.0)
twilio-ruby (7.6.0)
faraday (>= 0.9, < 3.0)
jwt (>= 1.5, < 3.0)
nokogiri (>= 1.6, < 2.0)
@@ -984,7 +986,7 @@ DEPENDENCIES
telephone_number
test-prof
time_diff
twilio-ruby (~> 5.66)
twilio-ruby
twitty (~> 0.1.5)
tzinfo-data
uglifier
app/builders/messages/message_builder.rb
+1 -1
View File
@@ -7,7 +7,7 @@ class Messages::MessageBuilder
@private = params[:private] || false
@conversation = conversation
@user = user
@message_type = params[:message_type] || 'outgoing'
@message_type = params[:message_type].to_s || 'outgoing'
@attachments = params[:attachments]
@automation_rule = content_attributes&.dig(:automation_rule_id)
return unless params.instance_of?(ActionController::Parameters)
app/controllers/api/v1/accounts/voice_controller.rb
+16 -1
View File
@@ -174,7 +174,22 @@ class Api::V1::Accounts::VoiceController < Api::V1::Accounts::BaseController
# ---- TwiML -----------------------------------------------------------------
def build_twiml(conference_name)
# For agent legs, we need to add transcription too
account_id = params[:account_id] || Current.account&.id
agent_id = params[:agent_id] || current_user&.id
transcription_url = "#{base_url}/twilio/transcription_callback?account_id=#{account_id}&conference_sid=#{conference_name}&speaker_type=agent&agent_id=#{agent_id}"
Twilio::TwiML::VoiceResponse.new do |r|
# Add transcription for the agent leg too
r.start do |start|
start.transcription(
status_callback_url: transcription_url,
status_callback_method: 'POST',
track: 'inbound_track', # Use inbound_track consistently for conference calls
language_code: 'en-US'
)
end
r.dial do |dial|
dial.conference(
conference_name,
@@ -187,7 +202,7 @@ class Api::V1::Accounts::VoiceController < Api::V1::Accounts::BaseController
statusCallback: conference_callback_url,
statusCallbackEvent: 'start end join leave',
statusCallbackMethod: 'POST',
participantLabel: "agent-#{params[:agent_id] || current_user.id}"
participantLabel: "agent-#{params[:agent_id] || current_user&.id}"
)
end
end.to_s
app/controllers/twilio/transcription_controller.rb
+62
View File
@@ -0,0 +1,62 @@
class Twilio::TranscriptionController < ActionController::Base
skip_forgery_protection
# Receives real-time transcription updates from Twilio
def transcription_callback
# Set Current.account
Current.account = Account.find_by(id: params[:account_id])
# Only process transcription content events
if params['TranscriptionEvent'] == 'transcription-content'
process_transcription_content
end
head :ok
end
private
def process_transcription_content
# Extract transcript content from JSON
data = JSON.parse(params['TranscriptionData'])
transcript_content = data['transcript']
confidence = data['confidence']
# Find conversation by conference_sid from our standard format
display_id = params[:conference_sid].match(/^conf_account_\d+_conv_(\d+)$/)[1]
conversation = Current.account.conversations.find_by(display_id: display_id)
# Create message based on speaker_type
create_message(conversation, transcript_content, confidence)
end
def create_message(conversation, content, confidence)
if params[:speaker_type] == 'contact'
# Contact message (incoming)
sender = conversation.contact
message_type = :incoming
else
# Agent message (outgoing)
sender = User.find_by(id: params[:agent_id])
message_type = :outgoing
end
# Create the message
Messages::MessageBuilder.new(
sender,
conversation,
content: content,
message_type: message_type,
private: false,
additional_attributes: {
transcription: true,
call_sid: params['CallSid'],
conference_sid: params[:conference_sid],
speaker_type: params[:speaker_type],
confidence: confidence,
track: params['Track']
}
).perform
end
end
app/controllers/twilio/voice_controller.rb
+21 -1
View File
@@ -54,6 +54,26 @@ class Twilio::VoiceController < ActionController::Base
render_twiml do |r|
r.say(message: 'Please wait while we connect you to an agent')
# Enable real-time transcription for this call leg
# For outbound calls, we're connecting to the contact, so this track is for the contact
contact_id = conversation.contact_id
callback_url = "#{base_url}/twilio/transcription_callback?account_id=#{@inbox.account_id}&conference_sid=#{conference_name}&speaker_type=contact&contact_id=#{contact_id}"
Rails.logger.info("📞 VoiceController: Setting transcription callback to: #{callback_url}")
r.start do |start|
start.transcription(
status_callback_url: callback_url,
status_callback_method: 'POST',
track: 'inbound_track',
language_code: 'en-US'
)
end
# Set up the conference
conference_callback_url = "#{base_url}/api/v1/accounts/#{@inbox.account_id}/channels/voice/webhooks/conference_status"
Rails.logger.info("📞 VoiceController: Setting conference callback to: #{conference_callback_url}")
r.dial do |d|
d.conference(
conference_name,
@@ -63,7 +83,7 @@ class Twilio::VoiceController < ActionController::Base
muted: false,
waitUrl: '',
earlyMedia: true,
statusCallback: "#{base_url}/api/v1/accounts/#{@inbox.account_id}/channels/voice/webhooks/conference_status",
statusCallback: conference_callback_url,
statusCallbackMethod: 'POST',
statusCallbackEvent: 'start end join leave',
participantLabel: "caller-#{@call_sid.last(8)}"
app/javascript/dashboard/components/widgets/conversation/MessagePreview.vue
+90 -62
View File
@@ -30,6 +30,13 @@ export default {
};
},
computed: {
shouldShowCallStatus() {
// Always show call status for voice channels if present
return (
this.conversation?.meta?.channel === 'Channel::Voice' &&
!!this.conversation?.additional_attributes?.call_status
);
},
messageByAgent() {
const { message_type: messageType } = this.message;
return messageType === MESSAGE_TYPE.OUTGOING;
@@ -44,33 +51,27 @@ export default {
},
// Simple check: Is this a voice channel conversation?
isVoiceChannel() {
return this.conversation?.meta?.inbox?.channel_type === 'Channel::Voice';
return this.conversation?.meta?.channel === 'Channel::Voice';
},
// Check if this is a voice call message
isVoiceCall() {
return (
this.message?.content_type === 'voice_call' ||
this.message?.content_attributes?.type === 'voice_call' ||
this.message?.content_attributes?.data?.callType === 'voice_call' ||
this.isVoiceChannel
this.message?.content_type === 'voice_call'
);
},
// Get call direction for voice calls
isIncomingCall() {
if (!this.isVoiceCall) return false;
if (!this.isVoiceChannel) return false;
// First check conversation attributes
const direction = this.conversation?.additional_attributes?.call_direction;
if (direction) {
return direction === 'inbound';
}
// Then fall back to message type
return this.message.message_type === MESSAGE_TYPE.INCOMING;
},
// Get normalized call status
callStatus() {
if (!this.isVoiceCall) return null;
if (!this.isVoiceChannel) return null;
// Get raw status from conversation
const status = this.conversation?.additional_attributes?.call_status;
@@ -90,11 +91,11 @@ export default {
if (status === 'ringing') return 'ringing';
// Default status
return 'ended';
return 'active';
},
// Voice call icon based on status
voiceCallIcon() {
if (!this.isVoiceCall) return null;
if (!this.isVoiceChannel) return null;
const status = this.callStatus;
const isIncoming = this.isIncomingCall;
@@ -120,13 +121,17 @@ export default {
},
parsedLastMessage() {
// For voice calls, return status text
if (this.isVoiceCall) {
if (this.isVoiceChannel) {
// Get status-based text
const status = this.callStatus;
const isIncoming = this.isIncomingCall;
// Return appropriate status text based on call status and direction
if (status === 'active') {
// return last message content if message is not activity and not voice call
if (!this.isMessageAnActivity && !this.isVoiceCall) {
return this.getPlainText(this.message.content);
}
return this.$t('CONVERSATION.VOICE_CALL.CALL_IN_PROGRESS');
}
@@ -186,17 +191,9 @@ export default {
<template>
<div class="overflow-hidden text-ellipsis whitespace-nowrap">
<template v-if="showMessageType">
<fluent-icon
v-if="isMessagePrivate"
size="16"
class="-mt-0.5 align-middle text-slate-600 dark:text-slate-300 inline-block"
icon="lock-closed"
/>
<!-- Voice calls with phosphor icons (non-filled variants) -->
<!-- Always show call status for voice channels if present -->
<template v-if="shouldShowCallStatus">
<span
v-else-if="isVoiceCall"
class="-mt-0.5 align-middle inline-block mr-1"
:class="{
'text-red-600 dark:text-red-400': callStatus === 'missed' || callStatus === 'no-answer',
@@ -207,55 +204,86 @@ export default {
<!-- Missed call icon -->
<i v-if="callStatus === 'missed' || callStatus === 'no-answer'"
class="i-ph-phone-x text-base"></i>
<!-- Active call icon -->
<i v-else-if="callStatus === 'active'"
class="i-ph-phone-call text-base"></i>
<!-- Incoming call icon -->
<i v-else-if="(callStatus === 'ended' && isIncomingCall) || (isIncomingCall)"
class="i-ph-phone-incoming text-base"></i>
<!-- Outgoing call icon -->
<i v-else
class="i-ph-phone-outgoing text-base"></i>
</span>
<fluent-icon
v-else-if="messageByAgent"
size="16"
class="-mt-0.5 align-middle text-slate-600 dark:text-slate-300 inline-block"
icon="arrow-reply"
/>
<fluent-icon
v-else-if="isMessageAnActivity"
size="16"
class="-mt-0.5 align-middle text-slate-600 dark:text-slate-300 inline-block"
icon="info"
/>
<span>{{ parsedLastMessage }}</span>
</template>
<template v-else>
<template v-if="showMessageType">
<fluent-icon
v-if="isMessagePrivate"
size="16"
class="-mt-0.5 align-middle text-slate-600 dark:text-slate-300 inline-block"
icon="lock-closed"
/>
<!-- Voice calls with phosphor icons (non-filled variants) -->
<span
v-else-if="isVoiceCall"
class="-mt-0.5 align-middle inline-block mr-1"
:class="{
'text-red-600 dark:text-red-400': callStatus === 'missed' || callStatus === 'no-answer',
'text-green-600 dark:text-green-400': callStatus === 'active' || callStatus === 'ringing',
'text-slate-600 dark:text-slate-300': callStatus === 'ended'
}"
>
<!-- Missed call icon -->
<i v-if="callStatus === 'missed' || callStatus === 'no-answer'"
class="i-ph-phone-x text-base"></i>
<!-- Active call icon -->
<i v-else-if="callStatus === 'active'"
class="i-ph-phone-call text-base"></i>
<!-- Incoming call icon -->
<i v-else-if="(callStatus === 'ended' && isIncomingCall) || (isIncomingCall)"
class="i-ph-phone-incoming text-base"></i>
<!-- Outgoing call icon -->
<i v-else
class="i-ph-phone-outgoing text-base"></i>
</span>
<fluent-icon
v-else-if="messageByAgent"
size="16"
class="-mt-0.5 align-middle text-slate-600 dark:text-slate-300 inline-block"
icon="arrow-reply"
/>
<fluent-icon
v-else-if="isMessageAnActivity"
size="16"
class="-mt-0.5 align-middle text-slate-600 dark:text-slate-300 inline-block"
icon="info"
/>
</template>
<span v-if="message.content && isMessageSticker">
<fluent-icon
size="16"
class="-mt-0.5 align-middle inline-block text-slate-600 dark:text-slate-300"
icon="image"
/>
{{ $t('CHAT_LIST.ATTACHMENTS.image.CONTENT') }}
</span>
<span v-else-if="message.content || isVoiceCall">
{{ parsedLastMessage }}
</span>
<span v-else-if="message.attachments">
<fluent-icon
v-if="attachmentIcon && showMessageType"
size="16"
class="-mt-0.5 align-middle inline-block text-slate-600 dark:text-slate-300"
:icon="attachmentIcon"
/>
{{ $t(`${attachmentMessageContent}`) }}
</span>
<span v-else>
{{ defaultEmptyMessage || $t('CHAT_LIST.NO_CONTENT') }}
</span>
</template>
<span v-if="message.content && isMessageSticker">
<fluent-icon
size="16"
class="-mt-0.5 align-middle inline-block text-slate-600 dark:text-slate-300"
icon="image"
/>
{{ $t('CHAT_LIST.ATTACHMENTS.image.CONTENT') }}
</span>
<span v-else-if="message.content || isVoiceCall">
{{ parsedLastMessage }}
</span>
<span v-else-if="message.attachments">
<fluent-icon
v-if="attachmentIcon && showMessageType"
size="16"
class="-mt-0.5 align-middle inline-block text-slate-600 dark:text-slate-300"
:icon="attachmentIcon"
/>
{{ $t(`${attachmentMessageContent}`) }}
</span>
<span v-else>
{{ defaultEmptyMessage || $t('CHAT_LIST.NO_CONTENT') }}
</span>
</div>
</template>
app/javascript/dashboard/i18n/locale/en/conversation.json
+3 -3
View File
@@ -243,11 +243,11 @@
"ACTIVE": "Call in progress",
"MISSED": "Missed Call",
"ENDED": "Call Ended",
"INCOMING": "Incoming call...",
"INCOMING": "Incoming call",
"OUTGOING": "Call started...",
"INCOMING_CALL": "Incoming call...",
"INCOMING_CALL": "Incoming call",
"OUTGOING_CALL": "Outgoing call",
"CALL_IN_PROGRESS": "Call in progress...",
"CALL_IN_PROGRESS": "Call in progress",
"NO_ANSWER": "No answer",
"MISSED_CALL": "Missed call",
"CALL_ENDED": "Call ended",
app/presenters/conversations/event_data_presenter.rb
+2 -1
View File
@@ -32,7 +32,8 @@ class Conversations::EventDataPresenter < SimpleDelegator
sender: contact.push_event_data,
assignee: assignee&.push_event_data,
team: team&.push_event_data,
hmac_verified: contact_inbox&.hmac_verified
hmac_verified: contact_inbox&.hmac_verified,
channel: inbox.try(:channel_type)
}
end
app/services/voice/call_status/manager.rb
+15 -12
View File
@@ -81,19 +81,22 @@ module Voice
# This provides a clean migration path from MessageUpdateService
def create_activity_message(content, additional_attributes = {})
return nil if content.blank?
Rails.logger.info("📝 [CallStatusManager] Creating activity message: '#{content}'")
# Create message
Messages::MessageBuilder.new(
nil,
conversation,
{
content: content,
message_type: :activity,
additional_attributes: additional_attributes
}
).perform
# Activity messages should not have a sender
# Pass nil for user and set the sender explicitly to nil
message = conversation.messages.create!(
account_id: conversation.account_id,
inbox_id: conversation.inbox_id,
message_type: :activity,
content: content,
sender: nil,
additional_attributes: additional_attributes
)
Rails.logger.info("📝 [CallStatusManager] Created activity message ID #{message.id}")
message
end
# Process a call status update from any provider (e.g., Twilio, Vonage)
app/services/voice/incoming_call_service.rb
+25 -5
View File
@@ -22,6 +22,8 @@ module Voice
twiml
rescue StandardError => e
# Log the error
Rails.logger.error("Error processing incoming call: #{e.message}")
# Return a simple error TwiML
error_twiml(e.message)
@@ -155,11 +157,10 @@ module Voice
# First process ringing status
status_manager.process_status_update('ringing', nil, true)
# Then add a custom message about the incoming call
# Then add a custom message about the incoming call - it will be created without a sender
activity_message = status_manager.create_activity_message(
"Incoming call from #{@contact.name.presence || caller_info[:from_number]}"
)
end
def broadcast_call_status
@@ -196,12 +197,29 @@ module Voice
def generate_twiml_response
conference_name = @conversation.additional_attributes['conference_sid']
Rails.logger.info("📞 IncomingCallService: Generating TwiML with conference name: #{conference_name}")
response = Twilio::TwiML::VoiceResponse.new
response.say(message: 'Thank you for calling. Please wait while we connect you with an agent.')
callback_url = "#{base_url}/api/v1/accounts/#{account.id}/channels/voice/webhooks/conference_status"
# Setup callback URLs - include conference name and speaker_type in transcription URL
conference_callback_url = "#{base_url}/api/v1/accounts/#{account.id}/channels/voice/webhooks/conference_status"
transcription_url = "#{base_url}/twilio/transcription_callback?account_id=#{account.id}&conference_sid=#{conference_name}&speaker_type=contact&contact_id=#{@contact.id}"
Rails.logger.info("📞 IncomingCallService: Setting transcription callback to: #{transcription_url}")
Rails.logger.info("📞 IncomingCallService: Setting conference callback to: #{conference_callback_url}")
# Start real-time transcription for this caller's leg
response.start do |s|
s.transcription(
status_callback_url: transcription_url,
status_callback_method: 'POST',
track: 'inbound_track', # Must be inbound_track or outbound_track per Twilio API
language_code: 'en-US'
)
end
# Now add the caller to the conference
response.dial do |dial|
dial.conference(
conference_name,
@@ -210,14 +228,16 @@ module Voice
beep: false,
muted: false,
waitUrl: '',
statusCallback: callback_url,
statusCallback: conference_callback_url,
statusCallbackMethod: 'POST',
statusCallbackEvent: 'start end join leave',
participantLabel: "caller-#{caller_info[:call_sid].last(8)}"
)
end
response.to_s
result = response.to_s
Rails.logger.info("📞 IncomingCallService: Generated TwiML: #{result}")
result
end
def error_twiml(message)
app/services/voice/outgoing_call_service.rb
-10
View File
@@ -203,16 +203,6 @@ module Voice
data: broadcast_data
}
)
# Broadcast the conversation and message
ActionCableBroadcastJob.perform_later(
@conversation.account_id,
'conversation.created',
@conversation.push_event_data.merge(
message: @widget_message.push_event_data,
status: 'open'
)
)
end
end
end
config/routes.rb
+3
View File
@@ -510,6 +510,9 @@ Rails.application.routes.draw do
resources :callback, only: [:create]
resources :delivery_status, only: [:create]
# Transcription webhook
post :transcription_callback, to: 'transcription#transcription_callback'
# Use resource scope to avoid plural/singular confusion
resource :voice, only: [], controller: 'voice' do
collection do