diff --git a/app/javascript/dashboard/api/whatsappCalls.js b/app/javascript/dashboard/api/whatsappCalls.js index 450390897..3b35588b1 100644 --- a/app/javascript/dashboard/api/whatsappCalls.js +++ b/app/javascript/dashboard/api/whatsappCalls.js @@ -6,6 +6,10 @@ class WhatsappCallsAPI extends ApiClient { super('whatsapp_calls', { accountScoped: true }); } + show(callId) { + return axios.get(`${this.url}/${callId}`); + } + accept(callId, sdpAnswer) { return axios.post(`${this.url}/${callId}/accept`, { sdp_answer: sdpAnswer, @@ -26,6 +30,14 @@ class WhatsappCallsAPI extends ApiClient { sdp_offer: sdpOffer, }); } + + uploadRecording(callId, blob) { + const formData = new FormData(); + formData.append('recording', blob, `call-${callId}.webm`); + return axios.post(`${this.url}/${callId}/upload_recording`, formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }); + } } export default new WhatsappCallsAPI(); diff --git a/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue b/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue index 5a7d39a4e..c192e6ddc 100644 --- a/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue +++ b/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue @@ -1,7 +1,9 @@ @@ -90,14 +168,75 @@ const bgColor = computed(() => BG_COLOR_MAP[status.value] || 'bg-n-teal-9'); /> - + {{ $t(labelKey) }} - + + {{ + $t('CONVERSATION.VOICE_CALL.ANSWERED_BY', { + name: answeredByText, + }) + }} + + {{ $t(subtextKey) }} + + {{ formattedDuration }} + + + + + + {{ $t(joinButtonLabel) }} + + + + + + {{ $t('CONVERSATION.VOICE_CALL.AUDIO_NOT_SUPPORTED') }} + + + + + + + {{ $t('CONVERSATION.VOICE_CALL.TRANSCRIPT') }} + + + {{ transcript }} + diff --git a/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue b/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue index 6b44ca9c5..98131a824 100644 --- a/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue +++ b/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue @@ -20,6 +20,7 @@ import { useWhatsappCallsStore, setOutboundCallProperty, } from 'dashboard/stores/whatsappCalls'; +import { startCallRecording } from 'dashboard/composables/useWhatsappCallSession'; const props = defineProps({ chat: { @@ -128,6 +129,7 @@ const initiateWhatsappCall = async () => { isInitiatingCall.value = true; let pc = null; let localStream = null; + let waCallId = null; try { localStream = await navigator.mediaDevices.getUserMedia({ audio: true }); pc = new RTCPeerConnection({ @@ -146,6 +148,8 @@ const initiateWhatsappCall = async () => { setOutboundCallProperty('audio', audio); // Remote audio arrived — callee picked up, transition from ringing to connected whatsappCallsStore.markActiveCallConnected(); + // Start recording both local + remote audio + if (waCallId) startCallRecording(pc, localStream, waCallId); }; pc.oniceconnectionstatechange = () => { @@ -186,6 +190,7 @@ const initiateWhatsappCall = async () => { }); const outboundCallId = response.data?.call_id; + waCallId = response.data?.id; setOutboundCallProperty('pc', pc); setOutboundCallProperty('stream', localStream); setOutboundCallProperty('callId', outboundCallId); diff --git a/app/javascript/dashboard/composables/useWhatsappCallSession.js b/app/javascript/dashboard/composables/useWhatsappCallSession.js index 210032741..cd720d309 100644 --- a/app/javascript/dashboard/composables/useWhatsappCallSession.js +++ b/app/javascript/dashboard/composables/useWhatsappCallSession.js @@ -5,18 +5,258 @@ import { getOutboundCallState, } from 'dashboard/stores/whatsappCalls'; import WhatsappCallsAPI from 'dashboard/api/whatsappCalls'; +import Auth from 'dashboard/api/auth'; import Timer from 'dashboard/helper/Timer'; +// ── Module-level WebRTC state for inbound calls accepted from anywhere ── +// Kept at module scope so both the composable and acceptWhatsappCallById share it. +let inboundPc = null; +let inboundStream = null; +let inboundAudio = null; + +// ── Module-level recording state ── +let mediaRecorder = null; +let recordedChunks = []; +let recordingCallId = null; + +function cleanupInboundWebRTC() { + if (inboundStream) { + inboundStream.getTracks().forEach(track => track.stop()); + inboundStream = null; + } + if (inboundPc) { + inboundPc.close(); + inboundPc = null; + } + if (inboundAudio) { + inboundAudio.srcObject = null; + if (inboundAudio.parentNode) { + inboundAudio.parentNode.removeChild(inboundAudio); + } + inboundAudio = null; + } +} + +/** + * Start recording both local and remote audio tracks via MediaRecorder. + * Mixes them into a single stream using AudioContext. + */ +export function startCallRecording(pc, localStream, callId) { + try { + const ctx = new AudioContext(); + const dest = ctx.createMediaStreamDestination(); + + // Add local mic track + if (localStream) { + const localSource = ctx.createMediaStreamSource(localStream); + localSource.connect(dest); + } + + // Add remote tracks from peer connection + pc.getReceivers().forEach(receiver => { + if (receiver.track && receiver.track.kind === 'audio') { + const remoteStream = new MediaStream([receiver.track]); + const remoteSource = ctx.createMediaStreamSource(remoteStream); + remoteSource.connect(dest); + } + }); + + recordedChunks = []; + recordingCallId = callId; + const recorder = new MediaRecorder(dest.stream, { + mimeType: 'audio/webm;codecs=opus', + }); + + recorder.ondataavailable = e => { + if (e.data.size > 0) recordedChunks.push(e.data); + }; + + mediaRecorder = recorder; + recorder.start(1000); + } catch (err) { + // eslint-disable-next-line no-console + console.error('[WhatsApp Call] Failed to start recording:', err); + } +} + +/** + * Stop recording and upload the audio blob to the backend. + */ +function stopAndUploadRecording(callId) { + if (!mediaRecorder || mediaRecorder.state === 'inactive') return; + + const id = callId || recordingCallId; + + mediaRecorder.onstop = () => { + if (recordedChunks.length === 0 || !id) return; + + const blob = new Blob(recordedChunks, { type: 'audio/webm' }); + recordedChunks = []; + recordingCallId = null; + + WhatsappCallsAPI.uploadRecording(id, blob).catch(err => { + // eslint-disable-next-line no-console + console.error('[WhatsApp Call] Failed to upload recording:', err); + }); + }; + + mediaRecorder.stop(); + mediaRecorder = null; +} + +function waitForIceGatheringComplete(pc) { + return new Promise((resolve, reject) => { + if (pc.iceGatheringState === 'complete') { + resolve(); + return; + } + const timeout = setTimeout(() => { + // eslint-disable-next-line no-console + console.warn( + '[WhatsApp Call] ICE gathering timed out, sending partial SDP' + ); + resolve(); + }, 10000); + + pc.onicegatheringstatechange = () => { + if (pc.iceGatheringState === 'complete') { + clearTimeout(timeout); + resolve(); + } + }; + pc.oniceconnectionstatechange = () => { + if (pc.iceConnectionState === 'failed') { + clearTimeout(timeout); + reject(new Error('ICE connection failed')); + } + }; + }); +} + +/** + * Core accept logic: creates WebRTC session and posts SDP to backend. + * Can be called from anywhere — composable, widget, or bubble. + * Returns { success: true } or { success: false, error }. + */ +async function doAcceptCall(call) { + cleanupInboundWebRTC(); + + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + inboundStream = stream; + + const iceServers = call.iceServers?.length + ? call.iceServers + : [{ urls: 'stun:stun.l.google.com:19302' }]; + + const pc = new RTCPeerConnection({ iceServers }); + inboundPc = pc; + + stream.getTracks().forEach(track => pc.addTrack(track, stream)); + + pc.ontrack = event => { + const [remoteStream] = event.streams; + if (!remoteStream) return; + if (!inboundAudio) { + const audio = document.createElement('audio'); + audio.autoplay = true; + document.body.appendChild(audio); + inboundAudio = audio; + } + inboundAudio.srcObject = remoteStream; + inboundAudio.play().catch(() => {}); + + // Start recording once remote audio is available + startCallRecording(pc, stream, call.id); + }; + + await pc.setRemoteDescription({ type: 'offer', sdp: call.sdpOffer }); + const answer = await pc.createAnswer(); + await pc.setLocalDescription(answer); + await waitForIceGatheringComplete(pc); + + const completeSdp = pc.localDescription.sdp; + await WhatsappCallsAPI.accept(call.id, completeSdp); + + return { success: true }; +} + +/** + * Standalone function callable from VoiceCall bubble. + * Fetches call data if needed, runs WebRTC accept, updates store. + */ +export async function acceptWhatsappCallById(waCallId) { + const callsStore = useWhatsappCallsStore(); + + if (callsStore.hasActiveCall) { + return { success: false, error: 'active_call_exists' }; + } + + // 1. Check if the call is already in the incoming store + let call = callsStore.incomingCalls.find( + c => c.id === waCallId || c.waCallId === waCallId + ); + + // 2. Not in store (page was refreshed) → fetch from API + if (!call) { + const { data } = await WhatsappCallsAPI.show(waCallId); + if (data.status !== 'ringing') { + return { success: false, error: 'not_ringing' }; + } + call = { + id: data.id, + callId: data.call_id, + waCallId: data.id, + direction: data.direction, + inboxId: data.inbox_id, + conversationId: data.conversation_id, + sdpOffer: data.sdp_offer, + iceServers: data.ice_servers, + caller: data.caller, + }; + callsStore.addIncomingCall(call); + } + + // 3. Run the WebRTC accept + await doAcceptCall(call); + + // 4. Move from incoming to active + callsStore.removeIncomingCall(call.callId); + callsStore.setActiveCall({ ...call }); + + return { success: true, call }; +} + +/** + * Fire-and-forget terminate request using fetch + keepalive. + * Works reliably inside beforeunload / pagehide where axios won't complete. + */ +function terminateCallOnUnload(callId) { + const authData = Auth.hasAuthCookie() ? Auth.getAuthData() : {}; + const accountId = + window.location.pathname.includes('/app/accounts') && + window.location.pathname.split('/')[3]; + if (!accountId) return; + + const url = `/api/v1/accounts/${accountId}/whatsapp_calls/${callId}/terminate`; + fetch(url, { + method: 'POST', + keepalive: true, + headers: { + 'Content-Type': 'application/json', + 'access-token': authData['access-token'] || '', + 'token-type': authData['token-type'] || '', + client: authData.client || '', + expiry: authData.expiry || '', + uid: authData.uid || '', + }, + }).catch(() => {}); +} + +// ── Composable (used by WhatsappCallWidget for floating UI + timer) ── export function useWhatsappCallSession() { const { t } = useI18n(); const callsStore = useWhatsappCallsStore(); - // WebRTC internals - let peerConnection = null; - let localStream = null; - const remoteAudio = ref(null); - - // UI state const isAccepting = ref(false); const isMuted = ref(false); const callError = ref(null); @@ -44,33 +284,25 @@ export function useWhatsappCallSession() { return `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`; }); - const cleanupWebRTC = () => { - if (localStream) { - localStream.getTracks().forEach(track => track.stop()); - localStream = null; - } - if (peerConnection) { - peerConnection.close(); - peerConnection = null; - } - if (remoteAudio.value) { - remoteAudio.value.srcObject = null; - // Remove dynamically created audio element from DOM - if (remoteAudio.value.parentNode) { - remoteAudio.value.parentNode.removeChild(remoteAudio.value); - } - remoteAudio.value = null; - } - }; - - // Register cleanup callback so store can trigger WebRTC teardown on external events + // Register cleanup so external call-end events can teardown WebRTC callsStore.registerCleanupCallback(() => { - cleanupWebRTC(); + stopAndUploadRecording(); + cleanupInboundWebRTC(); durationTimer.stop(); callDuration.value = 0; }); - // Start timer when an outbound call becomes connected (SDP answer received) + // Terminate active call on page close / reload + const handleBeforeUnload = () => { + const call = callsStore.activeCall; + if (call?.id) { + terminateCallOnUnload(call.id); + cleanupInboundWebRTC(); + } + }; + window.addEventListener('beforeunload', handleBeforeUnload); + + // Start timer when outbound call becomes connected watch(activeCall, call => { if ( call?.direction === 'outbound' && @@ -82,49 +314,8 @@ export function useWhatsappCallSession() { }); /** - * Waits for ICE candidate gathering to complete so the SDP contains all candidates. - * Meta's REST API doesn't support trickle ICE — the full SDP must be sent at once. - */ - const waitForIceGatheringComplete = pc => - new Promise((resolve, reject) => { - if (pc.iceGatheringState === 'complete') { - resolve(); - return; - } - - const timeout = setTimeout(() => { - // If gathering hasn't finished in 10s, send what we have - // eslint-disable-next-line no-console - console.warn( - '[WhatsApp Call] ICE gathering timed out, sending partial SDP' - ); - resolve(); - }, 10000); - - pc.onicegatheringstatechange = () => { - if (pc.iceGatheringState === 'complete') { - clearTimeout(timeout); - resolve(); - } - }; - - // Also reject if connection fails during gathering - pc.oniceconnectionstatechange = () => { - if (pc.iceConnectionState === 'failed') { - clearTimeout(timeout); - reject(new Error('ICE connection failed')); - } - }; - }); - - /** - * Accepts an incoming WhatsApp call: - * 1. Requests mic access - * 2. Creates RTCPeerConnection with ICE servers from the call payload - * 3. Sets remote description (the SDP offer from Meta) - * 4. Creates an SDP answer - * 5. Waits for ICE gathering to complete (Meta needs full SDP, no trickle ICE) - * 6. Posts the complete SDP answer to Chatwoot backend → Meta API + * Accept an incoming call — used by the floating widget buttons. + * Uses the same doAcceptCall core + starts the timer. */ const acceptCall = async call => { if (isAccepting.value) return; @@ -132,74 +323,9 @@ export function useWhatsappCallSession() { callError.value = null; try { - // 1. Get microphone access - localStream = await navigator.mediaDevices.getUserMedia({ audio: true }); - - // 2. Build ICE config - const iceServers = call.iceServers?.length - ? call.iceServers - : [{ urls: 'stun:stun.l.google.com:19302' }]; - - // 3. Create RTCPeerConnection - peerConnection = new RTCPeerConnection({ iceServers }); - - // 4. Add local audio tracks - localStream.getTracks().forEach(track => { - peerConnection.addTrack(track, localStream); - }); - - // 5. Handle remote audio stream → play via element - peerConnection.ontrack = event => { - const [stream] = event.streams; - if (!stream) return; - - if (remoteAudio.value) { - remoteAudio.value.srcObject = stream; - remoteAudio.value.play().catch(e => { - // eslint-disable-next-line no-console - console.warn('[WhatsApp Call] Audio autoplay blocked:', e); - }); - } else { - const audio = document.createElement('audio'); - audio.srcObject = stream; - audio.autoplay = true; - document.body.appendChild(audio); - remoteAudio.value = audio; - } - }; - - // 6. Monitor ICE connection state for debugging - peerConnection.oniceconnectionstatechange = () => { - // eslint-disable-next-line no-console - console.log( - '[WhatsApp Call] ICE state:', - peerConnection?.iceConnectionState - ); - }; - - // 7. Set remote description from Meta's SDP offer - await peerConnection.setRemoteDescription({ - type: 'offer', - sdp: call.sdpOffer, - }); - - // 8. Create SDP answer - const answer = await peerConnection.createAnswer(); - await peerConnection.setLocalDescription(answer); - - // 9. Wait for ICE gathering to complete so SDP has all candidates - await waitForIceGatheringComplete(peerConnection); - - // 10. Post the COMPLETE SDP answer (with all ICE candidates) to backend - const completeSdp = peerConnection.localDescription.sdp; - await WhatsappCallsAPI.accept(call.id, completeSdp); - - // 11. Mark as active in store + await doAcceptCall(call); callsStore.removeIncomingCall(call.callId); - callsStore.setActiveCall({ - ...call, - }); - + callsStore.setActiveCall({ ...call }); durationTimer.start(); } catch (err) { callError.value = @@ -208,7 +334,7 @@ export function useWhatsappCallSession() { : t('WHATSAPP_CALL.CALL_FAILED'); // eslint-disable-next-line no-console console.error('[WhatsApp Call] acceptCall error:', err); - cleanupWebRTC(); + cleanupInboundWebRTC(); } finally { isAccepting.value = false; } @@ -228,14 +354,14 @@ export function useWhatsappCallSession() { const call = activeCall.value; if (!call) return; + stopAndUploadRecording(call.id); + try { await WhatsappCallsAPI.terminate(call.id); } catch { - // Best effort — always cleanup locally + // Best effort } finally { - // For inbound calls, cleanup composable-managed WebRTC - cleanupWebRTC(); - // For outbound calls, cleanup module-scoped WebRTC via store + cleanupInboundWebRTC(); callsStore.handleCallEnded(call.callId); callsStore.clearActiveCall(); durationTimer.stop(); @@ -244,9 +370,7 @@ export function useWhatsappCallSession() { }; const toggleMute = () => { - // For inbound calls, localStream is managed by this composable. - // For outbound calls, the stream is in module-scoped outbound state. - const stream = localStream || getOutboundCallState().stream; + const stream = inboundStream || getOutboundCallState().stream; if (!stream) return; const audioTrack = stream.getAudioTracks()[0]; if (!audioTrack) return; @@ -259,6 +383,7 @@ export function useWhatsappCallSession() { }; onUnmounted(() => { + window.removeEventListener('beforeunload', handleBeforeUnload); durationTimer.stop(); }); diff --git a/app/javascript/dashboard/helper/voice.js b/app/javascript/dashboard/helper/voice.js index 9f753a811..b9eb7517e 100644 --- a/app/javascript/dashboard/helper/voice.js +++ b/app/javascript/dashboard/helper/voice.js @@ -18,6 +18,10 @@ const isVoiceCallMessage = message => { return CONTENT_TYPES.VOICE_CALL === message?.content_type; }; +const isWhatsappCall = message => { + return message?.content_attributes?.data?.call_source === 'whatsapp'; +}; + const shouldSkipCall = (callDirection, senderId, currentUserId) => { return callDirection === 'outbound' && senderId !== currentUserId; }; @@ -36,6 +40,10 @@ function extractCallData(message) { export function handleVoiceCallCreated(message, currentUserId) { if (!isVoiceCallMessage(message)) return; + // WhatsApp calls are managed by their own store (whatsappCalls), + // don't add them to the Twilio calls store. + if (isWhatsappCall(message)) return; + const { callSid, callDirection, conversationId, senderId } = extractCallData(message); @@ -56,14 +64,18 @@ export function handleVoiceCallUpdated(commit, message, currentUserId) { const { callSid, status, callDirection, conversationId, senderId } = extractCallData(message); - const callsStore = useCallsStore(); - - callsStore.handleCallStatusChanged({ callSid, status, conversationId }); - + // Vuex message/conversation status updates apply to all call sources const callInfo = { conversationId, callStatus: status }; commit(types.UPDATE_CONVERSATION_CALL_STATUS, callInfo); commit(types.UPDATE_MESSAGE_CALL_STATUS, callInfo); + // Twilio-specific store interactions — skip for WhatsApp calls + if (isWhatsappCall(message)) return; + + const callsStore = useCallsStore(); + + callsStore.handleCallStatusChanged({ callSid, status, conversationId }); + const isNewCall = status === 'ringing' && !shouldSkipCall(callDirection, senderId, currentUserId); diff --git a/app/javascript/dashboard/i18n/locale/en/conversation.json b/app/javascript/dashboard/i18n/locale/en/conversation.json index 835a7e512..7eb5f3b1b 100644 --- a/app/javascript/dashboard/i18n/locale/en/conversation.json +++ b/app/javascript/dashboard/i18n/locale/en/conversation.json @@ -82,7 +82,13 @@ "CALL_ENDED": "Call ended", "NOT_ANSWERED_YET": "Not answered yet", "THEY_ANSWERED": "They answered", - "YOU_ANSWERED": "You answered" + "YOU_ANSWERED": "You answered", + "ANSWERED_BY": "Answered by {name}", + "DURATION": "{duration}", + "ACCEPT_CALL": "Accept", + "JOIN_CALL": "Join", + "TRANSCRIPT": "Transcript", + "AUDIO_NOT_SUPPORTED": "Your browser does not support audio playback." }, "HEADER": { "RESOLVE_ACTION": "Resolve", diff --git a/config/routes.rb b/config/routes.rb index 1c7bb6232..cc6bd0dbe 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -297,11 +297,12 @@ Rails.application.routes.draw do resource :authorization, only: [:create] end - resources :whatsapp_calls, only: [] do + resources :whatsapp_calls, only: [:show] do member do post :accept post :reject post :terminate + post :upload_recording end collection do post :initiate diff --git a/db/migrate/20260323100000_add_message_id_to_whatsapp_calls.rb b/db/migrate/20260323100000_add_message_id_to_whatsapp_calls.rb new file mode 100644 index 000000000..3e88ff150 --- /dev/null +++ b/db/migrate/20260323100000_add_message_id_to_whatsapp_calls.rb @@ -0,0 +1,5 @@ +class AddMessageIdToWhatsappCalls < ActiveRecord::Migration[7.1] + def change + add_reference :whatsapp_calls, :message, null: true, foreign_key: true, index: true + end +end diff --git a/db/migrate/20260323110000_add_transcript_to_whatsapp_calls.rb b/db/migrate/20260323110000_add_transcript_to_whatsapp_calls.rb new file mode 100644 index 000000000..3e57546cf --- /dev/null +++ b/db/migrate/20260323110000_add_transcript_to_whatsapp_calls.rb @@ -0,0 +1,5 @@ +class AddTranscriptToWhatsappCalls < ActiveRecord::Migration[7.1] + def change + add_column :whatsapp_calls, :transcript, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 3814f678f..d00f0737d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_03_20_074636) do +ActiveRecord::Schema[7.1].define(version: 2026_03_23_110000) do # These extensions should be enabled to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" @@ -1289,9 +1289,12 @@ ActiveRecord::Schema[7.1].define(version: 2026_03_20_074636) do t.jsonb "meta", default: {}, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.bigint "message_id" + t.text "transcript" t.index ["account_id", "conversation_id"], name: "index_whatsapp_calls_on_account_id_and_conversation_id" t.index ["call_id"], name: "index_whatsapp_calls_on_call_id", unique: true t.index ["inbox_id", "status"], name: "index_whatsapp_calls_on_inbox_id_and_status" + t.index ["message_id"], name: "index_whatsapp_calls_on_message_id" end create_table "working_hours", force: :cascade do |t| @@ -1313,6 +1316,7 @@ ActiveRecord::Schema[7.1].define(version: 2026_03_20_074636) do add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" add_foreign_key "inboxes", "portals" + add_foreign_key "whatsapp_calls", "messages" create_trigger("accounts_after_insert_row_tr", :generated => true, :compatibility => 1). on("accounts"). after(:insert). diff --git a/docs/whatsapp-calling.md b/docs/whatsapp-calling.md new file mode 100644 index 000000000..2d501c454 --- /dev/null +++ b/docs/whatsapp-calling.md @@ -0,0 +1,1176 @@ +# WhatsApp Calling in Chatwoot + +WhatsApp Calling lets agents receive and make voice calls directly from the Chatwoot dashboard using the WhatsApp Cloud API. Calls are browser-based (WebRTC) — no phone hardware or Twilio required. + +> **Enterprise only.** Gated behind the `whatsapp_call` feature flag. + +--- + +## Table of Contents + +1. [How WhatsApp Calling Works (The Big Picture)](#1-how-whatsapp-calling-works-the-big-picture) +2. [Setup Guide](#2-setup-guide) +3. [Product Walkthrough](#3-product-walkthrough) +4. [Technical Architecture](#4-technical-architecture) +5. [Call Flows (Step by Step)](#5-call-flows-step-by-step) +6. [Call Recording & Transcription](#6-call-recording--transcription) +7. [Data Model](#7-data-model) +8. [API Reference](#8-api-reference) +9. [WebSocket Events](#9-websocket-events) +10. [Key Files](#10-key-files) +11. [Challenges & Design Decisions](#11-challenges--design-decisions) +12. [Extending the Feature](#12-extending-the-feature) + +--- + +## 1. How WhatsApp Calling Works (The Big Picture) + +Before diving into code, here's the mental model for the entire feature. + +### Three players, two communication channels + +``` + ┌─────────────────────────────────────────────────────────────────┐ + │ │ + │ 1. SIGNALING (who's calling whom, accept/reject, hang up) │ + │ Travels through: Meta API <-> Chatwoot Backend <-> Browser │ + │ │ + │ 2. AUDIO (the actual voice data) │ + │ Travels through: Meta Servers <-----------> Browser │ + │ (Chatwoot backend is NOT in this path) │ + │ │ + └─────────────────────────────────────────────────────────────────┘ +``` + +Think of it like a phone operator (signaling) connecting two people (audio). The operator sets up the call but doesn't listen in — the audio flows directly between the two parties. + +``` +┌──────────────┐ ┌──────────────────┐ +│ WhatsApp │ Signaling (REST/Webhooks) │ Chatwoot │ +│ Contact │◄─────────────────────────────────────►│ Backend │ +│ │ │ (Rails) │ +└──────┬───────┘ └────────┬─────────┘ + │ │ + │ WebSocket │ (ActionCable) + │ │ + │ ┌────────────────────────┐ ┌────────▼─────────┐ + │ │ Meta Media Servers │ │ Agent's │ + │ │ │ │ Browser │ + └────────►│ (routes the audio │◄══════════►│ (Vue + WebRTC) │ + │ between endpoints) │ Audio │ │ + └────────────────────────┘ (SRTP) └──────────────────┘ +``` + +**Why does Chatwoot's backend not touch the audio?** + +The audio is a peer-to-peer WebRTC connection between the agent's browser and Meta's media servers. This is intentional — it reduces latency, avoids the need for media proxies, and is how Meta designed their calling API. The browser has direct access to both audio tracks (local mic + remote caller), which is what makes client-side call recording possible. + +### What is WebRTC? + +WebRTC (Web Real-Time Communication) is a browser API that enables audio/video communication without plugins. The key concepts: + +| Concept | What it means | +|---|---| +| **RTCPeerConnection** | The browser object that manages the audio connection | +| **SDP (Session Description Protocol)** | A text blob describing what media you can send/receive, your IP address, ports, codecs, etc. Think of it as a "business card" for the call | +| **SDP Offer** | "Here's what I can do" — sent by the caller | +| **SDP Answer** | "Here's what I can do too" — sent by the receiver | +| **ICE (Interactive Connectivity Establishment)** | The process of discovering how two machines can reach each other through NATs/firewalls | +| **STUN server** | Helps you discover your public IP address | +| **SRTP** | Encrypted audio packets that flow once the connection is established | + +### The SDP handshake (simplified) + +``` + Caller (Meta) Receiver (Agent Browser) + │ │ + │ "I want to call you. Here's my SDP offer │ + │ (my IP, ports, codecs I support)" │ + │───────────────────────────────────────────────────────►│ + │ │ + │ "Got it. Here's my SDP answer │ + │ (my IP, ports, codecs I support)" │ + │◄───────────────────────────────────────────────────────│ + │ │ + │ │ + │◄═══════════════ Audio flows both ways ════════════════►│ +``` + +--- + +## 2. Setup Guide + +### Prerequisites + +| Requirement | Details | +|---|---| +| Chatwoot Enterprise | Feature flag `whatsapp_call` enabled for the account | +| WhatsApp Cloud API inbox | Provider must be `whatsapp_cloud` | +| Meta Business account | With a verified WhatsApp Business phone number | +| `calls` webhook field | Must be subscribed on the WABA (not included by default) | +| Browser microphone | Agents must grant mic permission when prompted | + +### Step-by-step + +#### 1. Enable the feature flag + +```ruby +account = Account.find() +account.enable_features('whatsapp_call') +account.save! +``` + +#### 2. Enable calling on the inbox + +```ruby +inbox = Inbox.find() +inbox.channel.provider_config['calling_enabled'] = true +inbox.channel.save! +``` + +#### 3. Subscribe to the `calls` webhook field + +By default, Chatwoot subscribes to `messages` and `smb_message_echoes`. The `calls` field must be added. + +**Via Graph API:** +```bash +curl -X POST \ + "https://graph.facebook.com/v22.0//subscribed_apps" \ + -H "Authorization: Bearer " \ + -d "subscribed_fields=messages,smb_message_echoes,calls" +``` + +**Via Meta App Dashboard:** WhatsApp > Configuration > Webhooks > enable `calls` field. + +#### 4. Verify the webhook endpoint + +``` +https:///webhooks/whatsapp/ +``` + +#### 5. For call transcription (optional) + +Requires `captain_integration` feature flag + OpenAI API key: + +```ruby +account.enable_features('captain_integration') +account.save! + +# Ensure this is configured: +InstallationConfig.find_or_create_by(name: 'CAPTAIN_OPEN_AI_API_KEY') do |c| + c.value = 'sk-...' +end +``` + +--- + +## 3. Product Walkthrough + +### Receiving a call (Inbound) + +``` + ┌──────────────────────────────────────────────────────────────────────┐ + │ What the agent sees │ + │ │ + │ 1. Floating widget appears (bottom-right) │ + │ ┌─────────────────────────────────────┐ │ + │ │ 📞 John Doe │ │ + │ │ Incoming WhatsApp Call │ │ + │ │ [Reject] [Accept] │ │ + │ └─────────────────────────────────────┘ │ + │ │ + │ 2. Message bubble in conversation │ + │ ┌─────────────────────────┐ │ + │ │ 📞 Incoming call │ │ + │ │ Not answered yet │ │ + │ │ [Accept] │ │ + │ └─────────────────────────┘ │ + │ │ + │ 3. After accepting │ + │ ┌─────────────────────────────────────┐ │ + │ │ 📞 John Doe [🔇] [📕] │ │ + │ │ 02:34 │ │ + │ └─────────────────────────────────────┘ │ + │ │ + │ 4. After call ends │ + │ ┌─────────────────────────┐ │ + │ │ 📞 Call ended │ │ + │ │ Answered by John │ │ + │ │ 2m 34s │ │ + │ │ ▶ ───●─────── 2:34 │ ← audio recording │ + │ │ ▸ Transcript │ ← expandable transcript │ + │ └─────────────────────────┘ │ + └──────────────────────────────────────────────────────────────────────┘ +``` + +### Making a call (Outbound) + +1. Agent clicks the phone icon in the conversation header. +2. Browser requests microphone permission. +3. A "Calling..." toast appears; the widget shows "Ringing...". +4. When the contact answers, audio flows and the timer starts. +5. Agent hangs up from the widget. + +### Call permission flow + +Meta requires contacts to explicitly opt-in to receive calls from a business. If the contact hasn't granted permission: + +1. Agent clicks call → Chatwoot sends a permission request (interactive message in WhatsApp chat). +2. Agent sees: "Permission requested — the contact will receive a prompt." +3. Contact approves in WhatsApp → agent gets notified and can retry the call. +4. Rate-limited to one request per 5 minutes per conversation. + +### Limitations + +- Audio-only — no video support. +- One active call per agent at a time. +- Requires the WhatsApp Cloud API provider (not on-premise API / 360dialog). +- **Ringing calls survive refresh** — the agent can accept from the message bubble after reloading. The SDP offer is stored on the server. +- **Active calls do NOT survive refresh** — WhatsApp calls are peer-to-peer WebRTC. If the agent refreshes or closes the tab during an active call, the call is automatically terminated and the recording is lost. (See [Page refresh / close](#page-refresh--close--what-happens-to-the-call) for full details.) +- Unanswered inbound calls auto-dismiss from the widget after 30 seconds. +- **Recording depends on clean call end** — if the browser crashes or the page is closed during a call, the in-memory recording is lost. Only calls that end normally (hang up button or caller disconnect) produce recordings. + +--- + +## 4. Technical Architecture + +### Technology stack + +| Layer | Technology | Role | +|---|---|---| +| **Signaling** | Meta WhatsApp Cloud API (`/{phone_id}/calls`) | Call setup, accept, reject, terminate | +| **Real-time events** | ActionCable (WebSocket) | Push call events to all agents in the account | +| **Media transport** | WebRTC (`RTCPeerConnection`) | Browser ↔ Meta peer-to-peer audio | +| **Call UI** | Vue 3 floating widget + VoiceCall.vue bubble | Agent-facing call controls | +| **State** | Pinia (`whatsappCalls.js`) + Vuex (messages) | Frontend call + message state | +| **Backend** | Rails controllers + services (enterprise) | API endpoints, webhook processing | +| **Background jobs** | Sidekiq | Webhook processing, transcription | +| **Recording** | Browser `MediaRecorder` API | Client-side call recording | +| **Transcription** | OpenAI Whisper (`whisper-1`) | Post-call speech-to-text | + +### How WhatsApp calls fit into the existing Voice/Twilio pattern + +WhatsApp calls use the **same `voice_call` content_type** as Twilio calls. This means: + +``` + ┌────────────────────────────┐ + │ voice_call message │ + │ content_type: voice_call │ + └──────────────┬─────────────┘ + │ + ┌──────────────┴─────────────┐ + │ │ + call_source: whatsapp call_source: (absent) + │ │ + ┌───────▼────────┐ ┌───────▼────────┐ + │ WhatsApp Calls │ │ Twilio Calls │ + │ Store (Pinia) │ │ Store (Pinia) │ + │ Peer-to-peer │ │ Conference │ + └────────────────┘ └────────────────┘ +``` + +The `voice.js` helper inspects `call_source` and routes events to the correct store. The `VoiceCall.vue` bubble renders both — with minor behavior differences (e.g., WhatsApp calls only show "Accept" while ringing, Twilio also shows "Join" for in-progress calls). + +### Status mapping + +WhatsApp call statuses are mapped to Voice statuses for UI compatibility: + +``` + WhatsApp Backend Message (content_attributes) UI Display + ───────────────── ──────────────────────────── ────────────────── + ringing ──► ringing ──► "Incoming call" + accepted ──► in-progress ──► "Call in progress" + rejected ──► failed ──► "Missed call" + missed ──► no-answer ──► "Missed call" + ended ──► completed ──► "Call ended" + failed ──► failed ──► "Missed call" +``` + +### Enterprise architecture + +All calling logic lives under `enterprise/`: + +``` +enterprise/ +├── app/ +│ ├── controllers/api/v1/accounts/ +│ │ └── whatsapp_calls_controller.rb ← REST API +│ ├── models/ +│ │ └── whatsapp_call.rb ← Data model + ActiveStorage recording +│ ├── services/whatsapp/ +│ │ ├── call_message_builder.rb ← Creates/updates voice_call messages +│ │ ├── call_service.rb ← Accept/reject/terminate with Meta +│ │ ├── call_transcription_service.rb ← Whisper transcription +│ │ ├── incoming_call_service.rb ← Webhook → call record + message +│ │ ├── call_permission_reply_service.rb +│ │ └── providers/ +│ │ └── whatsapp_cloud_call_methods.rb ← Meta Graph API calls +│ └── jobs/ +│ ├── enterprise/webhooks/ +│ │ └── whatsapp_events_job.rb ← Routes call webhooks +│ └── whatsapp/ +│ └── call_transcription_job.rb ← Async transcription +``` + +The OSS `Webhooks::WhatsappEventsJob` is extended via `prepend_mod_with` to detect and route call events to the enterprise services. + +--- + +## 5. Call Flows (Step by Step) + +### Inbound call + +``` + WhatsApp Chatwoot Agent + Contact Meta Cloud Backend Browser + │ │ │ │ + │ │ │ │ + 1. │── Dials ────────►│ │ │ + │ business # │ │ │ + │ │ │ │ + 2. │ │── Webhook ──────────►│ │ + │ │ event: connect │ │ + │ │ payload: { │ │ + │ │ id, from, │ │ + │ │ sdp_offer │ │ + │ │ } │ │ + │ │ │ │ + 3. │ │ ┌─────┴──────────┐ │ + │ │ │ IncomingCall │ │ + │ │ │ Service: │ │ + │ │ │ │ │ + │ │ │ a. Find/create │ │ + │ │ │ Contact │ │ + │ │ │ b. Find/create │ │ + │ │ │ Conversation │ │ + │ │ │ c. Create │ │ + │ │ │ WhatsappCall │ │ + │ │ │ (ringing) │ │ + │ │ │ d. Create │ │ + │ │ │ voice_call │ │ + │ │ │ message │ │ + │ │ └─────┬──────────┘ │ + │ │ │ │ + 4. │ │ │── ActionCable ──────────►│ + │ │ │ whatsapp_call.incoming │ + │ │ │ {sdp_offer, caller, │ + │ │ │ ice_servers} │ + │ │ │ │ + 5. │ │ │ ┌──────┴───────┐ + │ │ │ │ Show widget │ + │ │ │ │ Show bubble │ + │ │ │ │ with [Accept] │ + │ │ │ └──────┬───────┘ + │ │ │ │ + │ │ │ Agent clicks │ + │ │ │ "Accept" │ + │ │ │ │ + 6. │ │ │ ┌──────┴───────┐ + │ │ │ │ getUserMedia │ + │ │ │ │ (mic access) │ + │ │ │ │ │ + │ │ │ │ new RTC │ + │ │ │ │ PeerConnection│ + │ │ │ │ │ + │ │ │ │ setRemoteDesc │ + │ │ │ │ (Meta's SDP │ + │ │ │ │ offer) │ + │ │ │ │ │ + │ │ │ │ createAnswer │ + │ │ │ │ Wait for ICE │ + │ │ │ │ gathering │ + │ │ │ └──────┬───────┘ + │ │ │ │ + 7. │ │ │◄── POST /accept ────────│ + │ │ │ {sdp_answer} │ + │ │ │ │ + 8. │ │ ┌─────┴──────────┐ │ + │ │ │ CallService │ │ + │ │ │ (with row lock) │ │ + │ │ │ │ │ + │ │ │ a. Validate │ │ + │ │ │ still ringing│ │ + │ │ │ b. Fix SDP │ │ + │ │ │ (actpass → │ │ + │ │ │ active) │ │ + │ │ └─────┬──────────┘ │ + │ │ │ │ + 9. │ │◄── pre_accept ──────│ │ + │ │◄── accept ──────────│ │ + │ │ │ │ +10. │ │ ┌─────┴──────────┐ │ + │ │ │ Update call → │ │ + │ │ │ accepted │ │ + │ │ │ Update msg → │ │ + │ │ │ in-progress │ │ + │ │ │ + answered_by │ │ + │ │ └─────┬──────────┘ │ + │ │ │ │ +11. │ │ │── ActionCable ──────────►│ + │ │ │ whatsapp_call.accepted │ + │ │ │ │ + │ │ │ Other agents: │ + │ │ │ call removed │ + │ │ │ from their widget │ + │ │ │ │ +12. │◄════════════════ WebRTC Audio (SRTP, peer-to-peer) ════════════►│ + │ │ │ │ + │ │ │ ┌──────┴───────┐ + │ │ │ │ MediaRecorder │ + │ │ │ │ starts │ + │ │ │ │ recording │ + │ │ │ │ both tracks │ + │ │ │ └──────┬───────┘ + │ │ │ │ +13. │── Hangs up ─────►│ │ │ + │ │── Webhook ──────────►│ │ + │ │ event: terminate │ │ + │ │ {duration, reason} │ │ + │ │ │ │ +14. │ │ ┌─────┴──────────┐ │ + │ │ │ Update call → │ │ + │ │ │ ended │ │ + │ │ │ Update msg → │ │ + │ │ │ completed │ │ + │ │ │ + duration │ │ + │ │ └─────┬──────────┘ │ + │ │ │ │ +15. │ │ │── ActionCable ──────────►│ + │ │ │ whatsapp_call.ended │ + │ │ │ ┌──────┴───────┐ + │ │ │ │ Stop recorder │ + │ │ │ │ Upload .webm │ + │ │ │ │ Cleanup WebRTC│ + │ │ │ └──────┬───────┘ + │ │ │ │ +16. │ │ │◄── POST /upload ────────│ + │ │ │ _recording │ + │ │ ┌─────┴──────────┐ │ + │ │ │ Attach to │ │ + │ │ │ ActiveStorage │ │ + │ │ │ │ │ + │ │ │ Enqueue │ │ + │ │ │ transcription │ │ + │ │ │ job │ │ + │ │ └─────┬──────────┘ │ + │ │ │ │ +17. │ │ ┌─────┴──────────┐ │ + │ │ │ Whisper API │ │ + │ │ │ transcribes │ │ + │ │ │ │ │ + │ │ │ Update msg with │ │ + │ │ │ recording_url + │ │ + │ │ │ transcript │ │ + │ │ └────────────────┘ │ +``` + +### Outbound call + +``` + Agent Chatwoot Meta WhatsApp + Browser Backend Cloud Contact + │ │ │ │ + 1. │── Click phone icon │ │ │ + │ │ │ │ + 2. │ getUserMedia (mic) │ │ │ + │ new RTCPeerConnection │ │ │ + │ createOffer + ICE │ │ │ + │ │ │ │ + 3. │── POST /initiate ─────►│ │ │ + │ {conversation_id, │ │ │ + │ sdp_offer} │ │ │ + │ │── initiate_call ──────►│ │ + │ │ {to, sdp_offer} │── Rings phone ──────►│ + │ │ │ │ + │ │◄── {call_id} ──────────│ │ + │ │ │ │ + 4. │ ┌─────┴──────────┐ │ │ + │ │ Create │ │ │ + │ │ WhatsappCall │ │ │ + │ │ (outbound, │ │ │ + │ │ ringing) │ │ │ + │ │ Create │ │ │ + │ │ voice_call msg │ │ │ + │ └─────┬──────────┘ │ │ + │ │ │ │ + 5. │◄── {call_id, id} ─────│ │ │ + │ │ │ │ + │ Widget: "Ringing..." │ │ │ + │ │ │ │ + │ │ │ Contact answers │ + │ │ │◄─────────────────────│ + │ │ │ │ + 6. │ │◄── Webhook ────────────│ │ + │ │ event: connect │ │ + │ │ {call_id, sdp_answer} │ │ + │ │ │ │ + 7. │ ┌─────┴──────────┐ │ │ + │ │ Find existing │ │ │ + │ │ call by call_id │ │ │ + │ │ Update → │ │ │ + │ │ accepted │ │ │ + │ └─────┬──────────┘ │ │ + │ │ │ │ + 8. │◄── ActionCable ────────│ │ │ + │ whatsapp_call. │ │ │ + │ outbound_connected │ │ │ + │ {sdp_answer} │ │ │ + │ │ │ │ + 9. │ setRemoteDescription │ │ │ + │ (sdp_answer) │ │ │ + │ ontrack → play audio │ │ │ + │ Start recording │ │ │ + │ Timer starts │ │ │ + │ │ │ │ +10. │◄══════════════ WebRTC Audio (SRTP) ════════════════════════════════════►│ + │ │ │ │ + │ (same terminate flow as inbound — steps 13-17 above) │ +``` + +### Permission flow + +``` + Agent clicks call + │ + ▼ + POST /initiate ─────► Meta returns error 138006 (no permission) + │ + ▼ + Backend sends call_permission_request template to contact via Meta + │ + ▼ + Returns {status: "permission_requested"} ──► Agent sees toast + │ + ┈┈┈┈┈┈┈┈┈┈ (contact sees permission prompt in WhatsApp) ┈┈┈┈┈┈┈┈┈┈ + │ + Contact approves ──► Meta webhook (interactive/call_permission_reply) + │ + ▼ + ActionCable: whatsapp_call.permission_granted ──► Agent sees toast + │ + ▼ + Agent can now call (retry) +``` + +### Page refresh / close — what happens to the call? + +This is one of the trickiest parts of the feature. The behavior is **different depending on the call state**: + +#### Scenario A: Call is RINGING (not yet accepted) + +Page refresh does **not** kill the call. The agent can still accept it after the page reloads. + +``` + ┌──────────────────────────────────────────────────────────────────────┐ + │ RINGING CALL + PAGE REFRESH │ + │ │ + │ 1. Call comes in → widget + bubble appear │ + │ 2. Agent refreshes the page │ + │ - In-memory state (Pinia store, SDP offer) is LOST │ + │ - The WhatsappCall record on the server is still "ringing" │ + │ - The RTCPeerConnection never existed yet (no WebRTC to break) │ + │ 3. Page reloads → conversation loads → voice_call message renders │ + │ - VoiceCall.vue sees status = "ringing" → shows [Accept] button │ + │ 4. Agent clicks [Accept] │ + │ - acceptWhatsappCallById() runs │ + │ - Call NOT in Pinia store → falls back to API fetch: │ + │ GET /whatsapp_calls/:id → returns sdp_offer + ice_servers │ + │ - Creates new RTCPeerConnection with the fetched SDP │ + │ - Normal accept flow continues │ + │ │ + │ ✅ Call survives the refresh │ + └──────────────────────────────────────────────────────────────────────┘ +``` + +``` + Agent Browser Backend Meta + │ │ │ + │── Page refresh ──► │ │ + │ (all JS state lost) │ WhatsappCall still "ringing" │ + │ │ │ + │── Page loads ──► │ │ + │ Conversation renders │ │ + │ VoiceCall bubble: [Accept] │ │ + │ │ │ + │── Click Accept │ │ + │ │ │ + │── GET /whatsapp_calls/:id ──────►│ │ + │◄── {sdp_offer, ice_servers} ─────│ │ + │ │ │ + │── getUserMedia + WebRTC setup │ │ + │── POST /accept {sdp_answer} ────►│── pre_accept + accept ─────►│ + │ │ │ + │◄══════════════ Audio connected ═══════════════════════════════►│ +``` + +**Why this works:** The SDP offer is stored in the `WhatsappCall.meta` column on the server. The `show` endpoint returns it for ringing calls. The browser creates a fresh `RTCPeerConnection` with the server-stored SDP. + +**Why this only works for ringing calls:** Once a call is accepted, the SDP handshake is complete and audio is flowing through a specific `RTCPeerConnection` instance. That instance lives in browser memory — it cannot be reconstructed. This is a fundamental WebRTC limitation (peer-to-peer, no server-side media relay). + +#### Scenario B: Call is IN PROGRESS (already accepted) — Page close or refresh + +Page close/refresh **terminates** the call. This is intentional and unavoidable. + +``` + ┌──────────────────────────────────────────────────────────────────────┐ + │ ACTIVE CALL + PAGE CLOSE/REFRESH │ + │ │ + │ 1. Agent is on an active call (audio flowing) │ + │ 2. Agent closes tab / refreshes / navigates away │ + │ │ + │ browser fires "beforeunload" event │ + │ │ │ + │ ├──► fetch("/terminate", {keepalive: true}) │ + │ │ - Uses fetch API (not axios) because axios │ + │ │ requests are cancelled on page unload │ + │ │ - keepalive: true tells the browser to │ + │ │ complete the request even after the page dies │ + │ │ - Auth headers read from session cookie │ + │ │ - Fire-and-forget (no await, .catch(() => {})) │ + │ │ │ + │ └──► cleanupInboundWebRTC() │ + │ - Stops all mic tracks │ + │ - Closes RTCPeerConnection │ + │ - Removes element from DOM │ + │ │ + │ 3. Backend receives terminate request │ + │ - Calls Meta API: terminate_call(call_id) │ + │ - Updates WhatsappCall → "ended" │ + │ - Updates message → "completed" + duration │ + │ - Broadcasts whatsapp_call.ended via ActionCable │ + │ │ + │ 4. Recording is LOST (MediaRecorder chunks are in memory) │ + │ │ + │ ❌ Call cannot survive — WebRTC is peer-to-peer │ + └──────────────────────────────────────────────────────────────────────┘ +``` + +``` + Agent Browser Backend Meta + │ │ │ + │ ═══ Active call (audio) ════════════════════════════════════► │ + │ │ │ + │── beforeunload fires │ │ + │ │ │ + │── fetch("/terminate", │ │ + │ {keepalive: true}) ──────────►│ │ + │ │── terminate_call ───────────►│ + │── cleanupInboundWebRTC() │ │ + │ (mic off, PC closed) │── Update call → ended │ + │ │── Update msg → completed │ + │ │── ActionCable: ended │ + │── Page dies │ │ +``` + +**Why we read auth headers manually:** +- The app uses `devise_token_auth` with headers (`access-token`, `client`, `uid`, `expiry`) +- These are stored in a `cw_d_session_info` cookie as JSON +- We parse the cookie directly instead of using axios interceptors (which won't run during unload) + +**What about the recording?** +- The `MediaRecorder` chunks live in a JavaScript array (`recordedChunks[]`) +- On page unload, this memory is freed — the recording is **lost** +- Only recordings from calls that end normally (hang up button) are saved +- A future improvement could periodically upload partial chunks during the call + +#### Summary table + +| Call state | Page refresh | Page close | +|---|---|---| +| **Ringing** (not accepted) | Call survives. Agent can accept from message bubble after reload. | Call stays ringing on Meta's side. Auto-dismissed after 30s widget timeout. | +| **In progress** (accepted) | Call terminated via `beforeunload`. Recording lost. | Call terminated via `beforeunload`. Recording lost. | +| **Ended** | No effect. | No effect. | + +--- + +## 6. Call Recording & Transcription + +### Why client-side recording? + +Meta does **not** provide call recordings, transcriptions, or audio stream access through their API. From their FAQ: + +> *"Does Meta offer services such as voice recording, transcript, and voicemail features? No."* + +However, Meta **does** provide the raw audio stream via WebRTC. Since the `RTCPeerConnection` lives in the agent's browser, the browser has direct access to both audio tracks: + +- **Local track** — the agent's microphone (`getUserMedia`) +- **Remote track** — the caller's audio (delivered via `pc.ontrack`) + +This is what makes client-side recording possible without any Meta API support. + +### How recording works + +``` + ┌─────────────────────────────────────────────────────────────┐ + │ Agent's Browser │ + │ │ + │ Local mic ──────┐ │ + │ (getUserMedia) │ │ + │ ▼ │ + │ ┌───────────────┐ │ + │ │ AudioContext │ │ + │ │ │ │ + │ │ localSource ─┤ │ + │ │ ├──► MediaStreamDestination │ + │ │ remoteSource ┤ (mixed audio) │ + │ │ │ │ │ + │ └───────────────┘ │ │ + │ ▼ │ + │ Remote audio ──┘ ┌──────────────┐ │ + │ (pc.ontrack) │ MediaRecorder │ │ + │ │ audio/webm │ │ + │ │ opus codec │ │ + │ │ │ │ + │ │ Collects 1s │ │ + │ │ chunks into │ │ + │ │ array │ │ + │ └──────┬───────┘ │ + │ │ │ + └───────────────────────────────────────┼─────────────────────┘ + │ + On call end + │ + ▼ + ┌──────────────┐ + │ Blob (webm) │ + │ │ + │ POST /upload │ + │ _recording │ + │ (multipart) │ + └──────┬───────┘ + │ + ▼ + ┌───────────────┐ + │ Backend │ + │ │ + │ ActiveStorage │ + │ .attach() │ + │ │ + │ Update message │ + │ with recording │ + │ URL │ + │ │ + │ Enqueue │ + │ transcription │ + │ job │ + └───────┬───────┘ + │ + ▼ + ┌───────────────┐ + │ Transcription │ + │ Job (Sidekiq) │ + │ │ + │ Download from │ + │ ActiveStorage │ + │ │ │ + │ ▼ │ + │ OpenAI Whisper │ + │ (whisper-1) │ + │ │ │ + │ ▼ │ + │ Save transcript│ + │ to call record │ + │ + message │ + └───────────────┘ +``` + +### Recording step by step + +Here's exactly what happens in code, from call connect to transcript: + +**Step 1: Recording starts (on `pc.ontrack` — when remote audio arrives)** + +``` +File: useWhatsappCallSession.js → startCallRecording(pc, localStream, callId) + + a. Create an AudioContext (Web Audio API) + b. Create a MediaStreamDestination (the "mixing board") + c. Connect local mic track → destination + d. Connect remote caller track(s) → destination + (reads from pc.getReceivers() — all audio tracks from the peer connection) + e. Create MediaRecorder on the mixed destination stream + - Format: audio/webm;codecs=opus + - Chunk interval: 1 second + f. recorder.ondataavailable → push each chunk to recordedChunks[] + g. recorder.start(1000) — begin recording +``` + +For **inbound calls**, this happens inside `doAcceptCall()` → `pc.ontrack`. +For **outbound calls**, this happens inside `ConversationHeader.vue` → `pc.ontrack` (when the contact picks up). + +**Step 2: Call is active — recording accumulates in memory** + +``` + During the call: + - Every 1 second, MediaRecorder fires ondataavailable + - Each chunk (~5-15 KB) is pushed to the recordedChunks[] array + - A 5-minute call ≈ 300 chunks ≈ 1.5-4.5 MB in memory + - The recording is NOT yet on the server — it only exists in browser memory +``` + +**Step 3: Call ends — recording is uploaded** + +``` + Triggered by: + - Agent clicks "Hang up" → endActiveCall() → stopAndUploadRecording() + - External end (caller hangs up) → ActionCable whatsapp_call.ended + → store.handleCallEnded() → cleanupCallback → stopAndUploadRecording() + + stopAndUploadRecording(callId): + a. mediaRecorder.stop() + b. In recorder.onstop callback: + - new Blob(recordedChunks, {type: 'audio/webm'}) — merge all chunks + - WhatsappCallsAPI.uploadRecording(callId, blob) — POST multipart + - recordedChunks = [] — free memory +``` + +**Step 4: Backend processes the upload** + +``` + POST /api/v1/accounts/:id/whatsapp_calls/:id/upload_recording + a. Validate call is terminal (ended/missed/failed) + b. ActiveStorage.attach(recording file) + c. Update message content_attributes with recording_url + d. Enqueue Whatsapp::CallTranscriptionJob +``` + +**Step 5: Transcription job runs (async, Sidekiq low queue)** + +``` + Whatsapp::CallTranscriptionJob: + a. Download recording from ActiveStorage to tmp file + b. Send to OpenAI Whisper API (whisper-1 model) + c. Save transcript to WhatsappCall.transcript column + d. Update message content_attributes with transcript text + e. Clean up tmp file +``` + +### When recording is saved vs lost + +| Scenario | Recording saved? | Why | +|---|---|---| +| Agent clicks "Hang up" | Yes | `stopAndUploadRecording()` runs before cleanup | +| Caller hangs up (terminate webhook) | Yes | ActionCable `whatsapp_call.ended` triggers cleanup callback which uploads | +| Agent refreshes during active call | **No** | `beforeunload` terminates the call but `MediaRecorder` chunks in memory are freed before upload completes | +| Agent closes tab during active call | **No** | Same as refresh — memory freed on page death | +| Browser crashes | **No** | No cleanup handlers run at all | +| Call < 1 second | **No** | No chunks collected yet (1s interval) | + +> **Future improvement:** Periodically upload partial recording chunks during the call (e.g., every 30 seconds). This would ensure most of the recording survives even if the page dies. + +### What the agent sees (progressive updates) + +``` + ┌──────────────────────────────────────────────────────────────────┐ + │ │ + │ Immediately on call end: │ + │ ┌──────────────────────────┐ │ + │ │ 📞 Call ended │ │ + │ │ Answered by John │ │ + │ │ 2m 34s │ │ + │ └──────────────────────────┘ │ + │ │ + │ ~2-5 seconds later (recording upload completes): │ + │ ┌──────────────────────────┐ │ + │ │ 📞 Call ended │ │ + │ │ Answered by John │ │ + │ │ 2m 34s │ │ + │ │ ▶ ────●──────── 2:34 │ ← native player │ + │ └──────────────────────────┘ │ + │ │ + │ ~10-30 seconds later (transcription job completes): │ + │ ┌──────────────────────────┐ │ + │ │ 📞 Call ended │ │ + │ │ Answered by John │ │ + │ │ 2m 34s │ │ + │ │ ▶ ────●──────── 2:34 │ │ + │ │ ▸ Transcript │ ← click to expand │ + │ │ "Hi, I wanted to ask │ │ + │ │ about my order..." │ │ + │ └──────────────────────────┘ │ + │ │ + └──────────────────────────────────────────────────────────────────┘ +``` + +These updates happen via `message.updated` ActionCable events — the backend updates `content_attributes` and the VoiceCall bubble re-renders reactively. + +### Prerequisites for transcription + +| Requirement | How to check | +|---|---| +| `captain_integration` feature flag | `account.feature_enabled?('captain_integration')` | +| OpenAI API key configured | `InstallationConfig.find_by(name: 'CAPTAIN_OPEN_AI_API_KEY')` | +| Available usage quota | `account.usage_limits[:captain][:responses][:current_available] > 0` | + +If any prerequisite is missing, the recording is still saved — only transcription is skipped. No errors are raised; the job returns early with `{ error: 'Transcription not available' }`. + +--- + +## 7. Data Model + +### `whatsapp_calls` table + +| Column | Type | Description | +|---|---|---| +| `id` | bigint | Primary key | +| `account_id` | bigint | FK → accounts | +| `inbox_id` | bigint | FK → inboxes | +| `conversation_id` | bigint | FK → conversations | +| `accepted_by_agent_id` | bigint | FK → users (nullable) | +| `message_id` | bigint | FK → messages (nullable) — the voice_call message | +| `call_id` | string | Unique Meta call ID | +| `direction` | string | `inbound` or `outbound` | +| `status` | string | `ringing`, `accepted`, `rejected`, `missed`, `ended`, `failed` | +| `duration_seconds` | integer | Call duration (set on termination) | +| `end_reason` | string | Reason from Meta (e.g., `caller_hangup`) | +| `meta` | jsonb | Stores `sdp_offer`, `sdp_answer`, `ice_servers` | +| `transcript` | text | Whisper transcription of the call | +| `created_at` | datetime | | +| `updated_at` | datetime | | + +**ActiveStorage attachment:** `has_one_attached :recording` (audio/webm) + +**Indexes:** `call_id` (unique), `[account_id, conversation_id]`, `[inbox_id, status]`, `message_id` + +### Voice call message structure + +```json +{ + "content": "WhatsApp Call", + "content_type": "voice_call", + "message_type": 0, + "content_attributes": { + "data": { + "call_sid": "meta_call_id_abc123", + "status": "completed", + "call_direction": "inbound", + "call_source": "whatsapp", + "wa_call_id": 42, + "from_number": "+1234567890", + "to_number": "+0987654321", + "accepted_by": { "id": 7, "name": "Agent Smith" }, + "duration_seconds": 135, + "recording_url": "/rails/active_storage/blobs/.../call-42.webm", + "transcript": "Hi, I wanted to ask about my order...", + "meta": { "created_at": 1711180800 } + } + } +} +``` + +### Status lifecycle + +``` + ┌──────────┐ + │ ringing │ + └──┬───┬───┘ + │ │ + accepted │ │ no answer / timeout / reject + │ │ + ┌────▼┐ │ ┌──────────┐ + │accepted│ ├──►│ rejected │ + └────┬───┘ │ └──────────┘ + │ │ ┌──────────┐ + ended │ ├──►│ missed │ + │ │ └──────────┘ + ┌────▼──┐ │ ┌──────────┐ + │ ended │ └──►│ failed │ + └───────┘ └──────────┘ +``` + +--- + +## 8. API Reference + +Base: `POST /api/v1/accounts/{account_id}/whatsapp_calls` + +| Endpoint | Method | Purpose | Key params | +|---|---|---|---| +| `/{id}` | GET | Show call details | — | +| `/initiate` | POST | Start outbound call | `conversation_id`, `sdp_offer` | +| `/{id}/accept` | POST | Accept ringing call | `sdp_answer` | +| `/{id}/reject` | POST | Reject ringing call | — | +| `/{id}/terminate` | POST | End active call | — | +| `/{id}/upload_recording` | POST | Upload call recording | `recording` (multipart file) | + +### Notable responses + +**Initiate — permission flow:** +- `200 { status: 'calling', call_id, id }` — call initiated +- `200 { status: 'permission_requested' }` — contact needs to grant permission +- `200 { status: 'permission_pending' }` — already requested recently + +**Accept — race condition:** +- `422 { error: 'Call already accepted by another agent' }` — another agent won +- `422 { error: 'Call is not in ringing state' }` — call ended/timed out + +--- + +## 9. WebSocket Events + +All events are broadcast on the `account_{account_id}` ActionCable channel. + +| Event | When | Key payload | +|---|---|---| +| `whatsapp_call.incoming` | New inbound call | `id`, `call_id`, `sdp_offer`, `ice_servers`, `caller` | +| `whatsapp_call.accepted` | Agent accepted | `call_id`, `accepted_by_agent_id` | +| `whatsapp_call.outbound_connected` | Contact answered outbound call | `call_id`, `sdp_answer` | +| `whatsapp_call.ended` | Call ended (any reason) | `call_id`, `status`, `duration_seconds` | +| `whatsapp_call.permission_granted` | Contact granted call permission | `contact_name` | + +Standard `message.created` and `message.updated` events also fire when voice_call messages change. + +--- + +## 10. Key Files + +### Backend (Enterprise) + +| File | Purpose | +|---|---| +| `enterprise/app/models/whatsapp_call.rb` | Data model + `has_one_attached :recording` | +| `enterprise/app/controllers/api/v1/accounts/whatsapp_calls_controller.rb` | REST API | +| `enterprise/app/services/whatsapp/call_service.rb` | Accept/reject/terminate with Meta | +| `enterprise/app/services/whatsapp/incoming_call_service.rb` | Webhook → call record + message | +| `enterprise/app/services/whatsapp/call_message_builder.rb` | Creates/updates voice_call messages | +| `enterprise/app/services/whatsapp/call_transcription_service.rb` | Whisper transcription | +| `enterprise/app/jobs/whatsapp/call_transcription_job.rb` | Async transcription job | +| `enterprise/app/services/whatsapp/providers/whatsapp_cloud_call_methods.rb` | Meta API calls | +| `enterprise/app/jobs/enterprise/webhooks/whatsapp_events_job.rb` | Routes call webhooks | + +### Frontend + +| File | Purpose | +|---|---| +| `dashboard/components-next/message/bubbles/VoiceCall.vue` | Call message bubble (recording + transcript) | +| `dashboard/stores/whatsappCalls.js` | Pinia store for call state | +| `dashboard/composables/useWhatsappCallSession.js` | WebRTC, recording, mute, beforeunload | +| `dashboard/api/whatsappCalls.js` | API client | +| `dashboard/components/widgets/WhatsappCallWidget.vue` | Floating call widget | +| `dashboard/components/widgets/conversation/ConversationHeader.vue` | Outbound call initiation | +| `dashboard/helper/voice.js` | Routes WhatsApp vs Twilio events | +| `dashboard/helper/actionCable.js` | WebSocket event handlers | + +--- + +## 11. Challenges & Design Decisions + +### 1. No trickle ICE + +**Problem:** Most WebRTC implementations use "trickle ICE" — sending ICE candidates one by one as they're discovered. Meta's API doesn't support this. It requires a **complete SDP** with all candidates baked in. + +**Solution:** We wait up to 10 seconds for ICE gathering to complete before sending the SDP. If it times out, we send a partial SDP (which may cause connection issues on restrictive networks). + +```javascript +// useWhatsappCallSession.js +const timeout = setTimeout(() => resolve(), 10000); +pc.onicegatheringstatechange = () => { + if (pc.iceGatheringState === 'complete') { + clearTimeout(timeout); + resolve(); + } +}; +``` + +### 2. SDP setup line mismatch + +**Problem:** Browsers generate `a=setup:actpass` in SDP answers, but Meta requires `a=setup:active`. + +**Solution:** Backend rewrites the SDP before sending to Meta: + +```ruby +# call_service.rb +def fix_sdp_setup(sdp) + sdp.gsub('a=setup:actpass', 'a=setup:active') +end +``` + +### 3. Two-phase accept (pre_accept + accept) + +**Problem:** Meta requires two separate API calls to accept a call — `pre_accept_call` first, then `accept_call`. Both need the same SDP answer. + +**Solution:** `CallService#pre_accept_and_accept` runs both inside a database row lock to prevent race conditions when multiple agents try to accept simultaneously. + +### 4. Race condition — multiple agents accepting + +**Problem:** When a call comes in, all online agents see it. Two agents might click "Accept" at the same millisecond. + +**Solution:** The accept logic runs inside `wa_call.with_lock do ... end` (Postgres row-level lock). The first agent succeeds; the second gets a `422 AlreadyAccepted` error. The ActionCable `whatsapp_call.accepted` event removes the call from other agents' widgets. + +### 5. Page close kills the call + +**Problem:** If an agent closes the tab during a call, the WebRTC connection drops but the backend doesn't know — the call lingers as "in progress" on Meta's side. + +**Solution:** A `beforeunload` handler sends a fire-and-forget terminate request using `fetch` with `keepalive: true`, which completes even after the page unloads. Auth headers are read from the session cookie. + +### 6. Accept after page refresh + +**Problem:** If an agent refreshes while a call is ringing, the in-memory call state (Pinia store + SDP offer) is lost. + +**Solution:** The "Accept" button on the `VoiceCall.vue` message bubble calls `acceptWhatsappCallById()`, which fetches the call's SDP offer from the backend API (`GET /whatsapp_calls/:id`) if it's not in the store. This only works while the call is still ringing. + +### 7. Client-side recording + +**Problem:** Meta doesn't provide call recordings or audio stream access. The audio only exists in the browser's `RTCPeerConnection`. + +**Solution:** We use the Web Audio API to mix both tracks and `MediaRecorder` to capture the audio client-side. The recording is uploaded after the call ends. This means if the browser crashes, the recording is lost. + +### 8. WhatsApp vs Twilio in the same UI + +**Problem:** The `VoiceCall.vue` bubble and voice infrastructure were built for Twilio. WhatsApp calls have different semantics (peer-to-peer vs conference, different status names). + +**Solution:** +- `call_source: 'whatsapp'` in message content_attributes distinguishes the two +- `CallMessageBuilder` maps WhatsApp statuses to Voice statuses +- `voice.js` helper routes events to the correct store +- `showJoinButton` in VoiceCall.vue has different logic per call source (WhatsApp only shows during ringing; Twilio shows during in-progress too because it's conference-based) + +--- + +## 12. Extending the Feature + +### Adding video calls + +Meta's calling API currently supports audio only, but they've announced video is coming. When it arrives: + +1. Add `{ audio: true, video: true }` to `getUserMedia` +2. Add video track to `RTCPeerConnection` +3. Add a `` element for remote stream (instead of ``) +4. Update `MediaRecorder` to record video (change MIME type) +5. Update the call widget to show video preview + +### Adding real-time transcription + +Currently transcription is post-call. For real-time: + +1. Create an `AudioWorklet` or use `ScriptProcessorNode` to extract audio chunks from the remote track +2. Stream chunks to a real-time STT service (e.g., Deepgram, Google Cloud Speech streaming) +3. Display live captions in the call widget +4. Store final transcript on call end + +### Adding call transfer + +Not currently supported by Meta's API, but when available: + +1. Add `transfer_call(call_id, target_agent_id)` to the provider service +2. Create a transfer button in the call widget +3. Handle the new agent receiving the transferred SDP +4. Update the `WhatsappCall` record with the new agent + +### Adding call analytics / dashboards + +The `WhatsappCall` model already tracks duration, direction, status, and agent. Possible additions: + +1. Add `wait_time_seconds` (time from ringing to accept) +2. Add `rating` (post-call CSAT) +3. Build aggregate queries for call volume, avg duration, missed call rate +4. Create a dedicated calls dashboard view + +### Adding SIP trunk support + +Meta supports SIP as an alternative to WebRTC: + +1. Configure SIP endpoint in the provider config +2. Route calls through Asterisk/FreeSWITCH instead of browser WebRTC +3. This enables server-side recording, IVR, and conference bridges +4. See Meta docs: `Default: SIP with WebRTC` vs `SIP (Explicit): SIP with SDES media` diff --git a/enterprise/app/builders/enterprise/messages/message_builder.rb b/enterprise/app/builders/enterprise/messages/message_builder.rb index 727248956..50bb08a28 100644 --- a/enterprise/app/builders/enterprise/messages/message_builder.rb +++ b/enterprise/app/builders/enterprise/messages/message_builder.rb @@ -1,8 +1,10 @@ module Enterprise::Messages::MessageBuilder private + INCOMING_ALLOWED_CHANNEL_TYPES = %w[Channel::Voice Channel::Whatsapp].freeze + def message_type - return @message_type if @message_type == 'incoming' && @conversation.inbox.channel_type == 'Channel::Voice' + return @message_type if @message_type == 'incoming' && INCOMING_ALLOWED_CHANNEL_TYPES.include?(@conversation.inbox.channel_type) super end diff --git a/enterprise/app/controllers/api/v1/accounts/whatsapp_calls_controller.rb b/enterprise/app/controllers/api/v1/accounts/whatsapp_calls_controller.rb index 3662a32a9..693deb15f 100644 --- a/enterprise/app/controllers/api/v1/accounts/whatsapp_calls_controller.rb +++ b/enterprise/app/controllers/api/v1/accounts/whatsapp_calls_controller.rb @@ -1,13 +1,28 @@ class Api::V1::Accounts::WhatsappCallsController < Api::V1::Accounts::BaseController before_action :ensure_whatsapp_call_enabled - before_action :set_whatsapp_call, only: [:accept, :reject, :terminate] + before_action :set_whatsapp_call, only: [:show, :accept, :reject, :terminate, :upload_recording] + + def show + render json: { + id: @whatsapp_call.id, + call_id: @whatsapp_call.call_id, + status: @whatsapp_call.status, + direction: @whatsapp_call.direction, + conversation_id: @whatsapp_call.conversation_id, + inbox_id: @whatsapp_call.inbox_id, + message_id: @whatsapp_call.message_id, + sdp_offer: @whatsapp_call.ringing? ? @whatsapp_call.sdp_offer : nil, + ice_servers: @whatsapp_call.ice_servers, + caller: caller_info + } + end def accept sdp_answer = params[:sdp_answer] return render json: { error: 'sdp_answer is required' }, status: :unprocessable_entity if sdp_answer.blank? wa_call = Whatsapp::CallService.new(wa_call: @whatsapp_call, agent: current_user).pre_accept_and_accept(sdp_answer) - render json: { id: wa_call.id, status: wa_call.status } + render json: { id: wa_call.id, status: wa_call.status, message_id: wa_call.message_id } rescue Whatsapp::CallErrors::NotRinging, Whatsapp::CallErrors::AlreadyAccepted => e render json: { error: e.message }, status: :unprocessable_entity rescue StandardError => e @@ -31,13 +46,26 @@ class Api::V1::Accounts::WhatsappCallsController < Api::V1::Accounts::BaseContro render json: { error: 'Failed to terminate call' }, status: :internal_server_error end + def upload_recording + return render json: { error: 'No recording file provided' }, status: :unprocessable_entity if params[:recording].blank? + return render json: { error: 'Call is not ended' }, status: :unprocessable_entity unless @whatsapp_call.terminal? + + attach_recording_and_enqueue_transcription + render json: { id: @whatsapp_call.id, status: 'uploaded' } + rescue StandardError => e + Rails.logger.error "[WHATSAPP CALL] upload_recording failed: #{e.message}" + render json: { error: 'Failed to upload recording' }, status: :internal_server_error + end + def initiate conversation = current_account.conversations.find(params[:conversation_id]) error = validate_whatsapp_calling(conversation) return render json: { error: error }, status: :unprocessable_entity if error wa_call = create_outbound_call(conversation) - render json: { status: 'calling', call_id: wa_call.call_id, id: wa_call.id } + message = Whatsapp::CallMessageBuilder.create!(conversation: conversation, wa_call: wa_call, user: current_user) + wa_call.update!(message_id: message.id) + render json: { status: 'calling', call_id: wa_call.call_id, id: wa_call.id, message_id: message.id } rescue Whatsapp::CallErrors::NoCallPermission handle_no_call_permission(conversation) rescue ActiveRecord::RecordNotFound @@ -96,4 +124,17 @@ class Api::V1::Accounts::WhatsappCallsController < Api::V1::Accounts::BaseContro rescue ActiveRecord::RecordNotFound render json: { error: 'Call not found' }, status: :not_found end + + def attach_recording_and_enqueue_transcription + @whatsapp_call.recording.attach(params[:recording]) + Whatsapp::CallMessageBuilder.update_recording_url!(wa_call: @whatsapp_call) + Whatsapp::CallTranscriptionJob.perform_later(@whatsapp_call.id) + end + + def caller_info + contact = @whatsapp_call.conversation&.contact + return {} unless contact + + { name: contact.name, phone: contact.phone_number, avatar: contact.avatar_url } + end end diff --git a/enterprise/app/jobs/whatsapp/call_transcription_job.rb b/enterprise/app/jobs/whatsapp/call_transcription_job.rb new file mode 100644 index 000000000..5f6a1e3e5 --- /dev/null +++ b/enterprise/app/jobs/whatsapp/call_transcription_job.rb @@ -0,0 +1,15 @@ +class Whatsapp::CallTranscriptionJob < ApplicationJob + queue_as :low + + retry_on ActiveStorage::FileNotFoundError, wait: 2.seconds, attempts: 3 + discard_on Faraday::BadRequestError do |job, error| + Rails.logger.warn("[WHATSAPP CALL] Discarding transcription job: call_id=#{job.arguments.first}, status=#{error.response&.dig(:status)}") + end + + def perform(whatsapp_call_id) + wa_call = WhatsappCall.find_by(id: whatsapp_call_id) + return if wa_call.blank? || !wa_call.recording.attached? + + Whatsapp::CallTranscriptionService.new(wa_call).perform + end +end diff --git a/enterprise/app/models/whatsapp_call.rb b/enterprise/app/models/whatsapp_call.rb index 1b76abe7b..27e5c1a1f 100644 --- a/enterprise/app/models/whatsapp_call.rb +++ b/enterprise/app/models/whatsapp_call.rb @@ -6,6 +6,9 @@ class WhatsappCall < ApplicationRecord belongs_to :inbox belongs_to :conversation belongs_to :accepted_by_agent, class_name: 'User', optional: true + belongs_to :message, optional: true + + has_one_attached :recording validates :call_id, presence: true, uniqueness: true validates :direction, inclusion: { in: DIRECTIONS } @@ -33,4 +36,10 @@ class WhatsappCall < ApplicationRecord def ice_servers meta['ice_servers'] || [] end + + def recording_url + return unless recording.attached? + + Rails.application.routes.url_helpers.rails_blob_path(recording, only_path: true) + end end diff --git a/enterprise/app/services/whatsapp/call_message_builder.rb b/enterprise/app/services/whatsapp/call_message_builder.rb new file mode 100644 index 000000000..32e549b6d --- /dev/null +++ b/enterprise/app/services/whatsapp/call_message_builder.rb @@ -0,0 +1,108 @@ +class Whatsapp::CallMessageBuilder + WHATSAPP_TO_VOICE_STATUS = { + 'ringing' => 'ringing', + 'accepted' => 'in-progress', + 'rejected' => 'failed', + 'missed' => 'no-answer', + 'ended' => 'completed', + 'failed' => 'failed' + }.freeze + + def self.create!(conversation:, wa_call:, user: nil) + new(conversation: conversation, wa_call: wa_call, user: user).create! + end + + def self.update_status!(wa_call:, status: nil, agent: nil, duration_seconds: nil) + new(conversation: wa_call.conversation, wa_call: wa_call).update_status!( + status: status, agent: agent, duration_seconds: duration_seconds + ) + end + + def self.update_recording_url!(wa_call:) + message = wa_call.message + return unless message + + data = (message.content_attributes || {}).dup + data['data'] ||= {} + data['data']['recording_url'] = wa_call.recording_url + message.update!(content_attributes: data) + end + + def initialize(conversation:, wa_call:, user: nil) + @conversation = conversation + @wa_call = wa_call + @user = user + end + + def create! + params = { + content: 'WhatsApp Call', + message_type: message_type, + content_type: 'voice_call', + content_attributes: { 'data' => build_data_payload } + } + + Messages::MessageBuilder.new(sender, conversation, params).perform + end + + def update_status!(status:, agent: nil, duration_seconds: nil) + message = wa_call.message + return unless message + + data = (message.content_attributes || {}).dup + data['data'] ||= {} + data['data']['status'] = map_status(status) if status + data['data']['accepted_by'] = { 'id' => agent.id, 'name' => agent.name } if agent + data['data']['duration_seconds'] = duration_seconds if duration_seconds + + message.update!(content_attributes: data) + message + end + + private + + attr_reader :conversation, :wa_call, :user + + def build_data_payload + { + 'call_sid' => wa_call.call_id, + 'status' => map_status(wa_call.status), + 'call_direction' => wa_call.direction, + 'call_source' => 'whatsapp', + 'wa_call_id' => wa_call.id, + 'from_number' => from_number, + 'to_number' => to_number, + 'meta' => { 'created_at' => Time.zone.now.to_i } + } + end + + def message_type + wa_call.direction == 'outbound' ? 'outgoing' : 'incoming' + end + + def sender + return user if wa_call.direction == 'outbound' && user + + conversation.contact + end + + def from_number + if wa_call.direction == 'inbound' + conversation.contact&.phone_number + else + conversation.inbox.channel&.phone_number + end + end + + def to_number + if wa_call.direction == 'inbound' + conversation.inbox.channel&.phone_number + else + conversation.contact&.phone_number + end + end + + def map_status(status) + WHATSAPP_TO_VOICE_STATUS[status] || status + end +end diff --git a/enterprise/app/services/whatsapp/call_service.rb b/enterprise/app/services/whatsapp/call_service.rb index ddc41b62a..fe05bd734 100644 --- a/enterprise/app/services/whatsapp/call_service.rb +++ b/enterprise/app/services/whatsapp/call_service.rb @@ -24,6 +24,8 @@ class Whatsapp::CallService ) end + Whatsapp::CallMessageBuilder.update_status!(wa_call: wa_call, status: 'accepted', agent: agent) + update_conversation_call_status('in-progress') broadcast_accepted wa_call end @@ -37,6 +39,8 @@ class Whatsapp::CallService Rails.logger.error "[WHATSAPP CALL] reject_call API returned false for call #{wa_call.call_id}" unless success wa_call.update!(status: 'rejected') + Whatsapp::CallMessageBuilder.update_status!(wa_call: wa_call, status: 'rejected') + update_conversation_call_status('failed') broadcast_call_ended wa_call end @@ -49,6 +53,8 @@ class Whatsapp::CallService Rails.logger.error "[WHATSAPP CALL] terminate_call API returned false for call #{wa_call.call_id}" unless success wa_call.update!(status: 'ended') + Whatsapp::CallMessageBuilder.update_status!(wa_call: wa_call, status: 'ended') + update_conversation_call_status('completed') broadcast_call_ended wa_call end @@ -67,6 +73,12 @@ class Whatsapp::CallService sdp.gsub('a=setup:actpass', 'a=setup:active') end + def update_conversation_call_status(mapped_status) + conversation = wa_call.conversation + attrs = (conversation.additional_attributes || {}).merge('call_status' => mapped_status) + conversation.update!(additional_attributes: attrs) + end + def broadcast_accepted payload = { event: 'whatsapp_call.accepted', diff --git a/enterprise/app/services/whatsapp/call_transcription_service.rb b/enterprise/app/services/whatsapp/call_transcription_service.rb new file mode 100644 index 000000000..3dbef5efe --- /dev/null +++ b/enterprise/app/services/whatsapp/call_transcription_service.rb @@ -0,0 +1,82 @@ +class Whatsapp::CallTranscriptionService < Llm::LegacyBaseOpenAiService + WHISPER_MODEL = 'whisper-1'.freeze + + attr_reader :wa_call, :account + + def initialize(wa_call) + super() + @wa_call = wa_call + @account = wa_call.account + end + + def perform + return { error: 'Transcription not available' } unless can_transcribe? + return { error: 'No recording attached' } unless wa_call.recording.attached? + + transcribed_text = transcribe_audio + update_call_and_message(transcribed_text) + { success: true, transcript: transcribed_text } + rescue Faraday::UnauthorizedError + Rails.logger.warn('[WHATSAPP CALL] Skipping transcription: OpenAI configuration is invalid (401)') + { error: 'OpenAI configuration is invalid' } + end + + private + + def can_transcribe? + account.feature_enabled?('captain_integration') && + account.usage_limits[:captain][:responses][:current_available].positive? + end + + def transcribe_audio + temp_file_path = fetch_audio_file + transcribed_text = nil + + File.open(temp_file_path, 'rb') do |file| + response = @client.audio.transcribe( + parameters: { model: WHISPER_MODEL, file: file, temperature: 0.4 } + ) + transcribed_text = response['text'] + end + + transcribed_text + ensure + FileUtils.rm_f(temp_file_path) if temp_file_path.present? + end + + def fetch_audio_file + blob = wa_call.recording.blob + temp_dir = Rails.root.join('tmp/uploads/call-transcriptions') + FileUtils.mkdir_p(temp_dir) + + extension = extension_from_content_type(blob.content_type) + temp_file_path = File.join(temp_dir, "#{blob.key}.#{extension}") + + File.open(temp_file_path, 'wb') do |file| + blob.open { |blob_file| IO.copy_stream(blob_file, file) } + end + + temp_file_path + end + + def update_call_and_message(transcribed_text) + return if transcribed_text.blank? + + wa_call.update!(transcript: transcribed_text) + account.increment_response_usage + + message = wa_call.message + return unless message + + data = (message.content_attributes || {}).dup + data['data'] ||= {} + data['data']['transcript'] = transcribed_text + data['data']['recording_url'] = wa_call.recording_url + message.update!(content_attributes: data) + end + + def extension_from_content_type(content_type) + subtype = content_type.to_s.downcase.split(';').first.to_s.split('/').last.to_s + { 'webm' => 'webm', 'ogg' => 'ogg', 'x-m4a' => 'm4a', 'x-wav' => 'wav', 'mpeg' => 'mp3' }.fetch(subtype, 'webm') + end +end diff --git a/enterprise/app/services/whatsapp/incoming_call_service.rb b/enterprise/app/services/whatsapp/incoming_call_service.rb index 8bd7f7504..fc113e458 100644 --- a/enterprise/app/services/whatsapp/incoming_call_service.rb +++ b/enterprise/app/services/whatsapp/incoming_call_service.rb @@ -33,6 +33,8 @@ class Whatsapp::IncomingCallService Rails.logger.info "[WHATSAPP CALL] call_connect for existing call #{call_id} (direction=#{direction})" sdp_answer = fix_sdp_setup(call_payload.dig(:session, :sdp)) existing_call.update!(status: 'accepted', meta: existing_call.meta.merge('sdp_answer' => sdp_answer)) + Whatsapp::CallMessageBuilder.update_status!(wa_call: existing_call, status: 'accepted') + update_conversation_call_status(existing_call.conversation, 'in-progress', direction) broadcast_outbound_call_connected(existing_call, sdp_answer) return end @@ -44,12 +46,20 @@ class Whatsapp::IncomingCallService return unless conversation wa_call = create_call_record(call_payload, conversation, direction) - create_call_activity_message(conversation, 'incoming_call', direction) + create_voice_call_message(conversation, wa_call) + update_conversation_call_status(conversation, 'ringing', direction) broadcast_incoming_call(wa_call, contact, call_payload.dig(:session, :sdp)) rescue ActiveRecord::RecordNotUnique Rails.logger.warn "[WHATSAPP CALL] Duplicate call_id received: #{call_id}" end + def create_voice_call_message(conversation, wa_call, user: nil) + message = Whatsapp::CallMessageBuilder.create!(conversation: conversation, wa_call: wa_call, user: user) + wa_call.update!(message_id: message.id) + rescue StandardError => e + Rails.logger.error "[WHATSAPP CALL] Failed to create voice_call message: #{e.message}" + end + def create_call_record(call_payload, conversation, direction) WhatsappCall.create!( account: inbox.account, @@ -70,15 +80,20 @@ class Whatsapp::IncomingCallService wa_call = WhatsappCall.find_by(call_id: call_id) return unless wa_call - final_status = wa_call.accepted? ? 'ended' : 'missed' + # Determine if the call was answered: check accepted status, duration > 0, + # or accepted_by_agent_id presence (handles webhook race conditions) + was_answered = wa_call.accepted? || duration.to_i.positive? || wa_call.accepted_by_agent_id.present? + final_status = was_answered ? 'ended' : 'missed' wa_call.update!( status: final_status, duration_seconds: duration, end_reason: end_reason ) - call_event = duration.to_i.positive? ? 'call_ended' : 'call_missed' - create_call_activity_message(wa_call.conversation, call_event, wa_call.direction, duration: duration) + agent = wa_call.accepted_by_agent if wa_call.accepted_by_agent_id.present? + Whatsapp::CallMessageBuilder.update_status!(wa_call: wa_call, status: final_status, agent: agent, duration_seconds: duration) + mapped = Whatsapp::CallMessageBuilder::WHATSAPP_TO_VOICE_STATUS[final_status] || final_status + update_conversation_call_status(wa_call.conversation, mapped, wa_call.direction) broadcast_call_ended(wa_call) end @@ -113,36 +128,12 @@ class Whatsapp::IncomingCallService ) end - def create_call_activity_message(conversation, event, direction, duration: nil) - content = call_activity_content(event, direction, duration) - conversation.messages.create!( - account_id: conversation.account_id, - inbox_id: conversation.inbox_id, - message_type: :activity, - content: content, - content_attributes: { - call_event: event, - call_direction: direction, - call_duration_seconds: duration - } + def update_conversation_call_status(conversation, call_status, direction) + attrs = (conversation.additional_attributes || {}).merge( + 'call_status' => call_status, + 'call_direction' => direction ) - end - - def call_activity_content(event, direction, duration) - case event - when 'incoming_call' - direction == 'inbound' ? 'Incoming WhatsApp call' : 'Outgoing WhatsApp call' - when 'call_ended' then "WhatsApp call ended — #{format_duration(duration)}" - when 'call_missed' then 'Missed WhatsApp call' - else 'WhatsApp call' - end - end - - def format_duration(seconds) - return '0s' if seconds.nil? || seconds.zero? - - mins, secs = seconds.divmod(60) - mins.positive? ? "#{mins}m #{secs}s" : "#{secs}s" + conversation.update!(additional_attributes: attrs) end def broadcast_incoming_call(wa_call, contact, sdp_offer)
+ {{ transcript }} +