From 5e5dc21f2fd6d86fc5ba49538a488db09d9d3a1e Mon Sep 17 00:00:00 2001 From: tds-1 Date: Tue, 21 Apr 2026 04:47:53 +0000 Subject: [PATCH] feat(whatsapp-call): add server-side WebRTC media server for call persistence Implements a Pion Go B2BUA media server sidecar that sits between Meta's WhatsApp Cloud API and the agent's browser, enabling call persistence across page reloads, server-side recording, multi-participant support, audio injection, and AI integration readiness. Go Media Server (enterprise/media-server/): - Pion WebRTC v4 B2BUA with Peer A (Meta) and Peer B (Agent) connections - Real-time OGG/Opus recording with crash recovery - Audio bridge with multi-peer fan-out and AudioConsumer plugin interface - Audio injection from OGG files with loop support for hold music - Session manager with graceful shutdown and orphaned recording recovery - HTTP API with Bearer token auth, health checks, and metrics Rails Integration: - Whatsapp::MediaServerClient HTTP client for Go sidecar communication - Dual-mode CallService: legacy browser-direct and server-relay paths - Media server callback controller for agent disconnect/recording/terminate - CallRecordingFetchJob: downloads OGG from Go server to ActiveStorage/S3 - CallCleanupJob: sweeps stale ringing and in-progress calls - New endpoints: active, agent_answer, reconnect, join, play_audio - DB migration: media_session_id column with indexes Frontend: - Dual-mode composable auto-detecting legacy vs server-relay - handleAgentOffer() for receiving SDP from media server - useCallReconnection composable for page reload recovery - Removed terminateCallOnUnload in server-relay mode - Simplified outbound call flow in ConversationHeader - New ActionCable event: whatsapp_call.agent_offer Documentation: - Server-side WebRTC architecture spec (1505 lines) - Implementation plan with 119 trackable checklist items - Feature spec, PR breakdown, and relay architecture docs --- .env.example | 7 + app/javascript/dashboard/api/whatsappCalls.js | 47 +- .../message/bubbles/VoiceCall.vue | 15 +- .../components/widgets/WhatsappCallWidget.vue | 44 +- .../conversation/ConversationHeader.vue | 121 +- .../composables/useCallReconnection.js | 63 + .../composables/useWhatsappCallSession.js | 255 ++- .../dashboard/helper/actionCable.js | 62 +- .../i18n/locale/en/whatsappCall.json | 4 +- .../dashboard/stores/whatsappCalls.js | 31 +- config/routes.rb | 17 +- ...042235_add_media_server_fields_to_calls.rb | 7 + db/schema.rb | 6 +- docker-compose.production.yaml | 27 + docker-compose.yaml | 27 + docs/SERVER_SIDE_WEBRTC_ARCHITECTURE.md | 1504 +++++++++++++++++ .../SERVER_SIDE_WEBRTC_IMPLEMENTATION_PLAN.md | 1314 ++++++++++++++ docs/WHATSAPP_CALL_FEATURE_SPEC.md | 272 +++ docs/WHATSAPP_CALL_PR_BREAKDOWN.md | 347 ++++ ...WHATSAPP_CALL_SERVER_RELAY_ARCHITECTURE.md | 694 ++++++++ .../media_server/callbacks_controller.rb | 67 + .../v1/accounts/whatsapp_calls_controller.rb | 136 +- .../app/jobs/whatsapp/call_cleanup_job.rb | 33 + .../jobs/whatsapp/call_recording_fetch_job.rb | 26 + enterprise/app/models/call.rb | 5 + .../app/services/whatsapp/call_service.rb | 90 +- .../whatsapp/incoming_call_service.rb | 39 +- .../services/whatsapp/media_server_client.rb | 109 ++ enterprise/media-server/Dockerfile | 31 + enterprise/media-server/README.md | 155 ++ enterprise/media-server/cmd/server/main.go | 133 ++ enterprise/media-server/go.mod | 30 + .../media-server/internal/auth/middleware.go | 65 + .../internal/callback/rails_client.go | 138 ++ .../media-server/internal/config/config.go | 197 +++ .../media-server/internal/media/bridge.go | 323 ++++ .../media-server/internal/media/injector.go | 189 +++ .../media-server/internal/media/recorder.go | 208 +++ .../media-server/internal/peer/agent_peer.go | 236 +++ .../media-server/internal/peer/meta_peer.go | 257 +++ .../media-server/internal/server/handlers.go | 662 ++++++++ .../media-server/internal/server/router.go | 116 ++ .../media-server/internal/session/manager.go | 292 ++++ .../media-server/internal/session/session.go | 563 ++++++ 44 files changed, 8827 insertions(+), 137 deletions(-) create mode 100644 app/javascript/dashboard/composables/useCallReconnection.js create mode 100644 db/migrate/20260421042235_add_media_server_fields_to_calls.rb create mode 100644 docs/SERVER_SIDE_WEBRTC_ARCHITECTURE.md create mode 100644 docs/SERVER_SIDE_WEBRTC_IMPLEMENTATION_PLAN.md create mode 100644 docs/WHATSAPP_CALL_FEATURE_SPEC.md create mode 100644 docs/WHATSAPP_CALL_PR_BREAKDOWN.md create mode 100644 docs/WHATSAPP_CALL_SERVER_RELAY_ARCHITECTURE.md create mode 100644 enterprise/app/controllers/api/v1/accounts/media_server/callbacks_controller.rb create mode 100644 enterprise/app/jobs/whatsapp/call_cleanup_job.rb create mode 100644 enterprise/app/jobs/whatsapp/call_recording_fetch_job.rb create mode 100644 enterprise/app/services/whatsapp/media_server_client.rb create mode 100644 enterprise/media-server/Dockerfile create mode 100644 enterprise/media-server/README.md create mode 100644 enterprise/media-server/cmd/server/main.go create mode 100644 enterprise/media-server/go.mod create mode 100644 enterprise/media-server/internal/auth/middleware.go create mode 100644 enterprise/media-server/internal/callback/rails_client.go create mode 100644 enterprise/media-server/internal/config/config.go create mode 100644 enterprise/media-server/internal/media/bridge.go create mode 100644 enterprise/media-server/internal/media/injector.go create mode 100644 enterprise/media-server/internal/media/recorder.go create mode 100644 enterprise/media-server/internal/peer/agent_peer.go create mode 100644 enterprise/media-server/internal/peer/meta_peer.go create mode 100644 enterprise/media-server/internal/server/handlers.go create mode 100644 enterprise/media-server/internal/server/router.go create mode 100644 enterprise/media-server/internal/session/manager.go create mode 100644 enterprise/media-server/internal/session/session.go diff --git a/.env.example b/.env.example index bc7380a29..62ba1a24f 100644 --- a/.env.example +++ b/.env.example @@ -277,3 +277,10 @@ AZURE_APP_SECRET= # REDIS_ALFRED_SIZE=10 # REDIS_VELMA_SIZE=10 + +# Media Server (WhatsApp Calling - Server-Side WebRTC) +# Enable server-side WebRTC relay for call persistence across page reloads +# and server-side recording. Requires the chatwoot-media-server sidecar. +# MEDIA_SERVER_URL=http://localhost:4000 +# MEDIA_SERVER_AUTH_TOKEN= +# MEDIA_SERVER_PUBLIC_IP= diff --git a/app/javascript/dashboard/api/whatsappCalls.js b/app/javascript/dashboard/api/whatsappCalls.js index 3b35588b1..e8c3de378 100644 --- a/app/javascript/dashboard/api/whatsappCalls.js +++ b/app/javascript/dashboard/api/whatsappCalls.js @@ -10,10 +10,11 @@ class WhatsappCallsAPI extends ApiClient { return axios.get(`${this.url}/${callId}`); } + // Accept a ringing call. sdpAnswer is optional — omitted in server-relay mode + // where the media server handles WebRTC negotiation. accept(callId, sdpAnswer) { - return axios.post(`${this.url}/${callId}/accept`, { - sdp_answer: sdpAnswer, - }); + const body = sdpAnswer ? { sdp_answer: sdpAnswer } : {}; + return axios.post(`${this.url}/${callId}/accept`, body); } reject(callId) { @@ -24,13 +25,47 @@ class WhatsappCallsAPI extends ApiClient { return axios.post(`${this.url}/${callId}/terminate`); } + // Initiate an outbound call. sdpOffer is optional — omitted in server-relay + // mode where the media server generates the SDP offer for Meta. initiate(conversationId, sdpOffer) { - return axios.post(`${this.url}/initiate`, { - conversation_id: conversationId, - sdp_offer: sdpOffer, + const body = { conversation_id: conversationId }; + if (sdpOffer) body.sdp_offer = sdpOffer; + return axios.post(`${this.url}/initiate`, body); + } + + // Send the agent's SDP answer for the Peer B connection (server-relay mode). + agentAnswer(callId, sdpAnswer) { + return axios.post(`${this.url}/${callId}/agent_answer`, { + sdp_answer: sdpAnswer, }); } + // Get the current agent's active call (if any). Used for reconnection on page load. + active() { + return axios.get(`${this.url}/active`); + } + + // Reconnect to an active call after page reload. Server creates a new Peer B + // and returns a fresh SDP offer via ActionCable. + reconnect(callId) { + return axios.post(`${this.url}/${callId}/reconnect`); + } + + // Join an existing call as a supervisor (listen-only by default). + join(callId, role = 'listen_only') { + return axios.post(`${this.url}/${callId}/join`, { role }); + } + + // Play an audio file to the caller via the media server. + playAudio(callId, { filePath, mode = 'replace', loop = false }) { + return axios.post(`${this.url}/${callId}/play_audio`, { + file_path: filePath, + mode, + loop, + }); + } + + // Legacy: upload browser-side recording. Deprecated when media server is enabled. uploadRecording(callId, blob) { const formData = new FormData(); formData.append('recording', blob, `call-${callId}.webm`); diff --git a/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue b/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue index 053bcaeb8..a24fb1fe6 100644 --- a/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue +++ b/app/javascript/dashboard/components-next/message/bubbles/VoiceCall.vue @@ -47,7 +47,7 @@ const isFailed = computed(() => // Call source and metadata — all camelCase due to deep transform const isWhatsappCall = computed(() => data.value?.callSource === 'whatsapp'); -const waCallId = computed(() => data.value?.callId); +const callId = computed(() => data.value?.callId); const acceptedBy = computed(() => data.value?.acceptedBy); const durationSeconds = computed(() => data.value?.durationSeconds); const recordingUrl = computed(() => data.value?.recordingUrl); @@ -64,10 +64,19 @@ const formattedDuration = computed(() => { }); // Show join/accept button logic -// WhatsApp: only ringing (peer-to-peer WebRTC — cannot rejoin after accept) +// WhatsApp with media server: ringing + in_progress (server-relay supports rejoin) +// WhatsApp without media server: only ringing (peer-to-peer WebRTC — cannot rejoin after accept) // Twilio: ringing + in-progress (conference model supports rejoin) const showJoinButton = computed(() => { if (isWhatsappCall.value) { + // Server-relay mode enables rejoining in-progress calls + const isMediaServerMode = data.value?.mediaServerEnabled; + if (isMediaServerMode) { + return [ + VOICE_CALL_STATUS.RINGING, + VOICE_CALL_STATUS.IN_PROGRESS, + ].includes(status.value); + } return status.value === VOICE_CALL_STATUS.RINGING; } return [VOICE_CALL_STATUS.RINGING, VOICE_CALL_STATUS.IN_PROGRESS].includes( @@ -133,7 +142,7 @@ const handleJoinCall = async () => { try { if (isWhatsappCall.value) { - const result = await acceptWhatsappCallById(waCallId.value); + const result = await acceptWhatsappCallById(callId.value); if (result?.success && result.call) { router.push({ name: 'inbox_conversation', diff --git a/app/javascript/dashboard/components/widgets/WhatsappCallWidget.vue b/app/javascript/dashboard/components/widgets/WhatsappCallWidget.vue index d96bd7f79..690605371 100644 --- a/app/javascript/dashboard/components/widgets/WhatsappCallWidget.vue +++ b/app/javascript/dashboard/components/widgets/WhatsappCallWidget.vue @@ -1,9 +1,11 @@ @@ -173,14 +197,20 @@ onUnmounted(() => {

- {{ - isOutboundRinging - ? t('WHATSAPP_CALL.RINGING') - : formattedCallDuration - }} + + +

diff --git a/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue b/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue index 98131a824..6185ee9ba 100644 --- a/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue +++ b/app/javascript/dashboard/components/widgets/conversation/ConversationHeader.vue @@ -104,32 +104,118 @@ const hasSlaPolicyId = computed(() => props.chat?.sla_policy_id); const canInitiateWhatsappCall = computed(() => { if (!isAWhatsAppCloudChannel.value) return false; if (!inbox.value?.calling_enabled) return false; - // Block if there's already an active or ringing WhatsApp call if (whatsappCallsStore.hasWhatsappCall) return false; return true; }); +// Detect if the media server is enabled for this inbox. +// When enabled, the browser should NOT create its own WebRTC offer. +const isMediaServerEnabled = computed( + () => !!inbox.value?.media_server_enabled +); + const waitForOutboundIceGathering = pc => - new Promise(resolve => { + new Promise((resolve, reject) => { if (pc.iceGatheringState === 'complete') { resolve(); return; } - const timeout = setTimeout(() => resolve(), 10000); + + let timeout = null; + + const cleanup = () => { + clearTimeout(timeout); + pc.onicegatheringstatechange = null; + pc.oniceconnectionstatechange = null; + }; + + timeout = setTimeout(() => { + cleanup(); + resolve(); + }, 10000); + pc.onicegatheringstatechange = () => { if (pc.iceGatheringState === 'complete') { - clearTimeout(timeout); + cleanup(); resolve(); } }; + pc.oniceconnectionstatechange = () => { + if (pc.iceConnectionState === 'failed') { + cleanup(); + reject(new Error('ICE connection failed')); + } + }; }); -const initiateWhatsappCall = async () => { +/** + * Server-relay mode: POST /initiate without SDP. The media server creates + * Peer A (Meta-side) and later sends the agent Peer B offer via ActionCable + * (whatsapp_call.outbound_connected with sdp_offer). + */ +const initiateServerRelayCall = async () => { + if (isInitiatingCall.value || !currentChat.value?.id) return; + isInitiatingCall.value = true; + + try { + const response = await WhatsappCallsAPI.initiate(currentChat.value.id); + + const callStatus = response.data?.status; + if ( + callStatus === 'permission_requested' || + callStatus === 'permission_pending' + ) { + const message = + callStatus === 'permission_requested' + ? t('WHATSAPP_CALL.PERMISSION_REQUESTED') + : t('WHATSAPP_CALL.PERMISSION_PENDING'); + emitter.emit(BUS_EVENTS.SHOW_ALERT, { message, type: 'info' }); + return; + } + + emitter.emit(BUS_EVENTS.SHOW_ALERT, { + message: t('WHATSAPP_CALL.CALLING'), + type: 'success', + }); + + const outboundCallId = response.data?.call_id; + + // Set active call — WebRTC setup happens when ActionCable delivers agent_offer + whatsappCallsStore.setActiveCall({ + id: response.data?.id, + callId: outboundCallId, + direction: 'outbound', + status: 'ringing', + serverRelay: true, + conversationId: currentChat.value.id, + caller: { + name: currentContact.value?.name, + phone: currentContact.value?.phone_number, + avatar: currentContact.value?.thumbnail, + }, + }); + } catch (err) { + const errorMessage = + err.response?.data?.error || t('WHATSAPP_CALL.CALL_FAILED'); + emitter.emit(BUS_EVENTS.SHOW_ALERT, { + message: errorMessage, + type: 'error', + }); + } finally { + isInitiatingCall.value = false; + } +}; + +/** + * Legacy mode: Browser creates RTCPeerConnection, generates SDP offer, + * sends it to backend which forwards to Meta. + */ +const initiateLegacyCall = async () => { if (isInitiatingCall.value || !currentChat.value?.id) return; isInitiatingCall.value = true; let pc = null; let localStream = null; - let waCallId = null; + let recordCallId = null; try { localStream = await navigator.mediaDevices.getUserMedia({ audio: true }); pc = new RTCPeerConnection({ @@ -137,7 +223,6 @@ const initiateWhatsappCall = async () => { }); localStream.getTracks().forEach(track => pc.addTrack(track, localStream)); - // Handle remote audio from Meta — ontrack fires when the callee picks up pc.ontrack = event => { const [stream] = event.streams; if (!stream) return; @@ -146,21 +231,13 @@ const initiateWhatsappCall = async () => { audio.autoplay = true; document.body.appendChild(audio); setOutboundCallProperty('audio', audio); - // Remote audio arrived — callee picked up, transition from ringing to connected whatsappCallsStore.markActiveCallConnected(); - // Start recording both local + remote audio - if (waCallId) startCallRecording(pc, localStream, waCallId); - }; - - pc.oniceconnectionstatechange = () => { - // eslint-disable-next-line no-console - console.log('[WhatsApp Call] Outbound ICE state:', pc.iceConnectionState); + if (recordCallId) startCallRecording(pc, localStream, recordCallId); }; const offer = await pc.createOffer(); await pc.setLocalDescription(offer); - // Wait for ICE gathering to complete before sending offer await waitForOutboundIceGathering(pc); const completeSdp = pc.localDescription.sdp; @@ -190,18 +267,17 @@ const initiateWhatsappCall = async () => { }); const outboundCallId = response.data?.call_id; - waCallId = response.data?.id; + recordCallId = response.data?.id; setOutboundCallProperty('pc', pc); setOutboundCallProperty('stream', localStream); setOutboundCallProperty('callId', outboundCallId); - // Set active call in store so the WhatsappCallWidget renders - // Status starts as 'ringing' — updated to 'connected' when SDP answer arrives whatsappCallsStore.setActiveCall({ id: response.data?.id, callId: outboundCallId, direction: 'outbound', status: 'ringing', + serverRelay: false, conversationId: currentChat.value.id, caller: { name: currentContact.value?.name, @@ -222,6 +298,13 @@ const initiateWhatsappCall = async () => { isInitiatingCall.value = false; } }; + +const initiateWhatsappCall = () => { + if (isMediaServerEnabled.value) { + return initiateServerRelayCall(); + } + return initiateLegacyCall(); +};