feat(whatsapp): Add support for voice messages (#14606)
> Reopened from #13613, now from a personal fork (`gabrieljablonski/chatwoot`) so maintainers can push edits — organization-owned forks don't support "Allow edits from maintainers". The previous PR is closed in favor of this one; same commits, same diff. ## Description This PR adds support for sending voice messages (voice notes) through the WhatsApp Cloud API. When agents record audio in Chatwoot, it is now transcoded in the browser from WebM/Opus to OGG/Opus and sent with the `voice: true` flag, so it appears as a native voice note bubble on WhatsApp — not as a file/document attachment. Closes #13283 **Key Changes:** - Added `webmOpusToOgg.js` — a pure JS EBML parser + OGG page builder that remuxes browser-recorded WebM/Opus audio into OGG/Opus entirely client-side, with no server-side dependencies. - Updated `AudioRecorder.vue` to use an explicit `mimeType` hint, proper resource cleanup, and an `AUDIO_EXTENSION_MAP` for correct file extensions. - Renamed `mp3ConversionUtils.js` → `audioConversionUtils.js` and added OGG conversion support via the new remuxer. - Updated `ReplyBox.vue` to request OGG format for WhatsApp channels, pass `isVoiceMessage` per-attachment, and handle recording errors with a user-facing alert. - Updated `MessageBuilder` to read the `is_voice_message` param and persist it in attachment metadata. - Updated `WhatsappCloudService` to: - Normalize `audio/opus` → `audio/ogg` content type on ActiveStorage blobs (works around Marcel gem re-detection). - Send the `voice: true` flag when the attachment is a voice message with `audio/ogg` content type. - Use WhatsApp Cloud API `v24.0` for the attachment endpoint. - Added `AUDIO_CONVERSION_FAILED` i18n key. **How it works:** 1. The browser records audio as WebM/Opus (Chrome/Firefox default). 2. `audioConversionUtils.js` remuxes it to OGG/Opus using the pure-JS `webmOpusToOgg` remuxer — no server transcoding needed. 3. The OGG file is uploaded with `is_voice_message: true` in the form payload. 4. `MessageBuilder` persists `is_voice_message` in the attachment's `meta` hash. 5. `WhatsappCloudService` normalizes the blob content type if needed, then sends the attachment with `voice: true` so WhatsApp renders it as a voice note. ## Type of change - [X] New feature (non-breaking change which adds functionality) ## How Has This Been Tested? 1. Record a voice message in a WhatsApp Cloud conversation. 2. Verify the audio is transcoded to OGG (check file extension in the attachment preview). 3. Verify the message arrives on WhatsApp as a voice note bubble (not a document/file). 4. Send an image or document attachment and verify it still works as before (no `voice` flag). 5. Send a regular (non-voice) audio file and verify it arrives without the voice flag. --------- Co-authored-by: Muhsin Keloth <muhsinkeramam@gmail.com> Co-authored-by: Muhsin <12408980+muhsin-k@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gabriel Jablonski
co-authored by
Muhsin Keloth
Muhsin
Claude Opus 4.8
Muhsin Keloth
Muhsin
Claude Opus 4.8
parent
170b64d1f1
commit
37eed5de1e
@@ -13,6 +13,7 @@ class Messages::MessageBuilder
|
||||
@account = conversation.account
|
||||
@message_type = params[:message_type] || 'outgoing'
|
||||
@attachments = params[:attachments]
|
||||
@is_voice_message = ActiveModel::Type::Boolean.new.cast(params[:is_voice_message])
|
||||
@automation_rule = content_attributes&.dig(:automation_rule_id)
|
||||
return unless params.instance_of?(ActionController::Parameters)
|
||||
|
||||
@@ -56,16 +57,25 @@ class Messages::MessageBuilder
|
||||
file: uploaded_attachment
|
||||
)
|
||||
|
||||
attachment.file_type = if uploaded_attachment.is_a?(String)
|
||||
file_type_by_signed_id(
|
||||
uploaded_attachment
|
||||
)
|
||||
else
|
||||
file_type(uploaded_attachment&.content_type)
|
||||
end
|
||||
attachment.file_type = attachment_file_type(uploaded_attachment)
|
||||
tag_voice_message(attachment)
|
||||
end
|
||||
end
|
||||
|
||||
def attachment_file_type(uploaded_attachment)
|
||||
if uploaded_attachment.is_a?(String)
|
||||
file_type_by_signed_id(uploaded_attachment)
|
||||
else
|
||||
file_type(uploaded_attachment&.content_type)
|
||||
end
|
||||
end
|
||||
|
||||
def tag_voice_message(attachment)
|
||||
return unless @is_voice_message && attachment.file_type == 'audio'
|
||||
|
||||
attachment.meta = (attachment.meta || {}).merge('is_voice_message' => true)
|
||||
end
|
||||
|
||||
def process_emails
|
||||
return unless @conversation.inbox&.inbox_type == 'Email'
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ export const buildCreatePayload = ({
|
||||
bccEmails = '',
|
||||
toEmails = '',
|
||||
templateParams,
|
||||
isVoiceMessage = false,
|
||||
}) => {
|
||||
let payload;
|
||||
if (files && files.length !== 0) {
|
||||
@@ -33,6 +34,9 @@ export const buildCreatePayload = ({
|
||||
if (contentAttributes) {
|
||||
payload.append('content_attributes', JSON.stringify(contentAttributes));
|
||||
}
|
||||
if (isVoiceMessage) {
|
||||
payload.append('is_voice_message', true);
|
||||
}
|
||||
} else {
|
||||
payload = {
|
||||
content: message,
|
||||
@@ -64,6 +68,7 @@ class MessageApi extends ApiClient {
|
||||
bccEmails = '',
|
||||
toEmails = '',
|
||||
templateParams,
|
||||
isVoiceMessage = false,
|
||||
}) {
|
||||
return axios({
|
||||
method: 'post',
|
||||
@@ -78,6 +83,7 @@ class MessageApi extends ApiClient {
|
||||
bccEmails,
|
||||
toEmails,
|
||||
templateParams,
|
||||
isVoiceMessage,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -83,5 +83,29 @@ describe('#ConversationAPI', () => {
|
||||
template_params: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('appends is_voice_message when isVoiceMessage is true', () => {
|
||||
const formPayload = buildCreatePayload({
|
||||
message: 'voice message',
|
||||
echoId: 42,
|
||||
isPrivate: false,
|
||||
files: [new Blob(['audio-data'], { type: 'audio/ogg' })],
|
||||
isVoiceMessage: true,
|
||||
});
|
||||
expect(formPayload).toBeInstanceOf(FormData);
|
||||
expect(formPayload.get('is_voice_message')).toEqual('true');
|
||||
});
|
||||
|
||||
it('does not append is_voice_message when isVoiceMessage is false', () => {
|
||||
const formPayload = buildCreatePayload({
|
||||
message: 'regular audio',
|
||||
echoId: 43,
|
||||
isPrivate: false,
|
||||
files: [new Blob(['audio-data'], { type: 'audio/ogg' })],
|
||||
isVoiceMessage: false,
|
||||
});
|
||||
expect(formPayload).toBeInstanceOf(FormData);
|
||||
expect(formPayload.get('is_voice_message')).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,11 +14,11 @@ const props = defineProps({
|
||||
const emit = defineEmits(['removeAttachment']);
|
||||
|
||||
const nonRecordedAudioAttachments = computed(() => {
|
||||
return props.attachments.filter(attachment => !attachment?.isRecordedAudio);
|
||||
return props.attachments.filter(attachment => !attachment?.isVoiceMessage);
|
||||
});
|
||||
|
||||
const recordedAudioAttachments = computed(() =>
|
||||
props.attachments.filter(attachment => attachment.isRecordedAudio)
|
||||
props.attachments.filter(attachment => attachment.isVoiceMessage)
|
||||
);
|
||||
|
||||
const onRemoveAttachment = itemIndex => {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { ref, onMounted, onUnmounted } from 'vue';
|
||||
import WaveSurfer from 'wavesurfer.js';
|
||||
import RecordPlugin from 'wavesurfer.js/dist/plugins/record.js';
|
||||
import { format, intervalToDuration } from 'date-fns';
|
||||
import { convertAudio } from './utils/mp3ConversionUtils';
|
||||
import { convertAudio } from './utils/audioConversionUtils';
|
||||
|
||||
const props = defineProps({
|
||||
audioRecordFormat: {
|
||||
@@ -18,6 +18,7 @@ const emit = defineEmits([
|
||||
'finishRecord',
|
||||
'pause',
|
||||
'play',
|
||||
'recordError',
|
||||
]);
|
||||
|
||||
const waveformContainer = ref(null);
|
||||
@@ -26,6 +27,7 @@ const record = ref(null);
|
||||
const isRecording = ref(false);
|
||||
const isPlaying = ref(false);
|
||||
const hasRecording = ref(false);
|
||||
const recordedAudioUrl = ref(null);
|
||||
|
||||
const formatTimeProgress = time => {
|
||||
const duration = intervalToDuration({ start: 0, end: time });
|
||||
@@ -35,6 +37,28 @@ const formatTimeProgress = time => {
|
||||
);
|
||||
};
|
||||
|
||||
const AUDIO_EXTENSION_MAP = {
|
||||
'audio/ogg': 'ogg',
|
||||
'audio/mp3': 'mp3',
|
||||
'audio/mpeg': 'mp3',
|
||||
'audio/wav': 'wav',
|
||||
'audio/webm': 'webm',
|
||||
};
|
||||
|
||||
const getRecordPluginOptions = audioFormat => {
|
||||
const options = {
|
||||
scrollingWaveform: true,
|
||||
renderRecordedAudio: false,
|
||||
};
|
||||
if (
|
||||
audioFormat === 'audio/ogg' &&
|
||||
MediaRecorder.isTypeSupported('audio/ogg;codecs=opus')
|
||||
) {
|
||||
options.mimeType = 'audio/ogg;codecs=opus';
|
||||
}
|
||||
return options;
|
||||
};
|
||||
|
||||
const initWaveSurfer = () => {
|
||||
wavesurfer.value = WaveSurfer.create({
|
||||
container: waveformContainer.value,
|
||||
@@ -45,10 +69,7 @@ const initWaveSurfer = () => {
|
||||
barGap: 1,
|
||||
barRadius: 2,
|
||||
plugins: [
|
||||
RecordPlugin.create({
|
||||
scrollingWaveform: true,
|
||||
renderRecordedAudio: false,
|
||||
}),
|
||||
RecordPlugin.create(getRecordPluginOptions(props.audioRecordFormat)),
|
||||
],
|
||||
});
|
||||
|
||||
@@ -62,21 +83,34 @@ const initWaveSurfer = () => {
|
||||
});
|
||||
|
||||
record.value.on('record-end', async blob => {
|
||||
const audioUrl = URL.createObjectURL(blob);
|
||||
const audioBlob = await convertAudio(blob, props.audioRecordFormat);
|
||||
const fileName = `${getUuid()}.mp3`;
|
||||
const file = new File([audioBlob], fileName, {
|
||||
type: props.audioRecordFormat,
|
||||
});
|
||||
wavesurfer.value.load(audioUrl);
|
||||
emit('finishRecord', {
|
||||
name: file.name,
|
||||
type: file.type,
|
||||
size: file.size,
|
||||
file,
|
||||
});
|
||||
hasRecording.value = true;
|
||||
isRecording.value = false;
|
||||
try {
|
||||
const audioBlob = await convertAudio(blob, props.audioRecordFormat);
|
||||
// Use the converted blob's actual type, which may differ from the
|
||||
// requested format when the browser can't produce it (e.g. Safari falls
|
||||
// back to MP3 instead of OGG). This keeps the filename, content type, and
|
||||
// voice-note flag consistent with the real bytes.
|
||||
const audioType = audioBlob.type || props.audioRecordFormat;
|
||||
const ext = AUDIO_EXTENSION_MAP[audioType] || 'mp3';
|
||||
const fileName = `${getUuid()}.${ext}`;
|
||||
const file = new File([audioBlob], fileName, {
|
||||
type: audioType,
|
||||
});
|
||||
if (recordedAudioUrl.value) URL.revokeObjectURL(recordedAudioUrl.value);
|
||||
recordedAudioUrl.value = URL.createObjectURL(audioBlob);
|
||||
wavesurfer.value.load(recordedAudioUrl.value);
|
||||
emit('finishRecord', {
|
||||
name: file.name,
|
||||
type: file.type,
|
||||
size: file.size,
|
||||
file,
|
||||
});
|
||||
hasRecording.value = true;
|
||||
isRecording.value = false;
|
||||
} catch (error) {
|
||||
isRecording.value = false;
|
||||
hasRecording.value = false;
|
||||
emit('recordError', { error });
|
||||
}
|
||||
});
|
||||
|
||||
record.value.on('record-progress', time => {
|
||||
@@ -109,6 +143,10 @@ onMounted(() => {
|
||||
});
|
||||
|
||||
onUnmounted(() => {
|
||||
if (recordedAudioUrl.value) {
|
||||
URL.revokeObjectURL(recordedAudioUrl.value);
|
||||
recordedAudioUrl.value = null;
|
||||
}
|
||||
if (wavesurfer.value) {
|
||||
wavesurfer.value.destroy();
|
||||
}
|
||||
|
||||
+16
@@ -1,5 +1,7 @@
|
||||
import lamejs from '@breezystack/lamejs';
|
||||
|
||||
import { remuxWebmToOgg } from './webmOpusToOgg';
|
||||
|
||||
const writeString = (view, offset, string) => {
|
||||
// eslint-disable-next-line no-plusplus
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
@@ -139,6 +141,20 @@ export const convertAudio = async (inputBlob, outputFormat, bitrate = 128) => {
|
||||
audio = await convertToWav(inputBlob);
|
||||
} else if (outputFormat === 'audio/mp3') {
|
||||
audio = await convertToMp3(inputBlob, bitrate);
|
||||
} else if (outputFormat === 'audio/ogg') {
|
||||
const inputType = inputBlob.type.split(';')[0].trim();
|
||||
if (inputType === 'audio/webm' || inputType === 'video/webm') {
|
||||
audio = await remuxWebmToOgg(inputBlob);
|
||||
} else if (inputType === 'audio/ogg') {
|
||||
audio = inputBlob;
|
||||
} else {
|
||||
// Browsers that record neither WebM nor OGG (e.g. Safari records
|
||||
// audio/mp4) cannot produce OGG/Opus. Fall back to MP3 so the recording
|
||||
// still sends as a regular audio message instead of failing. The caller
|
||||
// keys the voice-note flag off the returned blob type, so an MP3 result
|
||||
// is never mislabeled as an OGG/Opus voice note.
|
||||
audio = await convertToMp3(inputBlob, bitrate);
|
||||
}
|
||||
} else {
|
||||
throw new Error('Unsupported output format');
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
/* eslint-disable no-bitwise */
|
||||
/**
|
||||
* WebM/Opus → OGG/Opus remuxer
|
||||
*
|
||||
* Chrome's MediaRecorder produces WebM containers even when
|
||||
* `audio/ogg;codecs=opus` is requested. WhatsApp Cloud API requires
|
||||
* proper OGG/Opus files for voice messages.
|
||||
*
|
||||
* This module extracts raw Opus packets from the WebM (EBML) container
|
||||
* and repackages them into a valid OGG bitstream. The audio data itself
|
||||
* is never re-encoded — only the container format changes.
|
||||
*
|
||||
* References:
|
||||
* EBML (container for WebM): RFC 8794 — https://www.rfc-editor.org/rfc/rfc8794
|
||||
* Matroska/WebM elements: https://www.matroska.org/technical/elements.html
|
||||
* OGG bitstream framing: RFC 3533 — https://www.rfc-editor.org/rfc/rfc3533
|
||||
* Opus codec: RFC 6716 — https://www.rfc-editor.org/rfc/rfc6716
|
||||
* Opus in OGG (OpusHead/Tags): RFC 7845 — https://www.rfc-editor.org/rfc/rfc7845
|
||||
*/
|
||||
|
||||
// ======================== EBML / WebM parser ========================
|
||||
|
||||
const EBML_IDS = {
|
||||
Segment: 0x18538067,
|
||||
SegmentInfo: 0x1549a966,
|
||||
Tracks: 0x1654ae6b,
|
||||
TrackEntry: 0xae,
|
||||
CodecPrivate: 0x63a2,
|
||||
Audio: 0xe1,
|
||||
SamplingFrequency: 0xb5,
|
||||
Channels: 0x9f,
|
||||
Cluster: 0x1f43b675,
|
||||
Timecode: 0xe7,
|
||||
SimpleBlock: 0xa3,
|
||||
BlockGroup: 0xa0,
|
||||
Block: 0xa1,
|
||||
};
|
||||
|
||||
const MASTER_ELEMENTS = new Set([
|
||||
0x1a45dfa3, // EBML header
|
||||
EBML_IDS.Segment,
|
||||
EBML_IDS.SegmentInfo,
|
||||
EBML_IDS.Tracks,
|
||||
EBML_IDS.TrackEntry,
|
||||
EBML_IDS.Audio,
|
||||
EBML_IDS.Cluster,
|
||||
EBML_IDS.BlockGroup,
|
||||
]);
|
||||
|
||||
/** Read an EBML variable-length integer (data size). */
|
||||
function readVint(data, pos) {
|
||||
if (pos >= data.length) return null;
|
||||
const first = data[pos];
|
||||
if (first === 0) return null;
|
||||
|
||||
let len = 1;
|
||||
let mask = 0x80;
|
||||
while (len <= 8 && !(first & mask)) {
|
||||
len += 1;
|
||||
mask >>= 1;
|
||||
}
|
||||
if (len > 8 || pos + len > data.length) return null;
|
||||
|
||||
let value = first & (mask - 1);
|
||||
for (let i = 1; i < len; i += 1) {
|
||||
value = value * 256 + data[pos + i];
|
||||
}
|
||||
return { value, length: len };
|
||||
}
|
||||
|
||||
/** Read an EBML element ID (leading marker bits are kept). */
|
||||
function readElementId(data, pos) {
|
||||
if (pos >= data.length) return null;
|
||||
const first = data[pos];
|
||||
if (first === 0) return null;
|
||||
|
||||
let len = 1;
|
||||
let mask = 0x80;
|
||||
while (len <= 4 && !(first & mask)) {
|
||||
len += 1;
|
||||
mask >>= 1;
|
||||
}
|
||||
if (len > 4 || pos + len > data.length) return null;
|
||||
|
||||
let id = first;
|
||||
for (let i = 1; i < len; i += 1) {
|
||||
id = id * 256 + data[pos + i];
|
||||
}
|
||||
return { id, length: len };
|
||||
}
|
||||
|
||||
function readUintBE(data, offset, length) {
|
||||
let v = 0;
|
||||
for (let i = 0; i < length; i += 1) v = v * 256 + data[offset + i];
|
||||
return v;
|
||||
}
|
||||
|
||||
function readFloatBE(data, offset, length) {
|
||||
if (length !== 4 && length !== 8) return NaN;
|
||||
const buf = new ArrayBuffer(length);
|
||||
const u8 = new Uint8Array(buf);
|
||||
for (let i = 0; i < length; i += 1) u8[i] = data[offset + i];
|
||||
const view = new DataView(buf);
|
||||
return length === 4 ? view.getFloat32(0) : view.getFloat64(0);
|
||||
}
|
||||
|
||||
/** Extract the raw Opus frame from a SimpleBlock / Block element. */
|
||||
function extractFrameFromBlock(data, offset, end) {
|
||||
const trackVint = readVint(data, offset);
|
||||
if (!trackVint) return null;
|
||||
let pos = offset + trackVint.length;
|
||||
|
||||
// int16 relative timecode (big-endian, signed) – skip
|
||||
pos += 2;
|
||||
// Flags byte – skip. Lacing (Xiph/EBML/fixed-size) is NOT supported;
|
||||
// this assumes single-frame blocks as produced by MediaRecorder.
|
||||
const flags = data[pos];
|
||||
const lacingBits = (flags >> 1) & 0x03;
|
||||
if (lacingBits !== 0) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
'webmOpusToOgg: laced SimpleBlock detected (unsupported), frame may be invalid'
|
||||
);
|
||||
}
|
||||
pos += 1;
|
||||
|
||||
if (pos >= end) return null;
|
||||
return data.slice(pos, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the EBML tree and collect metadata + Opus frames.
|
||||
* We only descend into master elements and only extract the fields we need.
|
||||
*/
|
||||
function parseWebM(buffer) {
|
||||
const data = new Uint8Array(buffer);
|
||||
const result = {
|
||||
channels: 1,
|
||||
sampleRate: 48000,
|
||||
codecPrivate: null,
|
||||
frames: [],
|
||||
};
|
||||
|
||||
function walk(start, end) {
|
||||
let pos = start;
|
||||
while (pos < end) {
|
||||
const idRes = readElementId(data, pos);
|
||||
if (!idRes) break;
|
||||
pos += idRes.length;
|
||||
|
||||
const sizeRes = readVint(data, pos);
|
||||
if (!sizeRes) break;
|
||||
pos += sizeRes.length;
|
||||
|
||||
// Handle "unknown size" (all-ones VINT) by treating it as the rest of the parent
|
||||
// Use Math.pow instead of bit-shift to avoid 32-bit overflow for 5+ byte VINTs
|
||||
const maxVint = 2 ** (7 * sizeRes.length) - 1;
|
||||
const elEnd =
|
||||
sizeRes.value === maxVint ? end : Math.min(pos + sizeRes.value, end);
|
||||
|
||||
if (MASTER_ELEMENTS.has(idRes.id)) {
|
||||
walk(pos, elEnd);
|
||||
} else {
|
||||
switch (idRes.id) {
|
||||
case EBML_IDS.Channels:
|
||||
result.channels = readUintBE(data, pos, sizeRes.value);
|
||||
break;
|
||||
case EBML_IDS.SamplingFrequency:
|
||||
result.sampleRate = readFloatBE(data, pos, sizeRes.value);
|
||||
break;
|
||||
case EBML_IDS.CodecPrivate:
|
||||
result.codecPrivate = data.slice(pos, elEnd);
|
||||
break;
|
||||
case EBML_IDS.SimpleBlock:
|
||||
case EBML_IDS.Block: {
|
||||
const frame = extractFrameFromBlock(data, pos, elEnd);
|
||||
if (frame && frame.length > 0) result.frames.push(frame);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
pos = elEnd;
|
||||
}
|
||||
}
|
||||
|
||||
walk(0, data.length);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ======================== OGG writer ========================
|
||||
|
||||
/** OGG CRC-32 table (polynomial 0x04C11DB7). */
|
||||
const CRC_TABLE = (() => {
|
||||
const t = new Uint32Array(256);
|
||||
for (let i = 0; i < 256; i += 1) {
|
||||
let c = i << 24;
|
||||
for (let j = 0; j < 8; j += 1) {
|
||||
c = ((c << 1) ^ (c & 0x80000000 ? 0x04c11db7 : 0)) >>> 0;
|
||||
}
|
||||
t[i] = c;
|
||||
}
|
||||
return t;
|
||||
})();
|
||||
|
||||
function oggCrc32(bytes) {
|
||||
let crc = 0;
|
||||
for (let i = 0; i < bytes.length; i += 1) {
|
||||
crc = (CRC_TABLE[((crc >>> 24) ^ bytes[i]) & 0xff] ^ (crc << 8)) >>> 0;
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build one OGG page.
|
||||
*
|
||||
* @param {number} headerType 0x02 = BOS, 0x04 = EOS, 0x00 = normal
|
||||
* @param {number} granulePosition 48 kHz sample count
|
||||
* @param {number} serialNumber logical stream id
|
||||
* @param {number} pageSeq page sequence counter
|
||||
* @param {Uint8Array[]} packets one or more complete Opus packets
|
||||
*/
|
||||
function createOggPage(
|
||||
headerType,
|
||||
granulePosition,
|
||||
serialNumber,
|
||||
pageSeq,
|
||||
packets
|
||||
) {
|
||||
// Build the lacing / segment table
|
||||
const segTable = [];
|
||||
let dataLen = 0;
|
||||
packets.forEach(pkt => {
|
||||
let rem = pkt.length;
|
||||
while (rem >= 255) {
|
||||
segTable.push(255);
|
||||
rem -= 255;
|
||||
}
|
||||
segTable.push(rem); // final segment (0 when pkt.length is a multiple of 255)
|
||||
dataLen += pkt.length;
|
||||
});
|
||||
|
||||
const hdrLen = 27 + segTable.length;
|
||||
const page = new Uint8Array(hdrLen + dataLen);
|
||||
const dv = new DataView(page.buffer);
|
||||
|
||||
// Capture pattern
|
||||
page.set([0x4f, 0x67, 0x67, 0x53]); // "OggS"
|
||||
page[4] = 0; // version
|
||||
page[5] = headerType;
|
||||
|
||||
// Granule position (int64 LE)
|
||||
dv.setUint32(6, granulePosition & 0xffffffff, true);
|
||||
dv.setUint32(
|
||||
10,
|
||||
Math.floor(granulePosition / 0x100000000) & 0xffffffff,
|
||||
true
|
||||
);
|
||||
|
||||
dv.setUint32(14, serialNumber, true); // serial
|
||||
dv.setUint32(18, pageSeq, true); // page sequence
|
||||
dv.setUint32(22, 0, true); // CRC placeholder
|
||||
|
||||
page[26] = segTable.length;
|
||||
for (let i = 0; i < segTable.length; i += 1) page[27 + i] = segTable[i];
|
||||
|
||||
let off = hdrLen;
|
||||
packets.forEach(pkt => {
|
||||
page.set(pkt, off);
|
||||
off += pkt.length;
|
||||
});
|
||||
|
||||
// Fill in the CRC
|
||||
dv.setUint32(22, oggCrc32(page), true);
|
||||
return page;
|
||||
}
|
||||
|
||||
// ======================== Opus helpers ========================
|
||||
|
||||
/** Lookup table: frame duration in ms for each Opus TOC config index (0-31). */
|
||||
const OPUS_FRAME_MS = [
|
||||
10,
|
||||
20,
|
||||
40,
|
||||
60, // 0-3 SILK NB
|
||||
10,
|
||||
20,
|
||||
40,
|
||||
60, // 4-7 SILK MB
|
||||
10,
|
||||
20,
|
||||
40,
|
||||
60, // 8-11 SILK WB
|
||||
10,
|
||||
20, // 12-13 Hybrid SWB
|
||||
10,
|
||||
20, // 14-15 Hybrid FB
|
||||
2.5,
|
||||
5,
|
||||
10,
|
||||
20, // 16-19 CELT NB
|
||||
2.5,
|
||||
5,
|
||||
10,
|
||||
20, // 20-23 CELT WB
|
||||
2.5,
|
||||
5,
|
||||
10,
|
||||
20, // 24-27 CELT SWB
|
||||
2.5,
|
||||
5,
|
||||
10,
|
||||
20, // 28-31 CELT FB
|
||||
];
|
||||
|
||||
/** Return the total number of 48 kHz PCM samples represented by an Opus packet. */
|
||||
function opusPacketSamples(pkt) {
|
||||
if (!pkt || pkt.length === 0) return 960; // default 20 ms
|
||||
const toc = pkt[0];
|
||||
const config = (toc >> 3) & 0x1f;
|
||||
const code = toc & 0x03;
|
||||
|
||||
const samplesPerFrame = ((OPUS_FRAME_MS[config] || 20) * 48000) / 1000;
|
||||
let frameCount;
|
||||
if (code <= 1) frameCount = code + 1;
|
||||
else if (code === 2) frameCount = 2;
|
||||
else frameCount = pkt.length >= 2 ? pkt[1] & 0x3f : 1;
|
||||
|
||||
return samplesPerFrame * frameCount;
|
||||
}
|
||||
|
||||
function buildOpusHead(channels, sampleRate, preSkip) {
|
||||
const buf = new Uint8Array(19);
|
||||
const dv = new DataView(buf.buffer);
|
||||
buf.set(new TextEncoder().encode('OpusHead'));
|
||||
buf[8] = 1; // version
|
||||
buf[9] = channels;
|
||||
dv.setUint16(10, preSkip, true);
|
||||
dv.setUint32(12, sampleRate, true);
|
||||
dv.setInt16(16, 0, true); // output gain
|
||||
buf[18] = 0; // channel mapping family
|
||||
return buf;
|
||||
}
|
||||
|
||||
function buildOpusTags() {
|
||||
const vendor = new TextEncoder().encode('chatwoot');
|
||||
const buf = new Uint8Array(8 + 4 + vendor.length + 4);
|
||||
const dv = new DataView(buf.buffer);
|
||||
buf.set(new TextEncoder().encode('OpusTags'));
|
||||
dv.setUint32(8, vendor.length, true);
|
||||
buf.set(vendor, 12);
|
||||
dv.setUint32(12 + vendor.length, 0, true); // 0 user comments
|
||||
return buf;
|
||||
}
|
||||
|
||||
// ======================== Public API ========================
|
||||
|
||||
const MAX_FRAMES_PER_PAGE = 50; // ~1 s at 20 ms/frame
|
||||
const MAX_SEGMENTS_PER_PAGE = 255;
|
||||
|
||||
/**
|
||||
* Remux a WebM/Opus blob into an OGG/Opus blob.
|
||||
* If the input is already OGG (starts with "OggS"), it is returned as-is.
|
||||
*
|
||||
* @param {Blob} webmBlob
|
||||
* @returns {Promise<Blob>} OGG/Opus blob
|
||||
*/
|
||||
export async function remuxWebmToOgg(webmBlob) {
|
||||
const buffer = await webmBlob.arrayBuffer();
|
||||
const bytes = new Uint8Array(buffer);
|
||||
|
||||
// Already OGG? Return unchanged.
|
||||
if (
|
||||
bytes.length >= 4 &&
|
||||
bytes[0] === 0x4f &&
|
||||
bytes[1] === 0x67 &&
|
||||
bytes[2] === 0x67 &&
|
||||
bytes[3] === 0x53
|
||||
) {
|
||||
return webmBlob;
|
||||
}
|
||||
|
||||
const { channels, sampleRate, codecPrivate, frames } = parseWebM(buffer);
|
||||
if (frames.length === 0) {
|
||||
throw new Error('No Opus frames found in WebM input');
|
||||
}
|
||||
|
||||
// Extract pre-skip from the WebM CodecPrivate (which IS the OpusHead)
|
||||
let preSkip = 312;
|
||||
if (codecPrivate && codecPrivate.length >= 12) {
|
||||
const magic = new TextDecoder().decode(codecPrivate.slice(0, 8));
|
||||
if (magic === 'OpusHead') {
|
||||
preSkip = new DataView(
|
||||
codecPrivate.buffer,
|
||||
codecPrivate.byteOffset,
|
||||
codecPrivate.length
|
||||
).getUint16(10, true);
|
||||
}
|
||||
}
|
||||
|
||||
const serial = (Math.random() * 0x100000000) >>> 0;
|
||||
let pageSeq = 0;
|
||||
const pages = [];
|
||||
|
||||
// Page 0 – OpusHead (BOS)
|
||||
pages.push(
|
||||
createOggPage(0x02, 0, serial, pageSeq, [
|
||||
buildOpusHead(channels, sampleRate, preSkip),
|
||||
])
|
||||
);
|
||||
pageSeq += 1;
|
||||
|
||||
// Page 1 – OpusTags
|
||||
pages.push(createOggPage(0x00, 0, serial, pageSeq, [buildOpusTags()]));
|
||||
pageSeq += 1;
|
||||
|
||||
// Audio pages
|
||||
let granule = 0;
|
||||
let idx = 0;
|
||||
|
||||
while (idx < frames.length) {
|
||||
const packets = [];
|
||||
let segs = 0;
|
||||
|
||||
while (idx < frames.length && packets.length < MAX_FRAMES_PER_PAGE) {
|
||||
const pkt = frames[idx];
|
||||
// createOggPage always appends a terminating lacing value, so a packet
|
||||
// spans floor(len/255)+1 segments (including the extra 0 when len is an
|
||||
// exact multiple of 255). Math.ceil would undercount those cases.
|
||||
const pktSegs = Math.floor(pkt.length / 255) + 1;
|
||||
if (segs + pktSegs > MAX_SEGMENTS_PER_PAGE && packets.length > 0) break;
|
||||
|
||||
packets.push(pkt);
|
||||
segs += pktSegs;
|
||||
granule += opusPacketSamples(pkt);
|
||||
idx += 1;
|
||||
}
|
||||
|
||||
const isLast = idx >= frames.length;
|
||||
pages.push(
|
||||
createOggPage(isLast ? 0x04 : 0x00, granule, serial, pageSeq, packets)
|
||||
);
|
||||
pageSeq += 1;
|
||||
}
|
||||
|
||||
// Concatenate pages into a single buffer
|
||||
const total = pages.reduce((s, p) => s + p.length, 0);
|
||||
const out = new Uint8Array(total);
|
||||
let off = 0;
|
||||
pages.forEach(p => {
|
||||
out.set(p, off);
|
||||
off += p.length;
|
||||
});
|
||||
|
||||
return new Blob([out], { type: 'audio/ogg' });
|
||||
}
|
||||
@@ -375,7 +375,10 @@ export default {
|
||||
return `draft-${this.conversationIdByRoute}-${this.replyType}`;
|
||||
},
|
||||
audioRecordFormat() {
|
||||
if (this.isAWhatsAppChannel || this.isATelegramChannel) {
|
||||
if (this.isAWhatsAppChannel) {
|
||||
return AUDIO_FORMATS.OGG;
|
||||
}
|
||||
if (this.isATelegramChannel) {
|
||||
return AUDIO_FORMATS.MP3;
|
||||
}
|
||||
if (this.isAPIInbox) {
|
||||
@@ -1008,14 +1011,18 @@ export default {
|
||||
onFinishRecorder(file) {
|
||||
this.recordingAudioState = 'stopped';
|
||||
this.hasRecordedAudio = true;
|
||||
// Added a new key isRecordedAudio to the file to find it's and recorded audio
|
||||
// Added a new key isVoiceMessage to the file to identify recorded audio
|
||||
// Because to filter and show only non recorded audio and other attachments
|
||||
const autoRecordedFile = {
|
||||
...file,
|
||||
isRecordedAudio: true,
|
||||
isVoiceMessage: true,
|
||||
};
|
||||
return file && this.onFileUpload(autoRecordedFile);
|
||||
},
|
||||
onRecordError() {
|
||||
this.toggleAudioRecorder();
|
||||
useAlert(this.$t('CONVERSATION.REPLYBOX.AUDIO_CONVERSION_FAILED'));
|
||||
},
|
||||
toggleTyping(status) {
|
||||
const conversationId = this.currentChat.id;
|
||||
const isPrivate = this.isPrivate;
|
||||
@@ -1042,7 +1049,7 @@ export default {
|
||||
isPrivate: this.isPrivate,
|
||||
thumb: reader.result,
|
||||
blobSignedId: blob ? blob.signed_id : undefined,
|
||||
isRecordedAudio: file?.isRecordedAudio || false,
|
||||
isVoiceMessage: file?.isVoiceMessage || false,
|
||||
});
|
||||
};
|
||||
},
|
||||
@@ -1078,6 +1085,7 @@ export default {
|
||||
private: false,
|
||||
message: caption,
|
||||
sender: this.sender,
|
||||
isVoiceMessage: attachment.isVoiceMessage || false,
|
||||
};
|
||||
|
||||
attachmentPayload = this.setReplyToInPayload(attachmentPayload);
|
||||
@@ -1127,6 +1135,9 @@ export default {
|
||||
this.attachedFiles.forEach(attachment => {
|
||||
if (this.globalConfig.directUploadsEnabled) {
|
||||
messagePayload.files.push(attachment.blobSignedId);
|
||||
if (attachment.isVoiceMessage) {
|
||||
messagePayload.isVoiceMessage = true;
|
||||
}
|
||||
} else {
|
||||
messagePayload.files.push(attachment.resource.file);
|
||||
}
|
||||
@@ -1215,7 +1226,7 @@ export default {
|
||||
this.hasRecordedAudio = false;
|
||||
// Only clear the recorded audio when we click toggle button.
|
||||
this.attachedFiles = this.attachedFiles.filter(
|
||||
file => !file?.isRecordedAudio
|
||||
file => !file?.isVoiceMessage
|
||||
);
|
||||
},
|
||||
toggleEditorSize() {
|
||||
@@ -1293,6 +1304,7 @@ export default {
|
||||
:audio-record-format="audioRecordFormat"
|
||||
@recorder-progress-changed="onRecordProgressChanged"
|
||||
@finish-record="onFinishRecorder"
|
||||
@record-error="onRecordError"
|
||||
@play="recordingAudioState = 'playing'"
|
||||
@pause="recordingAudioState = 'paused'"
|
||||
/>
|
||||
|
||||
@@ -231,6 +231,7 @@
|
||||
"TIP_AUDIORECORDER_ICON": "Record audio",
|
||||
"TIP_AUDIORECORDER_PERMISSION": "Allow access to audio",
|
||||
"TIP_AUDIORECORDER_ERROR": "Could not open the audio",
|
||||
"AUDIO_CONVERSION_FAILED": "Audio conversion failed. Please try again.",
|
||||
"DRAG_DROP": "Drag and drop here to attach",
|
||||
"START_AUDIO_RECORDING": "Start audio recording",
|
||||
"STOP_AUDIO_RECORDING": "Stop audio recording",
|
||||
|
||||
@@ -90,8 +90,8 @@ class Whatsapp::Providers::WhatsappCloudService < Whatsapp::Providers::BaseServi
|
||||
end
|
||||
|
||||
# TODO: See if we can unify the API versions and for both paths and make it consistent with out facebook app API versions
|
||||
def phone_id_path
|
||||
"#{api_base_path}/v13.0/#{whatsapp_channel.provider_config['phone_number_id']}"
|
||||
def phone_id_path(version = 'v13.0')
|
||||
"#{api_base_path}/#{version}/#{whatsapp_channel.provider_config['phone_number_id']}"
|
||||
end
|
||||
|
||||
def business_account_path
|
||||
@@ -116,14 +116,11 @@ class Whatsapp::Providers::WhatsappCloudService < Whatsapp::Providers::BaseServi
|
||||
|
||||
def send_attachment_message(phone_number, message)
|
||||
attachment = message.attachments.first
|
||||
normalize_opus_content_type(attachment)
|
||||
type = %w[image audio video].include?(attachment.file_type) ? attachment.file_type : 'document'
|
||||
type_content = {
|
||||
'link': attachment.download_url
|
||||
}
|
||||
type_content['caption'] = message.outgoing_content unless %w[audio sticker].include?(type)
|
||||
type_content['filename'] = attachment.file.filename if type == 'document'
|
||||
type_content = build_attachment_content(type, attachment, message)
|
||||
response = HTTParty.post(
|
||||
"#{phone_id_path}/messages",
|
||||
"#{phone_id_path('v24.0')}/messages",
|
||||
headers: api_headers,
|
||||
body: {
|
||||
:messaging_product => 'whatsapp',
|
||||
@@ -142,6 +139,33 @@ class Whatsapp::Providers::WhatsappCloudService < Whatsapp::Providers::BaseServi
|
||||
response.parsed_response&.dig('error', 'message')
|
||||
end
|
||||
|
||||
def voice_message?(type, attachment)
|
||||
type == 'audio' && attachment.meta&.dig('is_voice_message') && attachment.file.content_type == 'audio/ogg'
|
||||
end
|
||||
|
||||
# Marcel gem may re-detect OGG/Opus files as audio/opus after ActiveStorage
|
||||
# blob attachment, but WhatsApp Cloud API requires audio/ogg content type
|
||||
# for voice messages. Normalize so the download URL serves the correct
|
||||
# Content-Type header. No-op when the frontend already uploads as audio/ogg.
|
||||
def normalize_opus_content_type(attachment)
|
||||
return unless attachment.file.attached?
|
||||
|
||||
blob = attachment.file.blob
|
||||
return unless blob.content_type == 'audio/opus'
|
||||
|
||||
return if blob.update(content_type: 'audio/ogg')
|
||||
|
||||
Rails.logger.error("Failed to normalize blob #{blob.id} content_type from audio/opus to audio/ogg")
|
||||
end
|
||||
|
||||
def build_attachment_content(type, attachment, message)
|
||||
type_content = { 'link' => attachment.download_url }
|
||||
type_content['caption'] = message.outgoing_content unless %w[audio sticker].include?(type)
|
||||
type_content['filename'] = attachment.file.filename if type == 'document'
|
||||
type_content['voice'] = true if voice_message?(type, attachment)
|
||||
type_content
|
||||
end
|
||||
|
||||
def template_body_parameters(template_info)
|
||||
template_body = {
|
||||
name: template_info[:name],
|
||||
|
||||
@@ -149,6 +149,35 @@ describe Messages::MessageBuilder do
|
||||
end
|
||||
end
|
||||
|
||||
context 'when is_voice_message is true' do
|
||||
let(:params) do
|
||||
ActionController::Parameters.new({
|
||||
content: 'test',
|
||||
attachments: [Rack::Test::UploadedFile.new('spec/assets/sample.ogg', 'audio/ogg')],
|
||||
is_voice_message: true
|
||||
})
|
||||
end
|
||||
|
||||
it 'sets is_voice_message in attachment meta' do
|
||||
message = message_builder
|
||||
expect(message.attachments.first.meta).to include('is_voice_message' => true)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when is_voice_message is not provided' do
|
||||
let(:params) do
|
||||
ActionController::Parameters.new({
|
||||
content: 'test',
|
||||
attachments: [Rack::Test::UploadedFile.new('spec/assets/avatar.png', 'image/png')]
|
||||
})
|
||||
end
|
||||
|
||||
it 'does not set is_voice_message in attachment meta' do
|
||||
message = message_builder
|
||||
expect(message.attachments.first.meta).not_to include('is_voice_message')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when email channel messages' do
|
||||
let!(:channel_email) { create(:channel_email, account: account) }
|
||||
let(:inbox_member) { create(:inbox_member, inbox: channel_email.inbox) }
|
||||
|
||||
@@ -60,7 +60,7 @@ describe Whatsapp::Providers::WhatsappCloudService do
|
||||
attachment = message.attachments.new(account_id: message.account_id, file_type: :image)
|
||||
attachment.file.attach(io: Rails.root.join('spec/assets/avatar.png').open, filename: 'avatar.png', content_type: 'image/png')
|
||||
|
||||
stub_request(:post, 'https://graph.facebook.com/v13.0/123456789/messages')
|
||||
stub_request(:post, 'https://graph.facebook.com/v24.0/123456789/messages')
|
||||
.with(
|
||||
body: hash_including({
|
||||
messaging_product: 'whatsapp',
|
||||
@@ -79,7 +79,7 @@ describe Whatsapp::Providers::WhatsappCloudService do
|
||||
|
||||
# ref: https://github.com/bblimke/webmock/issues/900
|
||||
# reason for Webmock::API.hash_including
|
||||
stub_request(:post, 'https://graph.facebook.com/v13.0/123456789/messages')
|
||||
stub_request(:post, 'https://graph.facebook.com/v24.0/123456789/messages')
|
||||
.with(
|
||||
body: hash_including({
|
||||
messaging_product: 'whatsapp',
|
||||
@@ -91,6 +91,41 @@ describe Whatsapp::Providers::WhatsappCloudService do
|
||||
.to_return(status: 200, body: whatsapp_response.to_json, headers: response_headers)
|
||||
expect(service.send_message('+123456789', message)).to eq 'message_id'
|
||||
end
|
||||
|
||||
it 'calls message endpoints for audio voice message with voice flag' do
|
||||
attachment = message.attachments.new(account_id: message.account_id, file_type: :audio, meta: { 'is_voice_message' => true })
|
||||
attachment.file.attach(io: Rails.root.join('spec/assets/sample.ogg').open, filename: 'voice.ogg', content_type: 'audio/ogg')
|
||||
|
||||
stub_request(:post, 'https://graph.facebook.com/v24.0/123456789/messages')
|
||||
.with(
|
||||
body: hash_including({
|
||||
messaging_product: 'whatsapp',
|
||||
to: '+123456789',
|
||||
type: 'audio',
|
||||
audio: WebMock::API.hash_including({ link: anything, voice: true })
|
||||
})
|
||||
)
|
||||
.to_return(status: 200, body: whatsapp_response.to_json, headers: response_headers)
|
||||
expect(service.send_message('+123456789', message)).to eq 'message_id'
|
||||
end
|
||||
|
||||
it 'calls message endpoints for regular audio attachment without voice flag' do
|
||||
attachment = message.attachments.new(account_id: message.account_id, file_type: :audio)
|
||||
attachment.file.attach(io: Rails.root.join('spec/assets/sample.ogg').open, filename: 'audio.ogg', content_type: 'audio/ogg')
|
||||
|
||||
stub_request(:post, 'https://graph.facebook.com/v24.0/123456789/messages')
|
||||
.with(
|
||||
body: hash_including({
|
||||
messaging_product: 'whatsapp',
|
||||
to: '+123456789',
|
||||
type: 'audio'
|
||||
})
|
||||
)
|
||||
.to_return(status: 200, body: whatsapp_response.to_json, headers: response_headers)
|
||||
|
||||
result = service.send_message('+123456789', message)
|
||||
expect(result).to eq 'message_id'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user