From 31287976b95d052ccfa773fc423c2a502fa55b60 Mon Sep 17 00:00:00 2001 From: iamsivin Date: Wed, 29 Apr 2026 15:44:49 +0530 Subject: [PATCH] fix: show/hide quoted text on plain-text emails --- .../dashboard/helper/emailQuoteExtractor.js | 274 +++++++++++------- .../helper/specs/emailQuoteExtractor.spec.js | 232 +++++++++++++++ 2 files changed, 394 insertions(+), 112 deletions(-) diff --git a/app/javascript/dashboard/helper/emailQuoteExtractor.js b/app/javascript/dashboard/helper/emailQuoteExtractor.js index f29d48cca..6854532fd 100644 --- a/app/javascript/dashboard/helper/emailQuoteExtractor.js +++ b/app/javascript/dashboard/helper/emailQuoteExtractor.js @@ -1,6 +1,8 @@ import DOMPurify from 'dompurify'; -// Quote detection strategies +// Wrapper classes mainstream mail clients emit around the quoted reply. +// Removed depth-agnostically (Gmail, Outlook, Yahoo, Thunderbird, Apple…). +// Purely additive over develop — nothing here can match a non-quoted body. const QUOTE_INDICATORS = [ '.gmail_quote_container', '.gmail_quote', @@ -10,149 +12,197 @@ const QUOTE_INDICATORS = [ '.quote', '[class*="quote"]', '[class*="Quote"]', + '.moz-cite-prefix', // Thunderbird attribution + '.yahoo_quoted', // Yahoo Mail wrapper + '#divRplyFwdMsg', // Outlook web/desktop reply/forward header ]; -const BLOCKQUOTE_FALLBACK_SELECTOR = 'blockquote'; - -// Regex patterns for quote identification -const QUOTE_PATTERNS = [ +// Inline header / attribution patterns. A text node containing one of these +// causes its block-ancestor to be removed (matches develop behaviour exactly). +const HEADER_PATTERNS = [ /On .* wrote:/i, /-----Original Message-----/i, /Sent: /i, /From: /i, ]; +// "Hard" markers: the marker line plus every following sibling of its +// block-ancestor are removed, so the quoted body itself (not just the +// attribution line) gets stripped on forwarded / reply-with-original messages. +const HARD_HEADER_PATTERNS = [ + /-----Original Message-----/i, + /-{2,}\s*Forwarded message\s*-{2,}/i, + /Begin forwarded message:/i, +]; + +const BLOCK_TAGS = new Set(['DIV', 'P', 'BLOCKQUOTE', 'SECTION']); + export class EmailQuoteExtractor { - /** - * Remove quotes from email HTML and return cleaned HTML - * @param {string} htmlContent - Full HTML content of the email - * @returns {string} HTML content with quotes removed - */ - static extractQuotes(htmlContent) { - // Create a temporary DOM element to parse HTML - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = DOMPurify.sanitize(htmlContent); + // ---------- public API ---------- - // Remove elements matching class selectors + /** Strip the quoted-reply tail from `html` and return the cleaned HTML. */ + static extractQuotes(html) { + const root = this.parse(html); + this.removeIndicatorElements(root); + this.removeHardHeaderTails(root); + this.removeTrailingBlockquote(root); + this.removeHeaderBlocks(root); + this.removePlainTextTail(root); + return root.innerHTML; + } + + /** True iff any quote-detection strategy finds material to strip. */ + static hasQuotes(html) { + const root = this.parse(html); + return ( + this.hasIndicatorElement(root) || + this.hasTrailingBlockquote(root) || + this.findHardHeaderBlocks(root).length > 0 || + this.findHeaderBlocks(root).length > 0 || + this.findPlainTextTailStart(root) !== -1 + ); + } + + // ---------- shared parser ---------- + + static parse(html) { + const root = document.createElement('div'); + root.innerHTML = DOMPurify.sanitize(html); + return root; + } + + // ---------- 1. Indicator classes (depth-agnostic) ---------- + + static removeIndicatorElements(root) { QUOTE_INDICATORS.forEach(selector => { - tempDiv.querySelectorAll(selector).forEach(el => { - el.remove(); - }); + root.querySelectorAll(selector).forEach(el => el.remove()); }); - - this.removeTrailingBlockquote(tempDiv); - - // Remove text-based quotes - const textNodeQuotes = this.findTextNodeQuotes(tempDiv); - textNodeQuotes.forEach(el => { - el.remove(); - }); - - return tempDiv.innerHTML; } - /** - * Check if HTML content contains any quotes - * @param {string} htmlContent - Full HTML content of the email - * @returns {boolean} True if quotes are detected, false otherwise - */ - static hasQuotes(htmlContent) { - const tempDiv = document.createElement('div'); - tempDiv.innerHTML = DOMPurify.sanitize(htmlContent); + static hasIndicatorElement(root) { + return QUOTE_INDICATORS.some(selector => root.querySelector(selector)); + } - // Check for class-based quotes - // eslint-disable-next-line no-restricted-syntax - for (const selector of QUOTE_INDICATORS) { - if (tempDiv.querySelector(selector)) { - return true; + // ---------- 2. Hard header tails ---------- + // For every block containing a hard-header text, remove the block AND every + // following sibling within its parent. This strips the quoted body, not just + // the attribution line. + + static removeHardHeaderTails(root) { + this.findHardHeaderBlocks(root).forEach(block => { + let cursor = block; + while (cursor) { + const next = cursor.nextSibling; + cursor.remove(); + cursor = next; } - } - - if (this.findTrailingBlockquote(tempDiv)) { - return true; - } - - // Check for text-based quotes - const textNodeQuotes = this.findTextNodeQuotes(tempDiv); - return textNodeQuotes.length > 0; + }); } - /** - * Find text nodes that match quote patterns - * @param {Element} rootElement - Root element to search - * @returns {Element[]} Array of parent block elements containing quote-like text - */ - static findTextNodeQuotes(rootElement) { - const quoteBlocks = []; - const treeWalker = document.createTreeWalker( - rootElement, + static findHardHeaderBlocks(root) { + return this.findBlocksContainingText(root, HARD_HEADER_PATTERNS); + } + + // ---------- 3. Trailing
---------- + + static removeTrailingBlockquote(root) { + const last = root.lastElementChild; + if (last?.matches?.('blockquote')) last.remove(); + } + + static hasTrailingBlockquote(root) { + return root.lastElementChild?.matches?.('blockquote') ?? false; + } + + // ---------- 4. Header blocks (deep, develop-compatible) ---------- + // For every text node matching a header pattern, remove its block-ancestor. + // This is the develop-branch behaviour preserved verbatim. + + static removeHeaderBlocks(root) { + this.findHeaderBlocks(root).forEach(el => el.remove()); + } + + static findHeaderBlocks(root) { + return this.findBlocksContainingText(root, HEADER_PATTERNS); + } + + // ---------- 5. Plain-text RFC `>` tail ---------- + // For text/plain emails (after sanitizeTextForRender converts \n →
), + // find the earliest top-level text node whose visible lines all begin with + // `>` and strip from there to the end (collapsing leading
/whitespace + // separators back into the tail). + + static removePlainTextTail(root) { + const start = this.findPlainTextTailStart(root); + if (start === -1) return; + const nodes = Array.from(root.childNodes); + for (let i = start; i < nodes.length; i += 1) nodes[i].remove(); + } + + static findPlainTextTailStart(root) { + const children = Array.from(root.childNodes); + for (let i = 0; i < children.length; i += 1) { + if (!this.isQuotePrefixedTextNode(children[i])) continue; // eslint-disable-line no-continue + let start = i; + while (start > 0 && this.isNeutralNode(children[start - 1])) { + start -= 1; + } + return start; + } + return -1; + } + + static isQuotePrefixedTextNode(node) { + if (node.nodeType !== Node.TEXT_NODE) return false; + const text = node.textContent; + if (!text.trim()) return false; + return text + .split('\n') + .filter(line => line.trim() !== '') + .every(line => line.trim().startsWith('>')); + } + + static isNeutralNode(node) { + if (node.nodeType === Node.TEXT_NODE) { + return node.textContent.trim() === ''; + } + if (node.nodeType === Node.ELEMENT_NODE) { + return node.tagName === 'BR'; + } + return false; + } + + // ---------- shared text-walker primitive ---------- + + static findBlocksContainingText(root, patterns) { + const blocks = []; + const walker = document.createTreeWalker( + root, NodeFilter.SHOW_TEXT, null, false ); - for ( - let currentNode = treeWalker.nextNode(); - currentNode !== null; - currentNode = treeWalker.nextNode() + let node = walker.nextNode(); + node !== null; + node = walker.nextNode() ) { - const isQuoteLike = QUOTE_PATTERNS.some(pattern => - pattern.test(currentNode.textContent) - ); - - if (isQuoteLike) { - const parentBlock = this.findParentBlock(currentNode); - if (parentBlock && !quoteBlocks.includes(parentBlock)) { - quoteBlocks.push(parentBlock); - } + const text = node.textContent; + if (!patterns.some(p => p.test(text))) continue; // eslint-disable-line no-continue + const block = this.findBlockAncestor(node); + if (block && block !== root && !blocks.includes(block)) { + blocks.push(block); } } - - return quoteBlocks; + return blocks; } - /** - * Find the closest block-level parent element by recursively traversing up the DOM tree. - * This method searches for common block-level elements like DIV, P, BLOCKQUOTE, and SECTION - * that contain the text node. It's used to identify and remove entire block-level elements - * that contain quote-like text, rather than just removing the text node itself. This ensures - * proper structural removal of quoted content while maintaining HTML integrity. - * @param {Node} node - Starting node to find parent - * @returns {Element|null} Block-level parent element - */ - static findParentBlock(node) { - const blockElements = ['DIV', 'P', 'BLOCKQUOTE', 'SECTION']; + static findBlockAncestor(node) { let current = node.parentElement; - while (current) { - if (blockElements.includes(current.tagName)) { - return current; - } + if (BLOCK_TAGS.has(current.tagName)) return current; current = current.parentElement; } - - return null; - } - - /** - * Remove fallback blockquote if it is the last top-level element. - * @param {Element} rootElement - Root element containing the HTML - */ - static removeTrailingBlockquote(rootElement) { - const trailingBlockquote = this.findTrailingBlockquote(rootElement); - trailingBlockquote?.remove(); - } - - /** - * Locate a fallback blockquote that is the last top-level element. - * @param {Element} rootElement - Root element containing the HTML - * @returns {Element|null} The trailing blockquote element if present - */ - static findTrailingBlockquote(rootElement) { - const lastElement = rootElement.lastElementChild; - if (lastElement?.matches?.(BLOCKQUOTE_FALLBACK_SELECTOR)) { - return lastElement; - } return null; } } diff --git a/app/javascript/dashboard/helper/specs/emailQuoteExtractor.spec.js b/app/javascript/dashboard/helper/specs/emailQuoteExtractor.spec.js index 20b50581b..f3d338da9 100644 --- a/app/javascript/dashboard/helper/specs/emailQuoteExtractor.spec.js +++ b/app/javascript/dashboard/helper/specs/emailQuoteExtractor.spec.js @@ -43,7 +43,239 @@ const EMAIL_WITH_FOLLOW_UP_CONTENT = `

Regards,

`; +// Real-world mixed body: an HTML reply, then RFC-style `>`-prefixed plain-text +// quote lines, and finally the original email wrapped in a Gmail blockquote. +// The current branchy implementation picks ONE strategy and misses the other. +const MIXED_PLAINTEXT_AND_HTML_QUOTE = `

My HTML reply

+

Thanks,

+

Sivin

+
+> On Mon, Apr 6, 2026, Shruthi wrote:
+> Inline plain-text quote line
+> that I made
+
+

The original HTML quoted email

+
`; + +// Real-world body: Gmail web reply (matches the structure the Chatwoot +// fixture in components-next/message/fixtures/emailConversation.js produces). +const GMAIL_REAL_WORLD_REPLY = `

Dear Sam,

Thank you for the quotation. Could you share images?

Best,
Alex


On Wed, 4 Dec 2024 at 17:15, Sam from CottonMart <sam@cottonmart.test> wrote:

Dear Alex,

Thank you for your inquiry.

Best regards,
Sam

`; + +// Outlook web/desktop: header div carries id="divRplyFwdMsg" (no quote class) +// and the original is wrapped in a bare `
` after it. +const OUTLOOK_REAL_WORLD_REPLY = `

Hi team,

See attached the latest proposal.

Regards,
Pat

From: Sam <sam@example.test>
Sent: Wednesday, December 4, 2024 5:15 PM
To: Pat <pat@example.test>
Subject: Quotation

Hi Pat,

Quotation attached.

Sam

`; + +// Yahoo Mail: wraps the previous email in
. +const YAHOO_MAIL_REPLY = `
My reply text here.
Thanks,
Pat
On Wednesday, December 4, 2024, 5:15 PM, Sam <sam@example.test> wrote:
Original Yahoo-quoted message body.
`; + +// Thunderbird: the attribution paragraph carries class="moz-cite-prefix" and +// the original lives in
. +const THUNDERBIRD_REPLY = `

My reply.

Thanks,
Pat

On 4/12/24 17:15, Sam wrote:

Original Thunderbird-quoted message body.

`; + +// Gmail forwarded message — the body reads "---------- Forwarded message ---------" +// in plain text, with a header block beneath listing From/Date/Subject/To. +const GMAIL_FORWARDED_MESSAGE = `
FYI — see the original below.

---------- Forwarded message ---------
From: Sam <sam@example.test>
Date: Wed, 4 Dec 2024 at 17:15
Subject: Quotation
To: Pat <pat@example.test>

Original forwarded body content.
`; + +// Outlook plain text "----- Original Message -----" header. +const OUTLOOK_ORIGINAL_MESSAGE = `

Quick reply.

Thanks

-----Original Message-----
From: Sam <sam@example.test>
Sent: Wednesday, December 4, 2024 5:15 PM
To: Pat <pat@example.test>
Subject: Quotation

Original Outlook plain-style reply.

`; + +// Inline reply: bare
(no client class) sits in the middle, with +// content following it. Trailing-only rule should preserve the blockquote. +const INLINE_REPLY = `

See my responses inline below.

Question 1: pricing?

Answer: usd 100.

Question 2: timeline?

Answer: 2 weeks.

Let me know if any of that needs clarification.

Pat

`; + +// Plain conversational body with no quote markers — must not be mistakenly +// stripped and must not show the toggle. +const NO_QUOTE_BODY = `

Just checking in — any update on this?

Thanks,
Pat

`; + +// iPhone Mail / `text/plain` reply, after sanitizeTextForRender() has converted +// `\n` → `
` and escaped lone `< > &`. +const IPHONE_MAIL_PLAINTEXT = [ + 'Test payments email

', + 'Thanks,
Shruthi

', + 'Sent from my iPhone

', + '> On Apr 6, 2026, at 11:26 PM, Shruthi M ', + '<shruthi.rohini.7@gmail.com> wrote:
', + '>
> Hi email
> To Eli
', + '> Thanks,
> Shruthi
', + '>
> Sent from my iPhone', +].join(''); + describe('EmailQuoteExtractor', () => { + describe('real-world client shapes', () => { + it('Gmail web reply — strips .gmail_quote_container with attribution + blockquote', () => { + const cleaned = EmailQuoteExtractor.extractQuotes(GMAIL_REAL_WORLD_REPLY); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('Dear Sam'); + expect(c.textContent).toContain('Best,'); + expect(c.textContent).not.toContain('Thank you for your inquiry'); + expect(c.textContent).not.toContain('On Wed, 4 Dec 2024'); + expect(c.querySelector('.gmail_quote')).toBeNull(); + expect(EmailQuoteExtractor.hasQuotes(GMAIL_REAL_WORLD_REPLY)).toBe(true); + }); + + it('Outlook reply — strips #divRplyFwdMsg header AND the trailing bare blockquote', () => { + const cleaned = EmailQuoteExtractor.extractQuotes( + OUTLOOK_REAL_WORLD_REPLY + ); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('Hi team'); + expect(c.textContent).toContain('Regards,'); + expect(c.textContent).not.toContain('From: Sam'); + expect(c.textContent).not.toContain('Quotation attached'); + expect(c.querySelector('blockquote')).toBeNull(); + expect(c.querySelector('#divRplyFwdMsg')).toBeNull(); + expect(EmailQuoteExtractor.hasQuotes(OUTLOOK_REAL_WORLD_REPLY)).toBe( + true + ); + }); + + it('Yahoo Mail reply — strips .yahoo_quoted wrapper', () => { + const cleaned = EmailQuoteExtractor.extractQuotes(YAHOO_MAIL_REPLY); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('My reply text here'); + expect(c.textContent).not.toContain('Original Yahoo-quoted message'); + expect(c.querySelector('.yahoo_quoted')).toBeNull(); + expect(EmailQuoteExtractor.hasQuotes(YAHOO_MAIL_REPLY)).toBe(true); + }); + + it('Thunderbird reply — strips .moz-cite-prefix attribution AND
', () => { + const cleaned = EmailQuoteExtractor.extractQuotes(THUNDERBIRD_REPLY); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('My reply'); + expect(c.textContent).toContain('Thanks,'); + expect(c.textContent).not.toContain('On 4/12/24 17:15'); + expect(c.textContent).not.toContain( + 'Original Thunderbird-quoted message' + ); + expect(c.querySelector('blockquote')).toBeNull(); + expect(c.querySelector('.moz-cite-prefix')).toBeNull(); + expect(EmailQuoteExtractor.hasQuotes(THUNDERBIRD_REPLY)).toBe(true); + }); + + it('Gmail forwarded message — strips "---------- Forwarded message ----------" block', () => { + const cleaned = EmailQuoteExtractor.extractQuotes( + GMAIL_FORWARDED_MESSAGE + ); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('FYI — see the original below'); + expect(c.textContent).not.toContain('Forwarded message'); + expect(c.textContent).not.toContain('Original forwarded body content'); + expect(EmailQuoteExtractor.hasQuotes(GMAIL_FORWARDED_MESSAGE)).toBe(true); + }); + + it('Outlook plain-style "-----Original Message-----" header is stripped', () => { + const cleaned = EmailQuoteExtractor.extractQuotes( + OUTLOOK_ORIGINAL_MESSAGE + ); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('Quick reply'); + expect(c.textContent).not.toContain('Original Message'); + expect(c.textContent).not.toContain('Original Outlook plain-style reply'); + expect(EmailQuoteExtractor.hasQuotes(OUTLOOK_ORIGINAL_MESSAGE)).toBe( + true + ); + }); + + it('inline reply — when content follows the quoted block, the body is left intact', () => { + const cleaned = EmailQuoteExtractor.extractQuotes(INLINE_REPLY); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('See my responses inline below'); + expect(c.textContent).toContain('Question 1: pricing?'); + expect(c.textContent).toContain( + 'Let me know if any of that needs clarification' + ); + expect(c.querySelector('blockquote')).not.toBeNull(); + expect(EmailQuoteExtractor.hasQuotes(INLINE_REPLY)).toBe(false); + }); + + it('plain body with no quotes — body unchanged, no toggle', () => { + const cleaned = EmailQuoteExtractor.extractQuotes(NO_QUOTE_BODY); + const c = document.createElement('div'); + c.innerHTML = cleaned; + expect(c.textContent).toContain('Just checking in'); + expect(c.textContent).toContain('Thanks,'); + expect(EmailQuoteExtractor.hasQuotes(NO_QUOTE_BODY)).toBe(false); + }); + + it('empty body — no error, no toggle', () => { + expect(() => EmailQuoteExtractor.extractQuotes('')).not.toThrow(); + expect(EmailQuoteExtractor.hasQuotes('')).toBe(false); + }); + }); + + // Regression tests — develop-baseline behaviours that earlier rewrites broke. + describe('develop-baseline regression coverage', () => { + it('detects header quote inside a single outer wrapper
', () => { + const html = + '

My reply.

-----Original Message-----
From: Sam
Sent: ...

Old body line 1

'; + expect(EmailQuoteExtractor.hasQuotes(html)).toBe(true); + const cleaned = EmailQuoteExtractor.extractQuotes(html); + expect(cleaned).toContain('My reply'); + expect(cleaned).not.toContain('Original Message'); + }); + + it('detects "On … wrote:" header even when followed by un-prefixed old lines', () => { + const html = + '

On Wed, Sam wrote:

Old line 1

Old line 2

'; + expect(EmailQuoteExtractor.hasQuotes(html)).toBe(true); + const cleaned = EmailQuoteExtractor.extractQuotes(html); + expect(cleaned).not.toContain('On Wed, Sam wrote'); + }); + + it('detects "From:/Sent:" header even when followed by un-prefixed old lines', () => { + const html = + '

Reply text.

From: Sam
Sent: Wednesday

Old line 1

'; + expect(EmailQuoteExtractor.hasQuotes(html)).toBe(true); + const cleaned = EmailQuoteExtractor.extractQuotes(html); + expect(cleaned).toContain('Reply text'); + expect(cleaned).not.toContain('From: Sam'); + }); + }); + + it('strips RFC-style `>` quoted lines from a plain-text only body (iPhone Mail)', () => { + const cleanedHtml = EmailQuoteExtractor.extractQuotes( + IPHONE_MAIL_PLAINTEXT + ); + const container = document.createElement('div'); + container.innerHTML = cleanedHtml; + const text = container.textContent; + + expect(text).toContain('Test payments email'); + expect(text).toContain('Sent from my iPhone'); // signature stays + expect(text).not.toContain('On Apr 6, 2026'); + expect(text).not.toContain('Hi email'); + expect(text).not.toContain('To Eli'); + expect(EmailQuoteExtractor.hasQuotes(IPHONE_MAIL_PLAINTEXT)).toBe(true); + }); + + it('strips both plain-text `>` lines and HTML quote blocks in the same body', () => { + const cleanedHtml = EmailQuoteExtractor.extractQuotes( + MIXED_PLAINTEXT_AND_HTML_QUOTE + ); + const container = document.createElement('div'); + container.innerHTML = cleanedHtml; + const text = container.textContent; + + // Reply portion stays + expect(text).toContain('My HTML reply'); + expect(text).toContain('Sivin'); + + // Plain-text `>` quote lines are gone + expect(text).not.toContain('Inline plain-text quote line'); + expect(text).not.toContain('On Mon, Apr 6, 2026, Shruthi wrote'); + + // HTML quote block is gone + expect(text).not.toContain('The original HTML quoted email'); + expect(container.querySelectorAll('.gmail_quote').length).toBe(0); + }); + it('removes blockquote-based quotes from the email body', () => { const cleanedHtml = EmailQuoteExtractor.extractQuotes(SAMPLE_EMAIL_HTML);