From f2eaa845dc09c92803bd81283b1638a3ae8cfdac Mon Sep 17 00:00:00 2001 From: Vinay Keerthi <11478411+stonecharioteer@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:45:27 +0530 Subject: [PATCH 1/5] fix: Preserve multiple consecutive newlines in text-based messaging channels (#13032) ## Description This PR fixes an issue where multiple consecutive newlines (blank lines for visual spacing) were being collapsed in text-based messaging channels like WhatsApp, Instagram, and SMS. When users send messages via API with intentional spacing using multiple newlines (e.g., `\n\n\n\n`), the markdown renderer was following standard Markdown spec and collapsing them into single blank lines. While this is correct for document formatting, messaging platforms like WhatsApp and Instagram support and preserve multiple blank lines for visual spacing. The fix adds preprocessing to preserve multiple consecutive newlines (3+) by converting them to placeholder tokens before CommonMarker processing, then restoring the exact number of newlines in the final output. ## Changes - Added `preserve_multiple_newlines` and `restore_multiple_newlines` helper methods to `MarkdownRendererService` - Updated `render_whatsapp` to preserve multiple consecutive newlines - Updated `render_instagram` to preserve multiple consecutive newlines - Updated `render_plain_text` (affects SMS, Twilio SMS, Twitter) to preserve multiple consecutive newlines - Updated `render_line` to preserve multiple consecutive newlines - HTML-based renderers (Email, Telegram, WebWidget) remain unchanged as they handle spacing via HTML tags ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected) - [ ] This change requires a documentation update ## How Has This Been Tested? Added comprehensive test coverage: - 3 new tests for multi-newline preservation across WhatsApp, Instagram, and SMS channels - All 56 tests passing (up from 53) Testing scenarios: - Single newlines preserved: `"Line 1\nLine 2"` remains `"Line 1\nLine 2"` - Multiple newlines preserved: `"Para 1\n\n\n\nPara 2"` remains `"Para 1\n\n\n\nPara 2"` - Standard paragraph breaks (2 newlines) work as before - Markdown formatting (bold, italic, links) continues to work correctly - Backward compatibility maintained for all channels ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] Any dependent changes have been merged and published in downstream modules --- .../messages/markdown_renderer_service.rb | 35 ++++++++++++++++--- .../markdown_renderer_service_spec.rb | 21 +++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/app/services/messages/markdown_renderer_service.rb b/app/services/messages/markdown_renderer_service.rb index 9348b2bc7..8984374d1 100644 --- a/app/services/messages/markdown_renderer_service.rb +++ b/app/services/messages/markdown_renderer_service.rb @@ -53,22 +53,49 @@ class Messages::MarkdownRendererService end def render_whatsapp + content_with_preserved_newlines = preserve_multiple_newlines(@content) renderer = Messages::MarkdownRenderers::WhatsAppRenderer.new - renderer.render(commonmarker_doc).gsub(/\n+\z/, '') + doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) + result = renderer.render(doc).gsub(/\n+\z/, '') + restore_multiple_newlines(result) end def render_instagram + content_with_preserved_newlines = preserve_multiple_newlines(@content) renderer = Messages::MarkdownRenderers::InstagramRenderer.new - renderer.render(commonmarker_doc).gsub(/\n+\z/, '') + doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) + result = renderer.render(doc).gsub(/\n+\z/, '') + restore_multiple_newlines(result) end def render_line + content_with_preserved_newlines = preserve_multiple_newlines(@content) renderer = Messages::MarkdownRenderers::LineRenderer.new - renderer.render(commonmarker_doc).gsub(/\n+\z/, '') + doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) + result = renderer.render(doc).gsub(/\n+\z/, '') + restore_multiple_newlines(result) end def render_plain_text + content_with_preserved_newlines = preserve_multiple_newlines(@content) renderer = Messages::MarkdownRenderers::PlainTextRenderer.new - renderer.render(commonmarker_doc).gsub(/\n+\z/, '') + doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) + result = renderer.render(doc).gsub(/\n+\z/, '') + restore_multiple_newlines(result) + end + + # Preserve multiple consecutive newlines (3+) by replacing them with placeholders + # Standard markdown treats 2 newlines as paragraph break, we preserve 3+ + def preserve_multiple_newlines(content) + content.gsub(/\n{3,}/) do |match| + "{{PRESERVE_#{match.length}_NEWLINES}}" + end + end + + # Restore multiple newlines from placeholders + def restore_multiple_newlines(content) + content.gsub(/\{\{PRESERVE_(\d+)_NEWLINES\}\}/) do |_match| + "\n" * Regexp.last_match(1).to_i + end end end diff --git a/spec/services/messages/markdown_renderer_service_spec.rb b/spec/services/messages/markdown_renderer_service_spec.rb index 26c661495..d589c6ad1 100644 --- a/spec/services/messages/markdown_renderer_service_spec.rb +++ b/spec/services/messages/markdown_renderer_service_spec.rb @@ -67,6 +67,13 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do expect(result).to include("Line 1\nLine 2\nLine 3") expect(result).not_to include('Line 1 Line 2') end + + it 'preserves multiple consecutive newlines for spacing' do + content = "Para 1\n\n\n\nPara 2" + result = described_class.new(content, channel_type).render + expect(result.scan("\n").count).to eq(4) + expect(result).to include("Para 1\n\n\n\nPara 2") + end end context 'when channel is Channel::Instagram' do @@ -116,6 +123,13 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do expect(result).to include("Line 1\nLine 2\nLine 3") expect(result).not_to include('Line 1 Line 2') end + + it 'preserves multiple consecutive newlines for spacing' do + content = "Para 1\n\n\n\nPara 2" + result = described_class.new(content, channel_type).render + expect(result.scan("\n").count).to eq(4) + expect(result).to include("Para 1\n\n\n\nPara 2") + end end context 'when channel is Channel::Line' do @@ -201,6 +215,13 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do expect(result).to include("Line 1\nLine 2\nLine 3") expect(result).not_to include('Line 1 Line 2') end + + it 'preserves multiple consecutive newlines for spacing' do + content = "Para 1\n\n\n\nPara 2" + result = described_class.new(content, channel_type).render + expect(result.scan("\n").count).to eq(4) + expect(result).to include("Para 1\n\n\n\nPara 2") + end end context 'when channel is Channel::Telegram' do From 20fa5eeaa544b326d603fa8c87c39c183db442a1 Mon Sep 17 00:00:00 2001 From: Muhsin Keloth Date: Wed, 10 Dec 2025 12:28:47 +0530 Subject: [PATCH 2/5] fix: Prevent SLA deletion timeouts by moving to async job (#12944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes the HTTP 500 timeout errors occurring when deleting SLA policies that have large volumes of historical data. The fix moves the deletion workflow to asynchronous background processing using the existing `DeleteObjectJob`. By offloading heavy cascaded deletions (applied SLAs, SLA events, conversation nullifications) from the request cycle, the API can now return immediately while the cleanup continues in the background avoiding the `Rack::Timeout::RequestTimeoutException`. This ensures that SLA policies can be deleted reliably, regardless of data size. ### Problem Deleting an SLA policy via `DELETE /api/v1/accounts/{account_id}/sla_policies/{id}` fails consistently with `Rack::Timeout::RequestTimeoutException (15s)` for policies with large amounts of related data. Because the current implementation performs all dependent deletions **synchronously**, Rails processes: - `has_many :applied_slas, dependent: :destroy` (thousands) - Each `AppliedSla#destroy` → triggers destruction of many `SlaEvent` records - `has_many :conversations, dependent: :nullify` (thousands) This processing far exceeds the Rack timeout window and consistently triggers HTTP 500 errors for users. ### Solution This PR applies the same pattern used successfully in Inbox deletion. **Move deletion to async background jobs** - Uses `DeleteObjectJob` for centralized, reliable cleanup. - Allows the DELETE API call to respond immediately. **Chunk large datasets** - Records are processed in **batches of 5,000** to reduce DB load and avoid job timeouts. --- app/jobs/delete_object_job.rb | 15 +- .../v1/accounts/sla_policies_controller.rb | 2 +- .../app/jobs/enterprise/delete_object_job.rb | 10 +- enterprise/app/models/sla_policy.rb | 2 +- lib/tasks/apply_sla.rake | 100 ++++++++++ lib/tasks/bulk_conversations.rake | 176 ++++++++++++++++++ .../accounts/sla_policies_controller_spec.rb | 5 +- 7 files changed, 299 insertions(+), 11 deletions(-) create mode 100644 lib/tasks/apply_sla.rake create mode 100644 lib/tasks/bulk_conversations.rake diff --git a/app/jobs/delete_object_job.rb b/app/jobs/delete_object_job.rb index 756d0feb1..4e9030012 100644 --- a/app/jobs/delete_object_job.rb +++ b/app/jobs/delete_object_job.rb @@ -2,10 +2,6 @@ class DeleteObjectJob < ApplicationJob queue_as :low BATCH_SIZE = 5_000 - HEAVY_ASSOCIATIONS = { - Account => %i[conversations contacts inboxes reporting_events], - Inbox => %i[conversations contact_inboxes reporting_events] - }.freeze def perform(object, user = nil, ip = nil) # Pre-purge heavy associations for large objects to avoid @@ -19,11 +15,18 @@ class DeleteObjectJob < ApplicationJob private + def heavy_associations + { + Account => %i[conversations contacts inboxes reporting_events], + Inbox => %i[conversations contact_inboxes reporting_events] + }.freeze + end + def purge_heavy_associations(object) - klass = HEAVY_ASSOCIATIONS.keys.find { |k| object.is_a?(k) } + klass = heavy_associations.keys.find { |k| object.is_a?(k) } return unless klass - HEAVY_ASSOCIATIONS[klass].each do |assoc| + heavy_associations[klass].each do |assoc| next unless object.respond_to?(assoc) batch_destroy(object.public_send(assoc)) diff --git a/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb b/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb index ec879b042..b3b0f52d4 100644 --- a/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb +++ b/enterprise/app/controllers/api/v1/accounts/sla_policies_controller.rb @@ -17,7 +17,7 @@ class Api::V1::Accounts::SlaPoliciesController < Api::V1::Accounts::EnterpriseAc end def destroy - @sla_policy.destroy! + ::DeleteObjectJob.perform_later(@sla_policy, Current.user, request.ip) if @sla_policy.present? head :ok end diff --git a/enterprise/app/jobs/enterprise/delete_object_job.rb b/enterprise/app/jobs/enterprise/delete_object_job.rb index 147fa0d43..375015bc7 100644 --- a/enterprise/app/jobs/enterprise/delete_object_job.rb +++ b/enterprise/app/jobs/enterprise/delete_object_job.rb @@ -1,10 +1,18 @@ module Enterprise::DeleteObjectJob + private + + def heavy_associations + super.merge( + SlaPolicy => %i[applied_slas] + ).freeze + end + def process_post_deletion_tasks(object, user, ip) create_audit_entry(object, user, ip) end def create_audit_entry(object, user, ip) - return unless %w[Inbox Conversation].include?(object.class.to_s) && user.present? + return unless %w[Inbox Conversation SlaPolicy].include?(object.class.to_s) && user.present? Enterprise::AuditLog.create( auditable: object, diff --git a/enterprise/app/models/sla_policy.rb b/enterprise/app/models/sla_policy.rb index f53f00ed5..f61d5a1f3 100644 --- a/enterprise/app/models/sla_policy.rb +++ b/enterprise/app/models/sla_policy.rb @@ -22,7 +22,7 @@ class SlaPolicy < ApplicationRecord validates :name, presence: true has_many :conversations, dependent: :nullify - has_many :applied_slas, dependent: :destroy + has_many :applied_slas, dependent: :destroy_async def push_event_data { diff --git a/lib/tasks/apply_sla.rake b/lib/tasks/apply_sla.rake new file mode 100644 index 000000000..70adf8cf3 --- /dev/null +++ b/lib/tasks/apply_sla.rake @@ -0,0 +1,100 @@ +# Apply SLA Policy to Conversations +# +# This task applies an SLA policy to existing conversations that don't have one assigned. +# It processes conversations in batches and only affects conversations with sla_policy_id = nil. +# +# Usage Examples: +# # Using arguments (may need escaping in some shells) +# bundle exec rake "sla:apply_to_conversations[19,1,500]" +# +# # Using environment variables (recommended) +# SLA_POLICY_ID=19 ACCOUNT_ID=1 BATCH_SIZE=500 bundle exec rake sla:apply_to_conversations +# +# Parameters: +# SLA_POLICY_ID: ID of the SLA policy to apply (required) +# ACCOUNT_ID: ID of the account (required) +# BATCH_SIZE: Number of conversations to process (default: 1000) +# +# Notes: +# - Only runs in development environment +# - Processes conversations in order of newest first (id DESC) +# - Safe to run multiple times - skips conversations that already have SLA policies +# - Creates AppliedSla records automatically via Rails callbacks +# - SlaEvent records are created later by background jobs when violations occur +# +# rubocop:disable Metrics/BlockLength +namespace :sla do + desc 'Apply SLA policy to existing conversations' + task :apply_to_conversations, [:sla_policy_id, :account_id, :batch_size] => :environment do |_t, args| + unless Rails.env.development? + puts 'This task can only be run in the development environment.' + puts "Current environment: #{Rails.env}" + exit(1) + end + + sla_policy_id = args[:sla_policy_id] || ENV.fetch('SLA_POLICY_ID', nil) + account_id = args[:account_id] || ENV.fetch('ACCOUNT_ID', nil) + batch_size = (args[:batch_size] || ENV['BATCH_SIZE'] || 1000).to_i + + if sla_policy_id.blank? + puts 'Error: SLA_POLICY_ID is required' + puts 'Usage: bundle exec rake sla:apply_to_conversations[sla_policy_id,account_id,batch_size]' + puts 'Or: SLA_POLICY_ID=1 ACCOUNT_ID=1 BATCH_SIZE=500 bundle exec rake sla:apply_to_conversations' + exit(1) + end + + if account_id.blank? + puts 'Error: ACCOUNT_ID is required' + puts 'Usage: bundle exec rake sla:apply_to_conversations[sla_policy_id,account_id,batch_size]' + puts 'Or: SLA_POLICY_ID=1 ACCOUNT_ID=1 BATCH_SIZE=500 bundle exec rake sla:apply_to_conversations' + exit(1) + end + + account = Account.find_by(id: account_id) + unless account + puts "Error: Account with ID #{account_id} not found" + exit(1) + end + + sla_policy = account.sla_policies.find_by(id: sla_policy_id) + unless sla_policy + puts "Error: SLA Policy with ID #{sla_policy_id} not found for Account #{account_id}" + exit(1) + end + + conversations = account.conversations.where(sla_policy_id: nil).order(id: :desc).limit(batch_size) + total_count = conversations.count + + if total_count.zero? + puts 'No conversations found without SLA policy' + exit(0) + end + + puts "Applying SLA Policy '#{sla_policy.name}' (ID: #{sla_policy_id}) to #{total_count} conversations in Account #{account_id}" + puts "Processing in batches of #{batch_size}" + puts "Started at: #{Time.current}" + + start_time = Time.current + processed_count = 0 + error_count = 0 + + conversations.find_in_batches(batch_size: batch_size) do |batch| + batch.each do |conversation| + conversation.update!(sla_policy_id: sla_policy_id) + processed_count += 1 + puts "Processed #{processed_count}/#{total_count} conversations" if (processed_count % 100).zero? + rescue StandardError => e + error_count += 1 + puts "Error applying SLA to conversation #{conversation.id}: #{e.message}" + end + end + + elapsed_time = Time.current - start_time + puts "\nCompleted!" + puts "Successfully processed: #{processed_count} conversations" + puts "Errors encountered: #{error_count}" if error_count.positive? + puts "Total time: #{elapsed_time.round(2)}s" + puts "Average time per conversation: #{(elapsed_time / processed_count).round(3)}s" if processed_count.positive? + end +end +# rubocop:enable Metrics/BlockLength diff --git a/lib/tasks/bulk_conversations.rake b/lib/tasks/bulk_conversations.rake new file mode 100644 index 000000000..e17851f31 --- /dev/null +++ b/lib/tasks/bulk_conversations.rake @@ -0,0 +1,176 @@ +# Generate Bulk Conversations +# +# This task creates bulk conversations with fake contacts and movie dialogue messages +# for testing purposes. Each conversation gets random messages between contacts and agents. +# +# Usage Examples: +# # Using arguments (may need escaping in some shells) +# bundle exec rake "conversations:generate_bulk[100,1,1]" +# +# # Using environment variables (recommended) +# COUNT=100 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk +# +# # Generate 50 conversations +# COUNT=50 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk +# +# Parameters: +# COUNT: Number of conversations to create (default: 10) +# ACCOUNT_ID: ID of the account (required) +# INBOX_ID: ID of the inbox that belongs to the account (required) +# +# What it creates: +# - Unique contacts with fake names, emails, phone numbers +# - Conversations with random status (open/resolved/pending) +# - 3-10 messages per conversation with movie quotes +# - Alternating incoming/outgoing message flow +# +# Notes: +# - Only runs in development environment +# - Creates realistic test data for conversation testing +# - Progress shown every 10 conversations +# - All contacts get unique email addresses to avoid conflicts +# +# rubocop:disable Metrics/BlockLength +namespace :conversations do + desc 'Generate bulk conversations with contacts and movie dialogue messages' + task :generate_bulk, [:count, :account_id, :inbox_id] => :environment do |_t, args| + unless Rails.env.development? + puts 'This task can only be run in the development environment.' + puts "Current environment: #{Rails.env}" + exit(1) + end + + count = (args[:count] || ENV['COUNT'] || 10).to_i + account_id = args[:account_id] || ENV.fetch('ACCOUNT_ID', nil) + inbox_id = args[:inbox_id] || ENV.fetch('INBOX_ID', nil) + + if account_id.blank? + puts 'Error: ACCOUNT_ID is required' + puts 'Usage: bundle exec rake conversations:generate_bulk[count,account_id,inbox_id]' + puts 'Or: COUNT=100 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk' + exit(1) + end + + if inbox_id.blank? + puts 'Error: INBOX_ID is required' + puts 'Usage: bundle exec rake conversations:generate_bulk[count,account_id,inbox_id]' + puts 'Or: COUNT=100 ACCOUNT_ID=1 INBOX_ID=1 bundle exec rake conversations:generate_bulk' + exit(1) + end + + account = Account.find_by(id: account_id) + inbox = Inbox.find_by(id: inbox_id) + + unless account + puts "Error: Account with ID #{account_id} not found" + exit(1) + end + + unless inbox + puts "Error: Inbox with ID #{inbox_id} not found" + exit(1) + end + + unless inbox.account_id == account.id + puts "Error: Inbox #{inbox_id} does not belong to Account #{account_id}" + exit(1) + end + + puts "Generating #{count} conversations for Account ##{account.id} in Inbox ##{inbox.id}..." + puts "Started at: #{Time.current}" + + start_time = Time.current + created_count = 0 + + count.times do |i| + contact = create_contact(account) + contact_inbox = create_contact_inbox(contact, inbox) + conversation = create_conversation(contact_inbox) + add_messages(conversation) + + created_count += 1 + puts "Created conversation #{i + 1}/#{count} (ID: #{conversation.id})" if ((i + 1) % 10).zero? + rescue StandardError => e + puts "Error creating conversation #{i + 1}: #{e.message}" + puts e.backtrace.first(5).join("\n") + end + + elapsed_time = Time.current - start_time + puts "\nCompleted!" + puts "Successfully created: #{created_count} conversations" + puts "Total time: #{elapsed_time.round(2)}s" + puts "Average time per conversation: #{(elapsed_time / created_count).round(3)}s" if created_count.positive? + end + + def create_contact(account) + Contact.create!( + account: account, + name: Faker::Name.name, + email: "#{SecureRandom.uuid}@example.com", + phone_number: generate_e164_phone_number, + additional_attributes: { + source: 'bulk_generator', + company: Faker::Company.name, + city: Faker::Address.city + } + ) + end + + def generate_e164_phone_number + country_code = [1, 44, 61, 91, 81].sample + subscriber_number = rand(1_000_000..9_999_999_999).to_s + subscriber_number = subscriber_number[0...(15 - country_code.to_s.length)] + "+#{country_code}#{subscriber_number}" + end + + def create_contact_inbox(contact, inbox) + ContactInboxBuilder.new( + contact: contact, + inbox: inbox + ).perform + end + + def create_conversation(contact_inbox) + ConversationBuilder.new( + params: ActionController::Parameters.new( + status: %w[open resolved pending].sample, + additional_attributes: {}, + custom_attributes: {} + ), + contact_inbox: contact_inbox + ).perform + end + + def add_messages(conversation) + num_messages = rand(3..10) + message_type = %w[incoming outgoing].sample + + num_messages.times do + message_type = message_type == 'incoming' ? 'outgoing' : 'incoming' + create_message(conversation, message_type) + end + end + + def create_message(conversation, message_type) + sender = if message_type == 'incoming' + conversation.contact + else + conversation.account.users.sample || conversation.account.administrators.first + end + + conversation.messages.create!( + account: conversation.account, + inbox: conversation.inbox, + sender: sender, + message_type: message_type, + content: generate_movie_dialogue, + content_type: :text, + private: false + ) + end + + def generate_movie_dialogue + Faker::Movie.quote + end +end +# rubocop:enable Metrics/BlockLength diff --git a/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb b/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb index b1619ef85..e1a4fa538 100644 --- a/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb +++ b/spec/enterprise/controllers/api/v1/accounts/sla_policies_controller_spec.rb @@ -161,12 +161,13 @@ RSpec.describe 'Enterprise SLA API', type: :request do let(:sla_policy) { create(:sla_policy, account: account) } context 'when it is an authenticated user' do - it 'deletes the sla_policy' do + it 'queues the sla_policy for deletion' do + expect(DeleteObjectJob).to receive(:perform_later).with(sla_policy, administrator, kind_of(String)) + delete "/api/v1/accounts/#{account.id}/sla_policies/#{sla_policy.id}", headers: administrator.create_new_auth_token expect(response).to have_http_status(:success) - expect(SlaPolicy.count).to eq(1) end end From 0d8e249fe488ea230f5a76c57a182563b4751834 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 10 Dec 2025 15:25:18 +0530 Subject: [PATCH 3/5] feat: include chatwoot metadata with each tool call (#12907) --- enterprise/app/models/concerns/toolable.rb | 25 ++++ enterprise/lib/captain/tools/http_tool.rb | 13 +- .../lib/captain/tools/http_tool_spec.rb | 130 ++++++++++++++++++ .../models/captain/custom_tool_spec.rb | 92 +++++++++++++ 4 files changed, 257 insertions(+), 3 deletions(-) diff --git a/enterprise/app/models/concerns/toolable.rb b/enterprise/app/models/concerns/toolable.rb index ad047e8f8..51ec1be3e 100644 --- a/enterprise/app/models/concerns/toolable.rb +++ b/enterprise/app/models/concerns/toolable.rb @@ -66,6 +66,31 @@ module Concerns::Toolable [auth_config['username'], auth_config['password']] end + def build_metadata_headers(state) + {}.tap do |headers| + add_base_headers(headers, state) + add_conversation_headers(headers, state[:conversation]) if state[:conversation] + add_contact_headers(headers, state[:contact]) if state[:contact] + end + end + + def add_base_headers(headers, state) + headers['X-Chatwoot-Account-Id'] = state[:account_id].to_s if state[:account_id] + headers['X-Chatwoot-Assistant-Id'] = state[:assistant_id].to_s if state[:assistant_id] + headers['X-Chatwoot-Tool-Slug'] = slug if slug.present? + end + + def add_conversation_headers(headers, conversation) + headers['X-Chatwoot-Conversation-Id'] = conversation[:id].to_s if conversation[:id] + headers['X-Chatwoot-Conversation-Display-Id'] = conversation[:display_id].to_s if conversation[:display_id] + end + + def add_contact_headers(headers, contact) + headers['X-Chatwoot-Contact-Id'] = contact[:id].to_s if contact[:id] + headers['X-Chatwoot-Contact-Email'] = contact[:email].to_s if contact[:email].present? + headers['X-Chatwoot-Contact-Phone'] = contact[:phone_number].to_s if contact[:phone_number].present? + end + def format_response(raw_response_body) return raw_response_body if response_template.blank? diff --git a/enterprise/lib/captain/tools/http_tool.rb b/enterprise/lib/captain/tools/http_tool.rb index b634de04e..b4593d27f 100644 --- a/enterprise/lib/captain/tools/http_tool.rb +++ b/enterprise/lib/captain/tools/http_tool.rb @@ -11,11 +11,11 @@ class Captain::Tools::HttpTool < Agents::Tool @custom_tool.enabled? end - def perform(_tool_context, **params) + def perform(tool_context, **params) url = @custom_tool.build_request_url(params) body = @custom_tool.build_request_body(params) - response = execute_http_request(url, body) + response = execute_http_request(url, body, tool_context) @custom_tool.format_response(response.body) rescue StandardError => e Rails.logger.error("HttpTool execution error for #{@custom_tool.slug}: #{e.class} - #{e.message}") @@ -39,7 +39,7 @@ class Captain::Tools::HttpTool < Agents::Tool # 1MB of text ≈ 250K tokens, which exceeds most LLM context windows MAX_RESPONSE_SIZE = 1.megabyte - def execute_http_request(url, body) + def execute_http_request(url, body, tool_context) uri = URI.parse(url) # Check if resolved IP is private @@ -53,6 +53,7 @@ class Captain::Tools::HttpTool < Agents::Tool request = build_http_request(uri, body) apply_authentication(request) + apply_metadata_headers(request, tool_context) response = http.request(request) @@ -102,4 +103,10 @@ class Captain::Tools::HttpTool < Agents::Tool credentials = @custom_tool.build_basic_auth_credentials request.basic_auth(*credentials) if credentials end + + def apply_metadata_headers(request, tool_context) + state = tool_context&.state || {} + metadata_headers = @custom_tool.build_metadata_headers(state) + metadata_headers.each { |key, value| request[key] = value } + end end diff --git a/spec/enterprise/lib/captain/tools/http_tool_spec.rb b/spec/enterprise/lib/captain/tools/http_tool_spec.rb index d48af2752..967a10574 100644 --- a/spec/enterprise/lib/captain/tools/http_tool_spec.rb +++ b/spec/enterprise/lib/captain/tools/http_tool_spec.rb @@ -237,5 +237,135 @@ RSpec.describe Captain::Tools::HttpTool, type: :model do expect(result).to eq('Created order #ORD-789 for Widget') end end + + context 'with metadata headers' do + let(:conversation) { create(:conversation, account: account) } + let(:contact) { conversation.contact } + let(:tool_context_with_state) do + Struct.new(:state).new({ + account_id: account.id, + assistant_id: assistant.id, + conversation: { + id: conversation.id, + display_id: conversation.display_id + }, + contact: { + id: contact.id, + email: contact.email, + phone_number: contact.phone_number + } + }) + end + + before do + custom_tool.update!( + endpoint_url: 'https://example.com/api/data', + response_template: nil + ) + end + + it 'includes metadata headers in GET request' do + stub_request(:get, 'https://example.com/api/data') + .with(headers: { + 'X-Chatwoot-Account-Id' => account.id.to_s, + 'X-Chatwoot-Assistant-Id' => assistant.id.to_s, + 'X-Chatwoot-Tool-Slug' => custom_tool.slug, + 'X-Chatwoot-Conversation-Id' => conversation.id.to_s, + 'X-Chatwoot-Conversation-Display-Id' => conversation.display_id.to_s, + 'X-Chatwoot-Contact-Id' => contact.id.to_s, + 'X-Chatwoot-Contact-Email' => contact.email + }) + .to_return(status: 200, body: '{"success": true}') + + tool.perform(tool_context_with_state) + + expect(WebMock).to have_requested(:get, 'https://example.com/api/data') + .with(headers: { + 'X-Chatwoot-Account-Id' => account.id.to_s, + 'X-Chatwoot-Contact-Email' => contact.email + }) + end + + it 'includes metadata headers in POST request' do + custom_tool.update!(http_method: 'POST', request_template: '{"data": "test"}') + + stub_request(:post, 'https://example.com/api/data') + .with( + body: '{"data": "test"}', + headers: { + 'Content-Type' => 'application/json', + 'X-Chatwoot-Account-Id' => account.id.to_s, + 'X-Chatwoot-Tool-Slug' => custom_tool.slug, + 'X-Chatwoot-Contact-Email' => contact.email + } + ) + .to_return(status: 200, body: '{"success": true}') + + tool.perform(tool_context_with_state) + + expect(WebMock).to have_requested(:post, 'https://example.com/api/data') + end + + it 'includes metadata headers along with authentication headers' do + custom_tool.update!( + auth_type: 'bearer', + auth_config: { 'token' => 'test_token' } + ) + + stub_request(:get, 'https://example.com/api/data') + .with(headers: { + 'Authorization' => 'Bearer test_token', + 'X-Chatwoot-Account-Id' => account.id.to_s, + 'X-Chatwoot-Contact-Id' => contact.id.to_s + }) + .to_return(status: 200, body: '{"success": true}') + + tool.perform(tool_context_with_state) + + expect(WebMock).to have_requested(:get, 'https://example.com/api/data') + .with(headers: { + 'Authorization' => 'Bearer test_token', + 'X-Chatwoot-Contact-Id' => contact.id.to_s + }) + end + + it 'handles missing contact in tool context' do + tool_context_no_contact = Struct.new(:state).new({ + account_id: account.id, + assistant_id: assistant.id, + conversation: { + id: conversation.id, + display_id: conversation.display_id + } + }) + + stub_request(:get, 'https://example.com/api/data') + .with(headers: { + 'X-Chatwoot-Account-Id' => account.id.to_s, + 'X-Chatwoot-Conversation-Id' => conversation.id.to_s + }) + .to_return(status: 200, body: '{"success": true}') + + tool.perform(tool_context_no_contact) + + expect(WebMock).to have_requested(:get, 'https://example.com/api/data') + end + + it 'includes contact phone when present' do + contact.update!(phone_number: '+1234567890') + tool_context_with_state.state[:contact][:phone_number] = '+1234567890' + + stub_request(:get, 'https://example.com/api/data') + .with(headers: { + 'X-Chatwoot-Contact-Phone' => '+1234567890' + }) + .to_return(status: 200, body: '{"success": true}') + + tool.perform(tool_context_with_state) + + expect(WebMock).to have_requested(:get, 'https://example.com/api/data') + .with(headers: { 'X-Chatwoot-Contact-Phone' => '+1234567890' }) + end + end end end diff --git a/spec/enterprise/models/captain/custom_tool_spec.rb b/spec/enterprise/models/captain/custom_tool_spec.rb index 5f6c7b19a..f936eeaa5 100644 --- a/spec/enterprise/models/captain/custom_tool_spec.rb +++ b/spec/enterprise/models/captain/custom_tool_spec.rb @@ -327,6 +327,98 @@ RSpec.describe Captain::CustomTool, type: :model do end end + describe '#build_metadata_headers' do + let(:tool) { create(:captain_custom_tool, account: account, slug: 'custom_test_tool') } + let(:conversation) { create(:conversation, account: account) } + let(:contact) { conversation.contact } + + let(:state) do + { + account_id: account.id, + assistant_id: 123, + conversation: { + id: conversation.id, + display_id: conversation.display_id + }, + contact: { + id: contact.id, + email: contact.email, + phone_number: contact.phone_number + } + } + end + + it 'includes account and assistant metadata' do + headers = tool.build_metadata_headers(state) + + expect(headers['X-Chatwoot-Account-Id']).to eq(account.id.to_s) + expect(headers['X-Chatwoot-Assistant-Id']).to eq('123') + end + + it 'includes tool slug' do + headers = tool.build_metadata_headers(state) + + expect(headers['X-Chatwoot-Tool-Slug']).to eq('custom_test_tool') + end + + it 'includes conversation metadata when present' do + headers = tool.build_metadata_headers(state) + + expect(headers['X-Chatwoot-Conversation-Id']).to eq(conversation.id.to_s) + expect(headers['X-Chatwoot-Conversation-Display-Id']).to eq(conversation.display_id.to_s) + end + + it 'includes contact metadata when present' do + headers = tool.build_metadata_headers(state) + + expect(headers['X-Chatwoot-Contact-Id']).to eq(contact.id.to_s) + expect(headers['X-Chatwoot-Contact-Email']).to eq(contact.email) + end + + it 'handles missing conversation gracefully' do + state[:conversation] = nil + + headers = tool.build_metadata_headers(state) + + expect(headers['X-Chatwoot-Conversation-Id']).to be_nil + expect(headers['X-Chatwoot-Conversation-Display-Id']).to be_nil + expect(headers['X-Chatwoot-Account-Id']).to eq(account.id.to_s) + end + + it 'handles missing contact gracefully' do + state[:contact] = nil + + headers = tool.build_metadata_headers(state) + + expect(headers['X-Chatwoot-Contact-Id']).to be_nil + expect(headers['X-Chatwoot-Contact-Email']).to be_nil + expect(headers['X-Chatwoot-Account-Id']).to eq(account.id.to_s) + end + + it 'handles empty state' do + headers = tool.build_metadata_headers({}) + + expect(headers).to be_a(Hash) + expect(headers['X-Chatwoot-Tool-Slug']).to eq('custom_test_tool') + end + + it 'omits contact email header when email is blank' do + state[:contact][:email] = '' + + headers = tool.build_metadata_headers(state) + + expect(headers).not_to have_key('X-Chatwoot-Contact-Email') + end + + it 'omits contact phone header when phone number is blank' do + state[:contact][:phone_number] = '' + + headers = tool.build_metadata_headers(state) + + expect(headers).not_to have_key('X-Chatwoot-Contact-Phone') + end + end + describe '#to_tool_metadata' do it 'returns tool metadata hash with custom flag' do tool = create(:captain_custom_tool, account: account, From 89d02e2c92d42c4e8f531a5ab1983e86679fe5b3 Mon Sep 17 00:00:00 2001 From: Vinay Keerthi <11478411+stonecharioteer@users.noreply.github.com> Date: Wed, 10 Dec 2025 21:44:16 +0530 Subject: [PATCH 4/5] fix: Preserve multiple newlines with whitespace in text-based messaging channels (#13044) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Fixes an issue where multiple newlines with whitespace between them (e.g., `\n \n \n`) were being collapsed to single newlines in text-based messaging channels (Telegram, WhatsApp, Instagram, Facebook, Line, SMS). The frontend was sending messages with spaces/tabs between newlines, and the markdown renderer was treating these as paragraph content, collapsing them during rendering. ### Changes: 1. Added whitespace normalization in `render_telegram_html`, `render_whatsapp`, `render_instagram`, `render_line`, and `render_plain_text` methods 2. Strips whitespace from whitespace-only lines before markdown processing 3. Added comprehensive regression tests for all affected channels ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? 1. **Unit Tests**: Added 7 new specs testing multiple newlines with whitespace between them for all text-based channels 2. **Manual Testing**: Verified with actual frontend payload containing `\n \n \n` patterns 3. **Regression Testing**: All existing 63 specs pass ### Test Results: - ✅ All 63 markdown renderer specs pass (56 original + 7 new) - ✅ All 12 Telegram channel specs pass - ✅ All 27 WhatsApp + Instagram specs pass - ✅ Verified with real-world payload: 18 newlines preserved (previously collapsed to 1) ### Test Command: ```bash RAILS_ENV=test bundle exec rspec spec/services/messages/markdown_renderer_service_spec.rb RAILS_ENV=test bundle exec rspec spec/models/channel/telegram_spec.rb ``` ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective - [x] New and existing unit tests pass locally with my changes --- .../messages/markdown_renderer_service.rb | 24 +++++++++---- .../markdown_renderer_service_spec.rb | 35 +++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/app/services/messages/markdown_renderer_service.rb b/app/services/messages/markdown_renderer_service.rb index 8984374d1..a90cfca06 100644 --- a/app/services/messages/markdown_renderer_service.rb +++ b/app/services/messages/markdown_renderer_service.rb @@ -47,13 +47,19 @@ class Messages::MarkdownRendererService end def render_telegram_html + # Strip whitespace from whitespace-only lines to normalize newlines + normalized_content = @content.gsub(/^[ \t]+$/m, '') + content_with_preserved_newlines = preserve_multiple_newlines(normalized_content) renderer = Messages::MarkdownRenderers::TelegramRenderer.new - doc = CommonMarker.render_doc(@content, [:STRIKETHROUGH_DOUBLE_TILDE], [:strikethrough]) - renderer.render(doc).gsub(/\n+\z/, '') + doc = CommonMarker.render_doc(content_with_preserved_newlines, [:STRIKETHROUGH_DOUBLE_TILDE], [:strikethrough]) + result = renderer.render(doc).gsub(/\n+\z/, '') + restore_multiple_newlines(result) end def render_whatsapp - content_with_preserved_newlines = preserve_multiple_newlines(@content) + # Strip whitespace from whitespace-only lines to normalize newlines + normalized_content = @content.gsub(/^[ \t]+$/m, '') + content_with_preserved_newlines = preserve_multiple_newlines(normalized_content) renderer = Messages::MarkdownRenderers::WhatsAppRenderer.new doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) result = renderer.render(doc).gsub(/\n+\z/, '') @@ -61,7 +67,9 @@ class Messages::MarkdownRendererService end def render_instagram - content_with_preserved_newlines = preserve_multiple_newlines(@content) + # Strip whitespace from whitespace-only lines to normalize newlines + normalized_content = @content.gsub(/^[ \t]+$/m, '') + content_with_preserved_newlines = preserve_multiple_newlines(normalized_content) renderer = Messages::MarkdownRenderers::InstagramRenderer.new doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) result = renderer.render(doc).gsub(/\n+\z/, '') @@ -69,7 +77,9 @@ class Messages::MarkdownRendererService end def render_line - content_with_preserved_newlines = preserve_multiple_newlines(@content) + # Strip whitespace from whitespace-only lines to normalize newlines + normalized_content = @content.gsub(/^[ \t]+$/m, '') + content_with_preserved_newlines = preserve_multiple_newlines(normalized_content) renderer = Messages::MarkdownRenderers::LineRenderer.new doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) result = renderer.render(doc).gsub(/\n+\z/, '') @@ -77,7 +87,9 @@ class Messages::MarkdownRendererService end def render_plain_text - content_with_preserved_newlines = preserve_multiple_newlines(@content) + # Strip whitespace from whitespace-only lines to normalize newlines + normalized_content = @content.gsub(/^[ \t]+$/m, '') + content_with_preserved_newlines = preserve_multiple_newlines(normalized_content) renderer = Messages::MarkdownRenderers::PlainTextRenderer.new doc = CommonMarker.render_doc(content_with_preserved_newlines, [:DEFAULT, :STRIKETHROUGH_DOUBLE_TILDE]) result = renderer.render(doc).gsub(/\n+\z/, '') diff --git a/spec/services/messages/markdown_renderer_service_spec.rb b/spec/services/messages/markdown_renderer_service_spec.rb index d589c6ad1..89210dc0e 100644 --- a/spec/services/messages/markdown_renderer_service_spec.rb +++ b/spec/services/messages/markdown_renderer_service_spec.rb @@ -426,5 +426,40 @@ RSpec.describe Messages::MarkdownRendererService, type: :service do expect(result).to eq(content) end end + + # Shared test for all text-based channels that preserve multiple newlines + # This tests the real-world scenario where frontend sends newlines with whitespace between them + context 'when content has multiple newlines with whitespace between them' do + # This mimics what frontends often send: newlines with spaces/tabs between them + let(:content_with_whitespace_newlines) { "hello \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\nhello wow" } + + %w[ + Channel::Telegram + Channel::Whatsapp + Channel::Instagram + Channel::FacebookPage + Channel::Line + Channel::Sms + ].each do |channel_type| + context "when channel is #{channel_type}" do + it 'normalizes whitespace-only lines and preserves multiple newlines' do + result = described_class.new(content_with_whitespace_newlines, channel_type).render + # Should preserve most of the newlines (at least 10+) + # The exact count may vary slightly by renderer, but should be significantly more than 1-2 + expect(result.scan("\n").count).to be >= 10 + # Should not collapse everything to just 1-2 newlines + expect(result.scan("\n").count).to be > 5 + end + end + end + + context 'when channel is Channel::TwilioSms with WhatsApp' do + it 'normalizes whitespace-only lines and preserves multiple newlines' do + channel = instance_double(Channel::TwilioSms, whatsapp?: true) + result = described_class.new(content_with_whitespace_newlines, 'Channel::TwilioSms', channel).render + expect(result.scan("\n").count).to be >= 10 + end + end + end end end From f2054e703aa01b276a6981dfa9585a1ed685b928 Mon Sep 17 00:00:00 2001 From: Sivin Varghese <64252451+iamsivin@users.noreply.github.com> Date: Wed, 10 Dec 2025 23:13:04 +0530 Subject: [PATCH 5/5] fix: Handle rich message signatures & attachment overflow (#13045) --- .../components/ActionButtons.vue | 26 +++--- .../components/AttachmentPreviews.vue | 2 +- .../components/ComposeNewConversationForm.vue | 14 +-- .../components/MessageEditor.vue | 11 +-- .../components/widgets/WootWriter/Editor.vue | 2 +- .../widgets/conversation/ReplyBox.vue | 14 ++- app/javascript/dashboard/constants/editor.js | 11 ++- .../dashboard/helper/editorHelper.js | 89 ++++++++++++------ .../helper/specs/editorHelper.spec.js | 91 +++++++++++++++++++ 9 files changed, 197 insertions(+), 63 deletions(-) diff --git a/app/javascript/dashboard/components-next/NewConversation/components/ActionButtons.vue b/app/javascript/dashboard/components-next/NewConversation/components/ActionButtons.vue index 92c5850de..cb1f9d99d 100644 --- a/app/javascript/dashboard/components-next/NewConversation/components/ActionButtons.vue +++ b/app/javascript/dashboard/components-next/NewConversation/components/ActionButtons.vue @@ -7,7 +7,7 @@ import { vOnClickOutside } from '@vueuse/components'; import { ALLOWED_FILE_TYPES } from 'shared/constants/messages'; import { useKeyboardEvents } from 'dashboard/composables/useKeyboardEvents'; import FileUpload from 'vue-upload-component'; -import { extractTextFromMarkdown } from 'dashboard/helper/editorHelper'; +import { INBOX_TYPES } from 'dashboard/helper/inbox'; import Button from 'dashboard/components-next/button/Button.vue'; import WhatsAppOptions from './WhatsAppOptions.vue'; @@ -50,12 +50,6 @@ const EmojiInput = defineAsyncComponent( () => import('shared/components/emoji/EmojiInput.vue') ); -const signatureToApply = computed(() => - props.isEmailOrWebWidgetInbox - ? props.messageSignature - : extractTextFromMarkdown(props.messageSignature) -); - const { fetchSignatureFlagFromUISettings, setSignatureFlagForInbox, @@ -80,12 +74,20 @@ const isRegularMessageMode = computed(() => { return !props.isWhatsappInbox && !props.isTwilioWhatsAppInbox; }); +const isVoiceInbox = computed(() => props.channelType === INBOX_TYPES.VOICE); + +const shouldShowSignatureButton = computed(() => { + return ( + props.hasSelectedInbox && isRegularMessageMode.value && !isVoiceInbox.value + ); +}); + const setSignature = () => { - if (signatureToApply.value) { + if (props.messageSignature) { if (sendWithSignature.value) { - emit('addSignature', signatureToApply.value); + emit('addSignature', props.messageSignature); } else { - emit('removeSignature', signatureToApply.value); + emit('removeSignature', props.messageSignature); } } }; @@ -101,7 +103,7 @@ watch( () => props.hasSelectedInbox, newValue => { nextTick(() => { - if (newValue && props.isEmailOrWebWidgetInbox) setSignature(); + if (newValue && !isVoiceInbox.value) setSignature(); }); }, { immediate: true } @@ -220,7 +222,7 @@ useKeyboardEvents(keyboardEvents); />