From 02ab856520e7246cbfbaede557601e0adb3e64a3 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 7 Jan 2026 12:45:54 +0530 Subject: [PATCH 1/3] feat(CW-6187): include headers from incoming emails (#13139) --- app/presenters/mail_presenter.rb | 11 +++++++++ spec/mailboxes/reply_mailbox_spec.rb | 6 ++--- spec/presenters/mail_presenter_spec.rb | 34 ++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/app/presenters/mail_presenter.rb b/app/presenters/mail_presenter.rb index 202c255a1..08e370cd8 100644 --- a/app/presenters/mail_presenter.rb +++ b/app/presenters/mail_presenter.rb @@ -95,6 +95,7 @@ class MailPresenter < SimpleDelegator content_type: content_type, date: date, from: from, + headers: headers_data, html_content: html_content, in_reply_to: in_reply_to, message_id: message_id, @@ -136,6 +137,16 @@ class MailPresenter < SimpleDelegator from_email_address(@mail[:reply_to].try(:value)) || @mail['X-Original-Sender'].try(:value) || from_email_address(from.first) end + def headers_data + headers = { + 'x-original-from' => @mail['X-Original-From']&.value, + 'x-original-sender' => @mail['X-Original-Sender']&.value, + 'x-forwarded-for' => @mail['X-Forwarded-For']&.value + }.compact + + headers.presence + end + def from_email_address(email) Mail::Address.new(email).address end diff --git a/spec/mailboxes/reply_mailbox_spec.rb b/spec/mailboxes/reply_mailbox_spec.rb index 2658731bc..d062c7d73 100644 --- a/spec/mailboxes/reply_mailbox_spec.rb +++ b/spec/mailboxes/reply_mailbox_spec.rb @@ -12,8 +12,8 @@ RSpec.describe ReplyMailbox do let(:conversation) { create(:conversation, assignee: agent, inbox: create(:inbox, account: account, greeting_enabled: false), account: account) } let(:described_subject) { described_class.receive reply_mail } let(:serialized_attributes) do - %w[bcc cc content_type date from html_content in_reply_to message_id multipart number_of_attachments references subject text_content to - auto_reply] + %w[bcc cc content_type date from headers html_content in_reply_to message_id multipart number_of_attachments references subject text_content + to auto_reply] end context 'with reply uuid present' do @@ -397,7 +397,7 @@ RSpec.describe ReplyMailbox do let(:support_in_reply_to_mail) { create_inbound_email_from_fixture('support_in_reply_to.eml') } let(:described_subject) { described_class.receive support_mail } let(:serialized_attributes) do - %w[bcc cc content_type date from html_content in_reply_to message_id multipart number_of_attachments references subject + %w[bcc cc content_type date from headers html_content in_reply_to message_id multipart number_of_attachments references subject text_content to auto_reply] end let(:conversation) { Conversation.where(inbox_id: channel_email.inbox).last } diff --git a/spec/presenters/mail_presenter_spec.rb b/spec/presenters/mail_presenter_spec.rb index b9b23c8ce..bbe51c52d 100644 --- a/spec/presenters/mail_presenter_spec.rb +++ b/spec/presenters/mail_presenter_spec.rb @@ -41,6 +41,7 @@ RSpec.describe MailPresenter do :content_type, :date, :from, + :headers, :html_content, :in_reply_to, :message_id, @@ -60,6 +61,39 @@ RSpec.describe MailPresenter do expect(data[:auto_reply]).to eq(decorated_mail.auto_reply?) end + it 'includes forwarded headers in serialized_data' do + mail_with_headers = Mail.new do + from 'Sender ' + to 'Inbox ' + subject :header + body 'Hi' + header['X-Original-From'] = 'Original ' + header['X-Original-Sender'] = 'original@example.com' + header['X-Forwarded-For'] = 'forwarder@example.com' + end + + data = described_class.new(mail_with_headers).serialized_data + + expect(data[:headers]).to eq( + 'x-original-from' => 'Original ', + 'x-original-sender' => 'original@example.com', + 'x-forwarded-for' => 'forwarder@example.com' + ) + end + + it 'returns nil headers when forwarding headers are missing' do + mail_without_headers = Mail.new do + from 'Sender ' + to 'Inbox ' + subject :header + body 'Hi' + end + + data = described_class.new(mail_without_headers).serialized_data + + expect(data[:headers]).to be_nil + end + it 'give email from in downcased format' do expect(decorated_mail.from.first.eql?(mail.from.first.downcase)).to be true end From 566de02385ce216ae25e7831411db105ad81ca8c Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 7 Jan 2026 13:57:43 +0530 Subject: [PATCH 2/3] feat: allow agent bot and captain responses to reset waiting since (#13181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When AgentBot responds to customer messages, the `waiting_since` timestamp is not reset, causing inflated reply time metrics when a human agent eventually responds. This results in inaccurate reporting that incorrectly includes periods when customers were satisfied with bot responses. ### Timeline from Production Data ``` Dec 12, 16:20:14 - Customer sends message (ID: 368451924) ↓ waiting_since = Dec 12, 16:20:14 Dec 12, 16:20:17 - AgentBot replies (ID: 368451960) ↓ waiting_since STILL = Dec 12, 16:20:14 ❌ ↓ (Bot response doesn't clear it) 14-day gap - Customer satisfied, no messages ↓ waiting_since STILL = Dec 12, 16:20:14 ❌ Dec 26, 22:25:45 - Customer sends new message (ID: 383522275) ↓ waiting_since STILL = Dec 12, 16:20:14 ❌ ↓ (New message doesn't reset it) Dec 26-27 - More AgentBot interactions ↓ waiting_since STILL = Dec 12, 16:20:14 ❌ Dec 27, 07:36:53 - Human agent finally replies (ID: 383799517) ↓ Reply time calculated: 1,268,404 seconds ↓ = 14.7 DAYS ❌ ``` ## Root Cause The core issues is in `app/models/message.rb`, where **AgentBot messages does not clear `waiting_since`** - The `human_response?` method only returns true for `User` senders, so bot replies never trigger the clearing logic. This means once `waiting_since` is set, it stays set even when customers send new messages after receiving bot responses. The solution is to simply reset `waiting_since` **after a bot has responded**. This ensures reply time metrics reflect actual human agent response times, not bot-handled periods. ### What triggers the rest This is an intentional "gotcha", that only `AgentBot` and `Captain::Assistant` messages trigger the waiting time reset. Automation and campaign messages maintain current behavior (no reset). This is because interactive bot assistants provide conversational help that might satisfy customers. Automation and campaigns are one-way communications and shouldn't affect waiting time calculations. ## Related Work Extends PR #11787 which fixed `waiting_since` clearing on conversation resolution. This PR addresses the bot interaction scenario which was not covered by that fix. Scripts to clean data: https://gist.github.com/scmmishra/bd133208e219d0ab52fbfdf03036c48a --- app/models/message.rb | 24 ++++++++-- spec/enterprise/models/message_spec.rb | 3 +- spec/models/conversation_spec.rb | 65 ++++++++++++++++++++++++++ spec/models/message_spec.rb | 57 ++++++++++++++++++++++ 4 files changed, 143 insertions(+), 6 deletions(-) diff --git a/app/models/message.rb b/app/models/message.rb index 0bf7a176b..3be04c54f 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -322,12 +322,21 @@ class Message < ApplicationRecord end def update_waiting_since - if human_response? && !private && conversation.waiting_since.present? - Rails.configuration.dispatcher.dispatch( - REPLY_CREATED, Time.zone.now, waiting_since: conversation.waiting_since, message: self - ) - conversation.update(waiting_since: nil) + waiting_present = conversation.waiting_since.present? + + if waiting_present && !private + if human_response? + Rails.configuration.dispatcher.dispatch( + REPLY_CREATED, Time.zone.now, waiting_since: conversation.waiting_since, message: self + ) + conversation.update(waiting_since: nil) + elsif bot_response? + # Bot responses also clear waiting_since (simpler than checking on next customer message) + conversation.update(waiting_since: nil) + end end + + # Set waiting_since when customer sends a message (if currently blank) conversation.update(waiting_since: created_at) if incoming? && conversation.waiting_since.blank? end @@ -341,6 +350,11 @@ class Message < ApplicationRecord sender.is_a?(User) end + def bot_response? + # Check if this is a response from AgentBot or Captain::Assistant + outgoing? && sender_type.in?(['AgentBot', 'Captain::Assistant']) + end + def dispatch_create_events Rails.configuration.dispatcher.dispatch(MESSAGE_CREATED, Time.zone.now, message: self, performed_by: Current.executed_by) diff --git a/spec/enterprise/models/message_spec.rb b/spec/enterprise/models/message_spec.rb index 5a9dc4e27..aa1537e65 100644 --- a/spec/enterprise/models/message_spec.rb +++ b/spec/enterprise/models/message_spec.rb @@ -14,8 +14,9 @@ RSpec.describe Message do create(:message, message_type: :outgoing, conversation: conversation, sender: captain_assistant) + # Captain::Assistant responses clear waiting_since (like AgentBot) expect(conversation.first_reply_created_at).to be_nil - expect(conversation.waiting_since).to be_within(0.000001.seconds).of(conversation.created_at) + expect(conversation.waiting_since).to be_nil create(:message, message_type: :outgoing, conversation: conversation) diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb index 7a5398456..5a4acf329 100644 --- a/spec/models/conversation_spec.rb +++ b/spec/models/conversation_spec.rb @@ -978,5 +978,70 @@ RSpec.describe Conversation do reply_events = account.reporting_events.where(name: 'reply_time', conversation_id: conversation.id) expect(reply_events.count).to eq(0) end + + context 'when AgentBot responds between customer messages' do + let(:agent_bot) { create(:agent_bot, account: account) } + + def create_bot_message(conversation, created_at: Time.current) + message = nil + perform_enqueued_jobs do + message = create(:message, + message_type: 'outgoing', + account: conversation.account, + inbox: conversation.inbox, + conversation: conversation, + sender: agent_bot, + created_at: created_at) + end + message + end + + it 'calculates reply time from the most recent customer message after bot response' do + # Initial conversation: customer message -> agent first reply (to establish first_reply_created_at) + create_customer_message(conversation, created_at: 10.hours.ago) + create_agent_message(conversation, created_at: 9.hours.ago) + + # Customer message 1 + create_customer_message(conversation, created_at: 5.hours.ago) + + # Bot responds + create_bot_message(conversation, created_at: 4.hours.ago) + + # Customer message 2 (after bot response) - should reset waiting_since + create_customer_message(conversation, created_at: 2.hours.ago) + + # Human agent replies - should create reply_time event from customer message 2 + create_agent_message(conversation, created_at: 1.hour.ago) + + reply_events = account.reporting_events.where(name: 'reply_time', conversation_id: conversation.id) + expect(reply_events.count).to eq(1) # Only the second agent reply creates a reply_time event + # Reply time should be 1 hour (from customer message 2 to agent reply) + expect(reply_events.first.value).to be_within(60).of(3600) + end + + it 'handles multiple bot responses before customer messages again' do + # Initial conversation: customer message -> agent first reply + create_customer_message(conversation, created_at: 10.hours.ago) + create_agent_message(conversation, created_at: 9.hours.ago) + + # Customer message 1 + create_customer_message(conversation, created_at: 6.hours.ago) + + # Bot responds multiple times + create_bot_message(conversation, created_at: 5.hours.ago) + create_bot_message(conversation, created_at: 4.hours.ago) + + # Customer message 2 (after multiple bot responses) - should reset waiting_since + create_customer_message(conversation, created_at: 2.hours.ago) + + # Human agent replies + create_agent_message(conversation, created_at: 1.hour.ago) + + reply_events = account.reporting_events.where(name: 'reply_time', conversation_id: conversation.id) + expect(reply_events.count).to eq(1) # Only the second agent reply creates a reply_time event + # Reply time should be 1 hour (from customer message 2 to agent reply) + expect(reply_events.first.value).to be_within(60).of(3600) + end + end end end diff --git a/spec/models/message_spec.rb b/spec/models/message_spec.rb index 8905f5103..d606c266d 100644 --- a/spec/models/message_spec.rb +++ b/spec/models/message_spec.rb @@ -300,6 +300,63 @@ RSpec.describe Message do expect(conversation.waiting_since).to eq old_waiting_since end + + context 'when bot has responded to the conversation' do + let(:agent_bot) { create(:agent_bot, account: conversation.account) } + + before do + # Create initial customer message + create(:message, conversation: conversation, message_type: :incoming, + created_at: 2.hours.ago) + conversation.update(waiting_since: 2.hours.ago) + + # Bot responds + create(:message, conversation: conversation, message_type: :outgoing, + sender: agent_bot, created_at: 1.hour.ago) + end + + it 'resets waiting_since when customer sends a new message after bot response' do + new_message = build(:message, conversation: conversation, message_type: :incoming) + new_message.save! + + conversation.reload + expect(conversation.waiting_since).to be_within(1.second).of(new_message.created_at) + end + + it 'does not reset waiting_since if last response was from human agent' do + # Human agent responds (clears waiting_since) + create(:message, conversation: conversation, message_type: :outgoing, + sender: agent) + conversation.reload + expect(conversation.waiting_since).to be_nil + + # Customer sends new message + new_message = build(:message, conversation: conversation, message_type: :incoming) + new_message.save! + + conversation.reload + expect(conversation.waiting_since).to be_within(1.second).of(new_message.created_at) + end + + it 'clears waiting_since when bot responds' do + # After the bot response in before block, waiting_since should already be cleared + conversation.reload + expect(conversation.waiting_since).to be_nil + + # Customer sends another message + create(:message, conversation: conversation, message_type: :incoming, + created_at: 30.minutes.ago) + conversation.reload + expect(conversation.waiting_since).to be_within(1.second).of(30.minutes.ago) + + # Another bot response should clear it again + create(:message, conversation: conversation, message_type: :outgoing, + sender: agent_bot, created_at: 15.minutes.ago) + + conversation.reload + expect(conversation.waiting_since).to be_nil + end + end end context 'with webhook_data' do From 59cbf57e2022808573455a2e536b5ec2d6d49de4 Mon Sep 17 00:00:00 2001 From: Vinay Keerthi <11478411+stonecharioteer@users.noreply.github.com> Date: Wed, 7 Jan 2026 15:30:49 +0530 Subject: [PATCH 3/3] feat: Advanced Search Backend (#12917) ## Description Implements comprehensive search functionality with advanced filtering capabilities for Chatwoot (Linear: CW-5956). This PR adds: 1. **Time-based filtering** for contacts and conversations (SQL-based search) 2. **Advanced message search** with multiple filters (OpenSearch/Elasticsearch-based) - **`from` filter**: Filter messages by sender (format: `contact:42` or `agent:5`) - **`inbox_id` filter**: Filter messages by specific inbox - **Time range filters**: Filter messages using `since` and `until` parameters (Unix timestamps in seconds) - **90-day limit enforcement**: Automatically limits searches to the last 90 days to prevent performance issues The implementation extends the existing `Enterprise::SearchService` module for advanced features and adds time filtering to the base `SearchService` for SQL-based searches. ## API Documentation ### Base URL All search endpoints follow this pattern: ``` GET /api/v1/accounts/{account_id}/search/{resource} ``` ### Authentication All requests require authentication headers: ``` api_access_token: YOUR_ACCESS_TOKEN ``` --- ## 1. Search All Resources **Endpoint:** `GET /api/v1/accounts/{account_id}/search` Returns results from all searchable resources (contacts, conversations, messages, articles). ### Parameters | Parameter | Type | Description | Required | |-----------|------|-------------|----------| | `q` | string | Search query | Yes | | `page` | integer | Page number (15 items per page) | No | | `since` | integer | Unix timestamp (contacts/conversations only) | No | | `until` | integer | Unix timestamp (contacts/conversations only) | No | ### Example Request ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search?q=customer" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` ### Example Response ```json { "payload": { "contacts": [...], "conversations": [...], "messages": [...], "articles": [...] } } ``` --- ## 2. Search Contacts **Endpoint:** `GET /api/v1/accounts/{account_id}/search/contacts` Search contacts by name, email, phone number, or identifier with optional time filtering. ### Parameters | Parameter | Type | Description | Required | |-----------|------|-------------|----------| | `q` | string | Search query | Yes | | `page` | integer | Page number (15 items per page) | No | | `since` | integer | Unix timestamp - filter by last_activity_at | No | | `until` | integer | Unix timestamp - filter by last_activity_at | No | ### Example Requests **Basic search:** ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/contacts?q=john" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search contacts active in the last 7 days:** ```bash SINCE=$(date -v-7d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/contacts?q=john&since=${SINCE}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search contacts active between 30 and 7 days ago:** ```bash SINCE=$(date -v-30d +%s) UNTIL=$(date -v-7d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/contacts?q=john&since=${SINCE}&until=${UNTIL}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` ### Example Response ```json { "payload": { "contacts": [ { "id": 42, "email": "john@example.com", "name": "John Doe", "phone_number": "+1234567890", "identifier": "user_123", "additional_attributes": {}, "created_at": 1701234567 } ] } } ``` --- ## 3. Search Conversations **Endpoint:** `GET /api/v1/accounts/{account_id}/search/conversations` Search conversations by display ID, contact name, email, phone number, or identifier with optional time filtering. ### Parameters | Parameter | Type | Description | Required | |-----------|------|-------------|----------| | `q` | string | Search query | Yes | | `page` | integer | Page number (15 items per page) | No | | `since` | integer | Unix timestamp - filter by last_activity_at | No | | `until` | integer | Unix timestamp - filter by last_activity_at | No | ### Example Requests **Basic search:** ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/conversations?q=billing" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search conversations active in the last 24 hours:** ```bash SINCE=$(date -v-1d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/conversations?q=billing&since=${SINCE}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search conversations from last month:** ```bash SINCE=$(date -v-30d +%s) UNTIL=$(date +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/conversations?q=billing&since=${SINCE}&until=${UNTIL}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` ### Example Response ```json { "payload": { "conversations": [ { "id": 123, "display_id": 45, "inbox_id": 1, "status": "open", "messages": [...], "meta": {...} } ] } } ``` --- ## 4. Search Messages (Advanced) **Endpoint:** `GET /api/v1/accounts/{account_id}/search/messages` Advanced message search with multiple filters powered by OpenSearch/Elasticsearch. ### Prerequisites - OpenSearch/Elasticsearch must be running (`OPENSEARCH_URL` env var configured) - Account must have `advanced_search` feature flag enabled - Messages must be indexed in OpenSearch ### Parameters | Parameter | Type | Description | Required | |-----------|------|-------------|----------| | `q` | string | Search query | Yes | | `page` | integer | Page number (15 items per page) | No | | `from` | string | Filter by sender: `contact:{id}` or `agent:{id}` | No | | `inbox_id` | integer | Filter by specific inbox ID | No | | `since` | integer | Unix timestamp - searches from this time (max 90 days ago) | No | | `until` | integer | Unix timestamp - searches until this time | No | ### Important Notes - **90-Day Limit**: If `since` is not provided, searches default to the last 90 days - If `since` exceeds 90 days, returns `422` error: "Search is limited to the last 90 days" - All time filters use message `created_at` timestamp ### Example Requests **Basic message search:** ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search messages from a specific contact:** ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&from=contact:42" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search messages from a specific agent:** ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&from=agent:5" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search messages in a specific inbox:** ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&inbox_id=3" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search messages from the last 7 days:** ```bash SINCE=$(date -v-7d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&since=${SINCE}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Search messages between specific dates:** ```bash SINCE=$(date -v-30d +%s) UNTIL=$(date -v-7d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&since=${SINCE}&until=${UNTIL}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Combine all filters:** ```bash SINCE=$(date -v-14d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&from=contact:42&inbox_id=3&since=${SINCE}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` **Attempt to search beyond 90 days (returns error):** ```bash SINCE=$(date -v-120d +%s) curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/messages?q=refund&since=${SINCE}" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` ### Example Response (Success) ```json { "payload": { "messages": [ { "id": 789, "content": "I need a refund for my purchase", "message_type": "incoming", "created_at": 1701234567, "conversation_id": 123, "inbox_id": 3, "sender": { "id": 42, "type": "contact" } } ] } } ``` ### Example Response (90-day limit exceeded) ```json { "error": "Search is limited to the last 90 days" } ``` **Status Code:** `422 Unprocessable Entity` --- ## 5. Search Articles **Endpoint:** `GET /api/v1/accounts/{account_id}/search/articles` Search help center articles by title or content. ### Parameters | Parameter | Type | Description | Required | |-----------|------|-------------|----------| | `q` | string | Search query | Yes | | `page` | integer | Page number (15 items per page) | No | ### Example Request ```bash curl -X GET "https://app.chatwoot.com/api/v1/accounts/1/search/articles?q=installation" \ -H "api_access_token: YOUR_ACCESS_TOKEN" ``` ### Example Response ```json { "payload": { "articles": [ { "id": 456, "title": "Installation Guide", "slug": "installation-guide", "portal_slug": "help", "account_id": 1, "category_name": "Getting Started", "status": "published", "updated_at": 1701234567 } ] } } ``` --- ## Technical Implementation ### SQL-Based Search (Contacts, Conversations, Articles) - Uses PostgreSQL `ILIKE` queries by default - Optional GIN index support via `search_with_gin` feature flag for better performance - Time filtering uses `last_activity_at` for contacts/conversations - Returns paginated results (15 per page) ### Advanced Search (Messages) - Powered by OpenSearch/Elasticsearch via Searchkick gem - Requires `OPENSEARCH_URL` environment variable - Requires `advanced_search` account feature flag - Enforces 90-day lookback limit via `Limits::MESSAGE_SEARCH_TIME_RANGE_LIMIT_DAYS` - Validates inbox access permissions before filtering - Returns paginated results (15 per page) --- ## Type of change - [x] New feature (non-breaking change which adds functionality) - [x] Enhancement (improves existing functionality) --- ## How Has This Been Tested? ### Unit Tests - **Contact Search Tests**: 3 new test cases for time filtering (`since`, `until`, combined) - **Conversation Search Tests**: 3 new test cases for time filtering - **Message Search Tests**: 10+ test cases covering: - Individual filters (`from`, `inbox_id`, time range) - Combined filters - Permission validation for inbox access - Feature flag checks - 90-day limit enforcement - Error handling for exceeded time limits ### Test Commands ```bash # Run all search controller tests bundle exec rspec spec/controllers/api/v1/accounts/search_controller_spec.rb # Run search service tests (includes enterprise specs) bundle exec rspec spec/services/search_service_spec.rb ``` ### Manual Testing Setup A rake task is provided to create 50,000 test messages across multiple inboxes: ```bash # 1. Create test data bundle exec rake search:setup_test_data # 2. Start OpenSearch mise elasticsearch-start # 3. Reindex messages rails runner "Message.search_index.import Message.all" # 4. Enable feature flag rails runner "Account.first.enable_features('advanced_search')" # 5. Test via API or Rails console ``` --- ## Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [x] I have commented on my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation (this PR description) - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- ## Additional Notes ### Requirements - **OpenSearch/Elasticsearch**: Required for advanced message search - Set `OPENSEARCH_URL` environment variable - Example: `export OPENSEARCH_URL=http://localhost:9200` - **Feature Flags**: - `advanced_search`: Account-level flag for message advanced search - `search_with_gin` (optional): Account-level flag for GIN-based SQL search ### Performance Considerations - 90-day limit prevents expensive long-range queries on large datasets - GIN indexes recommended for high-volume search on SQL-based resources - OpenSearch/Elasticsearch provides faster full-text search for messages ### Breaking Changes - None. All new parameters are optional and backward compatible. ### Frontend Integration - Frontend PR tracking advanced search UI will consume these endpoints - Time range pickers should convert JavaScript `Date` to Unix timestamps (seconds) - Date conversion: `Math.floor(date.getTime() / 1000)` ### Error Handling - Invalid `from` parameter format is silently ignored (filter not applied) - Time range exceeding 90 days returns `422` with error message - Missing `q` parameter returns `422` (existing behavior) - Unauthorized inbox access is filtered out (no error, just excluded from results) --------- Co-authored-by: iamsivin Co-authored-by: Sivin Varghese <64252451+iamsivin@users.noreply.github.com> Co-authored-by: Pranav Co-authored-by: Muhsin Keloth --- .circleci/config.yml | 44 ++ .../api/v1/accounts/search_controller.rb | 2 + app/javascript/dashboard/api/search.js | 18 +- .../dashboard/api/specs/search.spec.js | 134 +++++ .../dashboard/components-next/CardLayout.vue | 2 +- .../dropdown-menu/DropdownMenu.vue | 35 +- .../components-next/message/chips/Audio.vue | 6 +- .../components-next/tabbar/TabBar.vue | 24 +- .../dashboard/constants/localStorage.js | 1 + app/javascript/dashboard/featureFlags.js | 1 + .../dashboard/i18n/locale/en/search.json | 46 +- .../search/components/MessageContent.vue | 114 ++-- .../modules/search/components/ReadMore.vue | 38 -- .../search/components/RecentSearches.vue | 116 +++++ .../components/SearchContactAgentSelector.vue | 239 +++++++++ .../components/SearchDateRangeSelector.vue | 271 ++++++++++ .../search/components/SearchFilters.vue | 104 ++++ .../search/components/SearchHeader.vue | 119 ++--- .../search/components/SearchInboxSelector.vue | 125 +++++ .../modules/search/components/SearchInput.vue | 136 +++++ .../components/SearchResultArticleItem.vue | 88 +++- .../components/SearchResultArticlesList.vue | 7 +- .../components/SearchResultContactItem.vue | 138 ++++- .../components/SearchResultContactsList.vue | 6 +- .../SearchResultConversationItem.vue | 111 ++-- .../SearchResultConversationsList.vue | 9 +- .../components/SearchResultMessageItem.vue | 169 ++++++ .../components/SearchResultMessagesList.vue | 16 +- .../search/components/SearchResultSection.vue | 18 +- .../modules/search/components/SearchTabs.vue | 44 +- .../modules/search/components/SearchView.vue | 110 +++- .../search/components/TranscribedText.vue | 45 ++ .../modules/search/helpers/searchHelper.js | 75 +++ .../search/helpers/specs/searchHelper.spec.js | 379 ++++++++++++++ .../store/modules/conversationSearch.js | 33 +- .../specs/conversationSearch/actions.spec.js | 26 + .../specs/useExpandableContent.spec.js | 130 +++++ .../composables/useExpandableContent.js | 62 +++ app/services/search_service.rb | 117 ++++- .../v1/accounts/search/_article.json.jbuilder | 2 + .../v1/accounts/search/_contact.json.jbuilder | 2 + config/locales/en.yml | 3 + .../app/services/enterprise/search_service.rb | 77 ++- lib/limits.rb | 1 + lib/tasks/search_test_data.rake | 183 +++++++ .../api/v1/accounts/search_controller_spec.rb | 240 +++++++++ spec/lib/webhooks/trigger_spec.rb | 2 + spec/services/search_service_spec.rb | 493 ++++++++++++++++++ spec/support/opensearch_check.rb | 40 ++ 49 files changed, 3816 insertions(+), 385 deletions(-) create mode 100644 app/javascript/dashboard/api/specs/search.spec.js delete mode 100644 app/javascript/dashboard/modules/search/components/ReadMore.vue create mode 100644 app/javascript/dashboard/modules/search/components/RecentSearches.vue create mode 100644 app/javascript/dashboard/modules/search/components/SearchContactAgentSelector.vue create mode 100644 app/javascript/dashboard/modules/search/components/SearchDateRangeSelector.vue create mode 100644 app/javascript/dashboard/modules/search/components/SearchFilters.vue create mode 100644 app/javascript/dashboard/modules/search/components/SearchInboxSelector.vue create mode 100644 app/javascript/dashboard/modules/search/components/SearchInput.vue create mode 100644 app/javascript/dashboard/modules/search/components/SearchResultMessageItem.vue create mode 100644 app/javascript/dashboard/modules/search/components/TranscribedText.vue create mode 100644 app/javascript/dashboard/modules/search/helpers/searchHelper.js create mode 100644 app/javascript/dashboard/modules/search/helpers/specs/searchHelper.spec.js create mode 100644 app/javascript/shared/composables/specs/useExpandableContent.spec.js create mode 100644 app/javascript/shared/composables/useExpandableContent.js create mode 100644 lib/tasks/search_test_data.rake create mode 100644 spec/support/opensearch_check.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index 804c63857..24119bc75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -218,6 +218,49 @@ jobs: source ~/.rvm/scripts/rvm bundle install + # Install and configure OpenSearch + - run: + name: Install OpenSearch + command: | + # Download and install OpenSearch 2.11.0 (compatible with Elasticsearch 7.x clients) + wget https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.0/opensearch-2.11.0-linux-x64.tar.gz + tar -xzf opensearch-2.11.0-linux-x64.tar.gz + sudo mv opensearch-2.11.0 /opt/opensearch + + - run: + name: Configure and Start OpenSearch + command: | + # Configure OpenSearch for single-node testing + cat > /opt/opensearch/config/opensearch.yml \<< EOF + cluster.name: chatwoot-test + node.name: node-1 + network.host: 0.0.0.0 + http.port: 9200 + discovery.type: single-node + plugins.security.disabled: true + EOF + + # Set ownership and permissions + sudo chown -R $USER:$USER /opt/opensearch + + # Start OpenSearch in background + /opt/opensearch/bin/opensearch -d -p /tmp/opensearch.pid + + - run: + name: Wait for OpenSearch to be ready + command: | + echo "Waiting for OpenSearch to start..." + for i in {1..30}; do + if curl -s http://localhost:9200/_cluster/health | grep -q '"status"'; then + echo "OpenSearch is ready!" + exit 0 + fi + echo "Waiting... ($i/30)" + sleep 2 + done + echo "OpenSearch failed to start" + exit 1 + # Configure environment and database - run: name: Database Setup and Configure Environment Variables @@ -234,6 +277,7 @@ jobs: sed -i -e '/POSTGRES_USERNAME/ s/=.*/=chatwoot/' .env sed -i -e "/POSTGRES_PASSWORD/ s/=.*/=$pg_pass/" .env echo -en "\nINSTALLATION_ENV=circleci" >> ".env" + echo -en "\nOPENSEARCH_URL=http://localhost:9200" >> ".env" # Database setup - run: diff --git a/app/controllers/api/v1/accounts/search_controller.rb b/app/controllers/api/v1/accounts/search_controller.rb index 13e3a6a6c..7ee25e02e 100644 --- a/app/controllers/api/v1/accounts/search_controller.rb +++ b/app/controllers/api/v1/accounts/search_controller.rb @@ -28,5 +28,7 @@ class Api::V1::Accounts::SearchController < Api::V1::Accounts::BaseController search_type: search_type, params: params ).perform + rescue ArgumentError => e + render json: { error: e.message }, status: :unprocessable_entity end end diff --git a/app/javascript/dashboard/api/search.js b/app/javascript/dashboard/api/search.js index d533c2f28..10214f3f5 100644 --- a/app/javascript/dashboard/api/search.js +++ b/app/javascript/dashboard/api/search.js @@ -14,38 +14,48 @@ class SearchAPI extends ApiClient { }); } - contacts({ q, page = 1 }) { + contacts({ q, page = 1, since, until }) { return axios.get(`${this.url}/contacts`, { params: { q, page: page, + since, + until, }, }); } - conversations({ q, page = 1 }) { + conversations({ q, page = 1, since, until }) { return axios.get(`${this.url}/conversations`, { params: { q, page: page, + since, + until, }, }); } - messages({ q, page = 1 }) { + messages({ q, page = 1, since, until, from, inboxId }) { return axios.get(`${this.url}/messages`, { params: { q, page: page, + since, + until, + from, + inbox_id: inboxId, }, }); } - articles({ q, page = 1 }) { + articles({ q, page = 1, since, until }) { return axios.get(`${this.url}/articles`, { params: { q, page: page, + since, + until, }, }); } diff --git a/app/javascript/dashboard/api/specs/search.spec.js b/app/javascript/dashboard/api/specs/search.spec.js new file mode 100644 index 000000000..251ea760e --- /dev/null +++ b/app/javascript/dashboard/api/specs/search.spec.js @@ -0,0 +1,134 @@ +import searchAPI from '../search'; +import ApiClient from '../ApiClient'; + +describe('#SearchAPI', () => { + it('creates correct instance', () => { + expect(searchAPI).toBeInstanceOf(ApiClient); + expect(searchAPI).toHaveProperty('get'); + expect(searchAPI).toHaveProperty('contacts'); + expect(searchAPI).toHaveProperty('conversations'); + expect(searchAPI).toHaveProperty('messages'); + expect(searchAPI).toHaveProperty('articles'); + }); + + describe('API calls', () => { + const originalAxios = window.axios; + const axiosMock = { + get: vi.fn(() => Promise.resolve()), + }; + + beforeEach(() => { + window.axios = axiosMock; + }); + + afterEach(() => { + window.axios = originalAxios; + vi.clearAllMocks(); + }); + + it('#get', () => { + searchAPI.get({ q: 'test query' }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search', { + params: { q: 'test query' }, + }); + }); + + it('#contacts', () => { + searchAPI.contacts({ q: 'test', page: 1 }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search/contacts', { + params: { q: 'test', page: 1, since: undefined, until: undefined }, + }); + }); + + it('#contacts with date filters', () => { + searchAPI.contacts({ + q: 'test', + page: 2, + since: 1700000000, + until: 1732000000, + }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search/contacts', { + params: { q: 'test', page: 2, since: 1700000000, until: 1732000000 }, + }); + }); + + it('#conversations', () => { + searchAPI.conversations({ q: 'test', page: 1 }); + expect(axiosMock.get).toHaveBeenCalledWith( + '/api/v1/search/conversations', + { + params: { q: 'test', page: 1, since: undefined, until: undefined }, + } + ); + }); + + it('#conversations with date filters', () => { + searchAPI.conversations({ + q: 'test', + page: 1, + since: 1700000000, + until: 1732000000, + }); + expect(axiosMock.get).toHaveBeenCalledWith( + '/api/v1/search/conversations', + { + params: { q: 'test', page: 1, since: 1700000000, until: 1732000000 }, + } + ); + }); + + it('#messages', () => { + searchAPI.messages({ q: 'test', page: 1 }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search/messages', { + params: { + q: 'test', + page: 1, + since: undefined, + until: undefined, + from: undefined, + inbox_id: undefined, + }, + }); + }); + + it('#messages with all filters', () => { + searchAPI.messages({ + q: 'test', + page: 1, + since: 1700000000, + until: 1732000000, + from: 'contact:42', + inboxId: 10, + }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search/messages', { + params: { + q: 'test', + page: 1, + since: 1700000000, + until: 1732000000, + from: 'contact:42', + inbox_id: 10, + }, + }); + }); + + it('#articles', () => { + searchAPI.articles({ q: 'test', page: 1 }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search/articles', { + params: { q: 'test', page: 1, since: undefined, until: undefined }, + }); + }); + + it('#articles with date filters', () => { + searchAPI.articles({ + q: 'test', + page: 2, + since: 1700000000, + until: 1732000000, + }); + expect(axiosMock.get).toHaveBeenCalledWith('/api/v1/search/articles', { + params: { q: 'test', page: 2, since: 1700000000, until: 1732000000 }, + }); + }); + }); +}); diff --git a/app/javascript/dashboard/components-next/CardLayout.vue b/app/javascript/dashboard/components-next/CardLayout.vue index 462402167..166f5ea4c 100644 --- a/app/javascript/dashboard/components-next/CardLayout.vue +++ b/app/javascript/dashboard/components-next/CardLayout.vue @@ -19,7 +19,7 @@ const handleClick = () => { diff --git a/app/javascript/dashboard/components-next/message/chips/Audio.vue b/app/javascript/dashboard/components-next/message/chips/Audio.vue index 667d2d7b6..9c7a44b23 100644 --- a/app/javascript/dashboard/components-next/message/chips/Audio.vue +++ b/app/javascript/dashboard/components-next/message/chips/Audio.vue @@ -17,6 +17,10 @@ const { attachment } = defineProps({ type: Object, required: true, }, + showTranscribedText: { + type: Boolean, + default: true, + }, }); defineOptions({ @@ -182,7 +186,7 @@ const downloadAudio = async () => {
{{ attachment.transcribedText }} diff --git a/app/javascript/dashboard/components-next/tabbar/TabBar.vue b/app/javascript/dashboard/components-next/tabbar/TabBar.vue index 1b08b838c..fbe105825 100644 --- a/app/javascript/dashboard/components-next/tabbar/TabBar.vue +++ b/app/javascript/dashboard/components-next/tabbar/TabBar.vue @@ -1,5 +1,5 @@ diff --git a/app/javascript/dashboard/modules/search/components/ReadMore.vue b/app/javascript/dashboard/modules/search/components/ReadMore.vue deleted file mode 100644 index 7b4296c89..000000000 --- a/app/javascript/dashboard/modules/search/components/ReadMore.vue +++ /dev/null @@ -1,38 +0,0 @@ - - - diff --git a/app/javascript/dashboard/modules/search/components/RecentSearches.vue b/app/javascript/dashboard/modules/search/components/RecentSearches.vue new file mode 100644 index 000000000..0c52ea3ac --- /dev/null +++ b/app/javascript/dashboard/modules/search/components/RecentSearches.vue @@ -0,0 +1,116 @@ + + +