From 5171c813847ab2c3ae95d70e9e2588c0c58fb90f Mon Sep 17 00:00:00 2001 From: Pranav Date: Fri, 8 Nov 2024 14:21:59 -0800 Subject: [PATCH] Add elastic search to the query --- Gemfile | 3 ++ Gemfile.lock | 8 +++++ .../migration/search/message_reindex_job.rb | 5 +++ app/models/message.rb | 4 +++ app/services/search_service.rb | 21 +++++++++--- config/initializers/seach_config.rb | 32 +++++++++++++++++++ config/schedule.yml | 32 +++++++++---------- config/sidekiq.yml | 1 + 8 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 app/jobs/migration/search/message_reindex_job.rb create mode 100644 config/initializers/seach_config.rb diff --git a/Gemfile b/Gemfile index 2e1e3ec14..55b481d8a 100644 --- a/Gemfile +++ b/Gemfile @@ -175,6 +175,9 @@ gem 'pgvector' # Convert Website HTML to Markdown gem 'reverse_markdown' +gem 'opensearch-ruby' +gem 'searchkick' + ### Gems required only in specific deployment environments ### ############################################################## diff --git a/Gemfile.lock b/Gemfile.lock index 41140bdda..a59c9fadc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -532,6 +532,9 @@ GEM omniauth-rails_csrf_protection (1.0.2) actionpack (>= 4.2) omniauth (~> 2.0) + opensearch-ruby (3.4.0) + faraday (>= 1.0, < 3) + multi_json (>= 1.0) openssl (3.2.0) orm_adapter (0.5.0) os (1.1.4) @@ -703,6 +706,9 @@ GEM parser scss_lint (0.60.0) sass (~> 3.5, >= 3.5.5) + searchkick (5.4.0) + activemodel (>= 6.1) + hashie seed_dump (3.3.1) activerecord (>= 4) activesupport (>= 4) @@ -910,6 +916,7 @@ DEPENDENCIES omniauth-google-oauth2 (>= 1.1.3) omniauth-oauth2 omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2) + opensearch-ruby pg pg_search pgvector @@ -935,6 +942,7 @@ DEPENDENCIES rubocop-rspec scout_apm scss_lint + searchkick seed_dump sentry-rails (>= 5.19.0) sentry-ruby diff --git a/app/jobs/migration/search/message_reindex_job.rb b/app/jobs/migration/search/message_reindex_job.rb new file mode 100644 index 000000000..0c67e10bb --- /dev/null +++ b/app/jobs/migration/search/message_reindex_job.rb @@ -0,0 +1,5 @@ +class Migration::Search::MessageReindexJob < ApplicationJob + queue :searchkick + + +end diff --git a/app/models/message.rb b/app/models/message.rb index 6244d050c..8e5b45f55 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -38,6 +38,10 @@ # class Message < ApplicationRecord + searchkick batch_size: 10_000 if SearchConfig.enabled? + + scope :search_import, -> { includes(:inbox, :conversation, :sender) } + include MessageFilterHelpers include Liquidable NUMBER_OF_PERMITTED_ATTACHMENTS = 15 diff --git a/app/services/search_service.rb b/app/services/search_service.rb index add7938a0..fa1cf769d 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -34,11 +34,22 @@ class SearchService end def filter_messages - @messages = current_account.messages.where(inbox_id: accessable_inbox_ids) - .where('messages.content ILIKE :search', search: "%#{search_query}%") - .where('created_at >= ?', 3.months.ago) - .reorder('created_at DESC') - .limit(10) + @messages = if SearchConfig.enabled? + Message.search( + search_query, where: { + inbox_id: accessable_inbox_ids, + account_id: current_account.id + }, order: { created_at: :desc }, limit: 20 + ) + else + current_account + .messages + .where(inbox_id: accessable_inbox_ids) + .where('messages.content ILIKE :search', search: "%#{search_query}%") + .where('created_at >= ?', 3.months.ago) + .reorder('created_at DESC') + .limit(10) + end end def filter_contacts diff --git a/config/initializers/seach_config.rb b/config/initializers/seach_config.rb new file mode 100644 index 000000000..f57531b57 --- /dev/null +++ b/config/initializers/seach_config.rb @@ -0,0 +1,32 @@ +class SearchConfig + class << self + def enabled? + opensearch_url.present? + end + + def setup! + return unless enabled? + + Searchkick.client_options = client_options + end + + def client_options + { + url: opensearch_url, + transport_options: { + request: { timeout: timeout } + } + } + end + + private + + def opensearch_url + ENV.fetch('OPENSEARCH_URL', nil) + end + + def timeout + ENV.fetch('OPENSEARCH_TIMEOUT', 10).to_i + end + end +end diff --git a/config/schedule.yml b/config/schedule.yml index 7a0d2faeb..fdadac95d 100644 --- a/config/schedule.yml +++ b/config/schedule.yml @@ -8,21 +8,21 @@ internal_check_new_versions_job: class: 'Internal::CheckNewVersionsJob' queue: scheduled_jobs -# executed At every 5th minute.. -trigger_scheduled_items_job: - cron: '*/5 * * * *' - class: 'TriggerScheduledItemsJob' - queue: scheduled_jobs +# # executed At every 5th minute.. +# trigger_scheduled_items_job: +# cron: '*/5 * * * *' +# class: 'TriggerScheduledItemsJob' +# queue: scheduled_jobs -# executed At every minute.. -trigger_imap_email_inboxes_job: - cron: '*/1 * * * *' - class: 'Inboxes::FetchImapEmailInboxesJob' - queue: scheduled_jobs +# # executed At every minute.. +# trigger_imap_email_inboxes_job: +# cron: '*/1 * * * *' +# class: 'Inboxes::FetchImapEmailInboxesJob' +# queue: scheduled_jobs -# executed daily at 2230 UTC -# which is our lowest traffic time -remove_stale_contact_inboxes_job.rb: - cron: '30 22 * * *' - class: 'Internal::RemoveStaleContactInboxesJob' - queue: scheduled_jobs +# # executed daily at 2230 UTC +# # which is our lowest traffic time +# remove_stale_contact_inboxes_job.rb: +# cron: '30 22 * * *' +# class: 'Internal::RemoveStaleContactInboxesJob' +# queue: scheduled_jobs diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 4d2abed9c..24d222fef 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -23,6 +23,7 @@ - action_mailbox_routing - low - scheduled_jobs + - searchkick - async_database_migration - active_storage_analysis - active_storage_purge