From 3cd8cf43ce2f85af873bae65bb60e325d1f38a60 Mon Sep 17 00:00:00 2001 From: Tanmay Deep Sharma <32020192+tds-1@users.noreply.github.com> Date: Thu, 21 May 2026 16:14:28 +0530 Subject: [PATCH 1/4] fix: atomically claim conversation to prevent duplicate assignment (#14495) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Fixes a bug under Assignment V2 where a single conversation could be reassigned dozens of times in a row by the system, producing long stacks of "Assigned to X by Automation System via " activity messages alternating between agents. After this change each unassigned conversation is assigned exactly once, even on busy inboxes. ## Fixes # (issue) ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) ## How Has This Been Tested? ## How to reproduce 1. Enable `assignment_v2` on an account with at least 2 online agents in an inbox. 2. Generate sustained resolve/snooze activity in the inbox (each one enqueues `AutoAssignment::AssignmentJob` for the whole inbox). 3. Watch any one unassigned conversation while the jobs drain — pre-fix it picks up multiple back-to-back "Assigned to …" activity rows alternating between agents. ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- app/jobs/auto_assignment/assignment_job.rb | 36 +++++++++++++++++-- .../periodic_assignment_job.rb | 2 +- .../concerns/auto_assignment_handler.rb | 6 ++-- .../auto_assignment/assignment_service.rb | 23 ++++++++++-- lib/redis/alfred.rb | 12 +++++++ lib/redis/redis_keys.rb | 2 ++ .../auto_assignment/assignment_job_spec.rb | 33 ++++++++++++++--- .../periodic_assignment_job_spec.rb | 10 +++--- 8 files changed, 107 insertions(+), 17 deletions(-) diff --git a/app/jobs/auto_assignment/assignment_job.rb b/app/jobs/auto_assignment/assignment_job.rb index 9c6760ecc..e70137001 100644 --- a/app/jobs/auto_assignment/assignment_job.rb +++ b/app/jobs/auto_assignment/assignment_job.rb @@ -1,21 +1,53 @@ class AutoAssignment::AssignmentJob < ApplicationJob queue_as :default - def perform(inbox_id:) + IN_FLIGHT_TTL = 5.minutes + + # Coalesce per inbox: at most one AssignmentJob per inbox is in-flight + # (queued or running) at any time. The marker carries a token so a job only + # releases its own claim (a newer job may have taken it after a TTL lapse). + def self.enqueue_for_inbox(inbox_id) + key = format(::Redis::Alfred::AUTO_ASSIGNMENT_IN_FLIGHT_KEY, inbox_id: inbox_id) + token = SecureRandom.uuid + return false unless ::Redis::Alfred.set(key, token, nx: true, ex: IN_FLIGHT_TTL) + + return true if perform_later(inbox_id: inbox_id, token: token) + + # Enqueue was halted; release our own claim so the inbox isn't gated until the TTL. + ::Redis::Alfred.delete_if_equals(key, token) + false + rescue StandardError + # Enqueue raised after we claimed the gate; release our own claim, then re-raise. + ::Redis::Alfred.delete_if_equals(key, token) + raise + end + + def perform(inbox_id:, token: nil) inbox = Inbox.find_by(id: inbox_id) return unless inbox service = AutoAssignment::AssignmentService.new(inbox: inbox) - assigned_count = service.perform_bulk_assignment(limit: bulk_assignment_limit) Rails.logger.info "Assigned #{assigned_count} conversations for inbox #{inbox.id}" rescue StandardError => e Rails.logger.error "Bulk assignment failed for inbox #{inbox_id}: #{e.message}" raise e if Rails.env.test? + ensure + release_in_flight(inbox_id, token) end private + # Release the in-flight marker only if we still own it. The atomic + # compare-and-delete ensures a job whose TTL lapsed can't delete a newer + # job's claim. Tokenless (pre-deploy) jobs never claimed a key, so skip. + def release_in_flight(inbox_id, token) + return if token.nil? + + key = format(::Redis::Alfred::AUTO_ASSIGNMENT_IN_FLIGHT_KEY, inbox_id: inbox_id) + ::Redis::Alfred.delete_if_equals(key, token) + end + def bulk_assignment_limit ENV.fetch('AUTO_ASSIGNMENT_BULK_LIMIT', 100).to_i end diff --git a/app/jobs/auto_assignment/periodic_assignment_job.rb b/app/jobs/auto_assignment/periodic_assignment_job.rb index 63500507e..2963c383a 100644 --- a/app/jobs/auto_assignment/periodic_assignment_job.rb +++ b/app/jobs/auto_assignment/periodic_assignment_job.rb @@ -10,7 +10,7 @@ class AutoAssignment::PeriodicAssignmentJob < ApplicationJob inboxes.each do |inbox| next unless inbox.auto_assignment_v2_enabled? - AutoAssignment::AssignmentJob.perform_later(inbox_id: inbox.id) + AutoAssignment::AssignmentJob.enqueue_for_inbox(inbox.id) end end end diff --git a/app/models/concerns/auto_assignment_handler.rb b/app/models/concerns/auto_assignment_handler.rb index 6be7a8d85..1110cbd27 100644 --- a/app/models/concerns/auto_assignment_handler.rb +++ b/app/models/concerns/auto_assignment_handler.rb @@ -15,8 +15,10 @@ module AutoAssignmentHandler return unless should_run_auto_assignment? if inbox.auto_assignment_v2_enabled? - # Use new assignment system - AutoAssignment::AssignmentJob.perform_later(inbox_id: inbox.id) + # Coalesces bursts of triggers per inbox. Fine if the job runs even when the + # surrounding save rolls back: it only scans the inbox's current unassigned + # conversations, so running it for an uncommitted change is harmless. + AutoAssignment::AssignmentJob.enqueue_for_inbox(inbox.id) else # Use legacy assignment system # If conversation has a team, only consider team members for assignment diff --git a/app/services/auto_assignment/assignment_service.rb b/app/services/auto_assignment/assignment_service.rb index e27f1e829..f2d2799ff 100644 --- a/app/services/auto_assignment/assignment_service.rb +++ b/app/services/auto_assignment/assignment_service.rb @@ -72,15 +72,32 @@ class AutoAssignment::AssignmentService end def assign_conversation(conversation, agent) - Current.executed_by = inbox.assignment_policy || inbox - conversation.update!(assignee: agent) - Current.executed_by = nil + return false unless claim_and_assign(conversation, agent) + + conversation.reload rate_limiter = build_rate_limiter(agent) rate_limiter.track_assignment(conversation) dispatch_assignment_event(conversation, agent) true + end + + # Atomically claim the row so two bulk runs that overlap (the in-flight gate + # is best-effort and can lapse on TTL) can't both assign the same conversation. + def claim_and_assign(conversation, agent) + Current.executed_by = inbox.assignment_policy || inbox + + Conversation.transaction do + locked = inbox.conversations + .where(id: conversation.id, assignee_id: nil) + .lock('FOR UPDATE SKIP LOCKED') + .first + next false unless locked + + locked.update!(assignee: agent) + true + end ensure Current.executed_by = nil end diff --git a/lib/redis/alfred.rb b/lib/redis/alfred.rb index 1554b8806..d913682db 100644 --- a/lib/redis/alfred.rb +++ b/lib/redis/alfred.rb @@ -25,6 +25,18 @@ module Redis::Alfred $alfred.with { |conn| conn.del(key) } end + # atomic compare-and-delete (release a lock only if you still own it); WATCH/MULTI + # aborts the delete if the key changes between the check and the delete. + def delete_if_equals(key, expected_value) + $alfred.with do |conn| + conn.watch(key) do + next conn.unwatch unless conn.get(key) == expected_value + + conn.multi { |transaction| transaction.del(key) } + end + end + end + # increment a key by 1. throws error if key value is incompatible # sets key to 0 before operation if key doesn't exist def incr(key) diff --git a/lib/redis/redis_keys.rb b/lib/redis/redis_keys.rb index 812553844..6b0d7f0fd 100644 --- a/lib/redis/redis_keys.rb +++ b/lib/redis/redis_keys.rb @@ -73,6 +73,8 @@ module Redis::RedisKeys # Track conversation assignments to agents for rate limiting ASSIGNMENT_KEY = 'ASSIGNMENT::%d::AGENT::%d::CONVERSATION::%d'.freeze ASSIGNMENT_KEY_PATTERN = 'ASSIGNMENT::%d::AGENT::%d::*'.freeze + # At-most-one AssignmentJob per inbox in-flight (queued or running); further enqueues are skipped + AUTO_ASSIGNMENT_IN_FLIGHT_KEY = 'AUTO_ASSIGNMENT_IN_FLIGHT::%d'.freeze ## Account Onboarding ACCOUNT_ONBOARDING_ENRICHMENT = 'ONBOARDING_ENRICHMENT::%d'.freeze diff --git a/spec/jobs/auto_assignment/assignment_job_spec.rb b/spec/jobs/auto_assignment/assignment_job_spec.rb index d13f9fef8..b6d95789f 100644 --- a/spec/jobs/auto_assignment/assignment_job_spec.rb +++ b/spec/jobs/auto_assignment/assignment_job_spec.rb @@ -24,10 +24,11 @@ RSpec.describe AutoAssignment::AssignmentJob, type: :job do service = instance_double(AutoAssignment::AssignmentService) allow(AutoAssignment::AssignmentService).to receive(:new).and_return(service) allow(service).to receive(:perform_bulk_assignment).and_return(3) - - expect(Rails.logger).to receive(:info).with("Assigned 3 conversations for inbox #{inbox.id}") + allow(Rails.logger).to receive(:info) described_class.new.perform(inbox_id: inbox.id) + + expect(Rails.logger).to have_received(:info).with("Assigned 3 conversations for inbox #{inbox.id}") end it 'uses custom bulk limit from environment' do @@ -67,16 +68,40 @@ RSpec.describe AutoAssignment::AssignmentJob, type: :job do service = instance_double(AutoAssignment::AssignmentService) allow(AutoAssignment::AssignmentService).to receive(:new).and_return(service) allow(service).to receive(:perform_bulk_assignment).and_raise(StandardError, 'Something went wrong') - - expect(Rails.logger).to receive(:error).with("Bulk assignment failed for inbox #{inbox.id}: Something went wrong") + allow(Rails.logger).to receive(:error) expect do described_class.new.perform(inbox_id: inbox.id) end.to raise_error(StandardError, 'Something went wrong') + + expect(Rails.logger).to have_received(:error).with("Bulk assignment failed for inbox #{inbox.id}: Something went wrong") end end end + describe '.enqueue_for_inbox' do + after { Redis::Alfred.delete(format(Redis::Alfred::AUTO_ASSIGNMENT_IN_FLIGHT_KEY, inbox_id: inbox.id)) } + + it 'enqueues one run per inbox and coalesces concurrent triggers' do + allow(described_class).to receive(:perform_later).and_return(true) + + expect(described_class.enqueue_for_inbox(inbox.id)).to be(true) + expect(described_class.enqueue_for_inbox(inbox.id)).to be(false) + expect(described_class).to have_received(:perform_later).once + end + + it 'does not release a newer run marker when its own token is stale' do + key = format(Redis::Alfred::AUTO_ASSIGNMENT_IN_FLIGHT_KEY, inbox_id: inbox.id) + Redis::Alfred.set(key, 'newer-token', ex: 300) + allow(AutoAssignment::AssignmentService).to receive(:new) + .and_return(instance_double(AutoAssignment::AssignmentService, perform_bulk_assignment: 0)) + + described_class.new.perform(inbox_id: inbox.id, token: 'stale-token') + + expect(Redis::Alfred.get(key)).to eq('newer-token') + end + end + describe 'job configuration' do it 'is queued in the default queue' do expect(described_class.queue_name).to eq('default') diff --git a/spec/jobs/auto_assignment/periodic_assignment_job_spec.rb b/spec/jobs/auto_assignment/periodic_assignment_job_spec.rb index e281f79f4..4f0a6f9d8 100644 --- a/spec/jobs/auto_assignment/periodic_assignment_job_spec.rb +++ b/spec/jobs/auto_assignment/periodic_assignment_job_spec.rb @@ -29,7 +29,7 @@ RSpec.describe AutoAssignment::PeriodicAssignmentJob, type: :job do it 'queues assignment job for eligible inboxes' do inbox_assignment_policy # ensure it exists - expect(AutoAssignment::AssignmentJob).to receive(:perform_later).with(inbox_id: inbox.id) + expect(AutoAssignment::AssignmentJob).to receive(:enqueue_for_inbox).with(inbox.id) described_class.new.perform end @@ -51,8 +51,8 @@ RSpec.describe AutoAssignment::PeriodicAssignmentJob, type: :job do allow(Account).to receive(:find_in_batches).and_yield([account]).and_yield([account2]) - expect(AutoAssignment::AssignmentJob).to receive(:perform_later).with(inbox_id: inbox.id) - expect(AutoAssignment::AssignmentJob).to receive(:perform_later).with(inbox_id: inbox2.id) + expect(AutoAssignment::AssignmentJob).to receive(:enqueue_for_inbox).with(inbox.id) + expect(AutoAssignment::AssignmentJob).to receive(:enqueue_for_inbox).with(inbox2.id) described_class.new.perform end @@ -65,7 +65,7 @@ RSpec.describe AutoAssignment::PeriodicAssignmentJob, type: :job do end it 'does not queue assignment job' do - expect(AutoAssignment::AssignmentJob).not_to receive(:perform_later) + expect(AutoAssignment::AssignmentJob).not_to receive(:enqueue_for_inbox) described_class.new.perform end @@ -78,7 +78,7 @@ RSpec.describe AutoAssignment::PeriodicAssignmentJob, type: :job do end it 'does not process the account' do - expect(AutoAssignment::AssignmentJob).not_to receive(:perform_later) + expect(AutoAssignment::AssignmentJob).not_to receive(:enqueue_for_inbox) described_class.new.perform end From 3d20a7b049279445b314e43a46544b63895490d5 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 21 May 2026 16:25:01 +0530 Subject: [PATCH 2/4] feat: generate Help Center for Onboarding (#14370) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Manually triggering help center generation Open a Rails console (`bundle exec rails console`): ```ruby account = Account.find() user = account.users.first # Optional: refresh brand info from the customer's website domain = 'example.com' result = WebsiteBrandingService.new("noreply@#{domain}").perform account.update!( name: result[:title].presence || account.name, custom_attributes: account.custom_attributes.merge('website' => domain, 'brand_info' => result) ) # Optional: wipe existing portals so a fresh one is created account.portals.destroy_all Onboarding::HelpCenterCreationService.new(account, user).perform ``` Sidekiq must be running — articles are written by `Onboarding::HelpCenterArticleGenerationJob`. Avoid running on production; generation calls the LLM provider. ### Generation flow (Happy Path) ```mermaid sequenceDiagram autonumber participant Kickoff as HelpCenterCreationService participant DB as DB participant GenJob as HelpCenterArticleGenerationJob participant Curator as HelpCenterCurator participant Firecrawl as Firecrawl participant CuratorLLM as Curation LLM participant Redis as Redis Progress participant WriterJob as HelpCenterArticleWriterJob participant Builder as HelpCenterArticleBuilder participant WriterLLM as Writer LLM participant Cable as ActionCable Kickoff->>DB: Create portal for account
homepage_link=https://chatwoot.com Kickoff->>DB: Attach brand logo if available Kickoff->>GenJob: Enqueue generation job
account_id, portal_id, user_id, generation_id GenJob->>Curator: Curate help center plan Curator->>Firecrawl: map https://chatwoot.com
search: docs help support faq Firecrawl-->>Curator: Return discovered links Curator->>CuratorLLM: Select categories + article plans
from discovered links only CuratorLLM-->>Curator: Return categories, articles, allowed_urls GenJob->>DB: Create portal categories GenJob->>GenJob: Stamp articles with category_id GenJob->>GenJob: Filter article URLs against allowed_urls GenJob->>GenJob: Drop articles with no category
or no approved source URLs GenJob->>Redis: Start progress
status=generating, total=N, finished=0 loop For each approved article GenJob->>WriterJob: Enqueue writer job
title, category_id, approved URLs end par Writer jobs run independently WriterJob->>Builder: Build article from approved URLs Builder->>Firecrawl: batch_scrape approved URLs Firecrawl-->>Builder: Return Markdown source pages Builder->>WriterLLM: Rewrite sources into one article WriterLLM-->>Builder: Return title, description, Markdown content Builder->>DB: Create draft portal article
meta.source_urls WriterJob->>Redis: Increment finished count WriterJob->>Cable: Broadcast help_center.article_generated end WriterJob->>Redis: If finished >= total
mark status=completed WriterJob->>Cable: Broadcast help_center.generation_completed ``` ### Redis State Management ```mermaid stateDiagram-v2 [*] --> active_pointer_set active_pointer_set --> generating: generation job creates valid plan active_pointer_set --> skipped: curation skipped/failed generating --> generating: each writer job increments finished generating --> completed: finished == total generating --> ignored_completion: generation_id superseded skipped --> [*] completed --> [*] ignored_completion --> [*] ``` --- Gemfile | 2 + Gemfile.lock | 2 + .../help_center_article_generation_job.rb | 103 ++++++++++ .../help_center_article_writer_job.rb | 52 +++++ .../captain/llm/article_writer_schema.rb | 12 ++ .../captain/llm/article_writer_service.rb | 102 ++++++++++ .../llm/help_center_curation_schema.rb | 30 +++ .../llm/help_center_curation_service.rb | 157 +++++++++++++++ .../app/services/firecrawl/configuration.rb | 31 +++ .../onboarding/help_center_article_builder.rb | 71 +++++++ .../onboarding/help_center_broadcaster.rb | 29 +++ .../help_center_creation_service.rb | 129 ++++++++++++ .../onboarding/help_center_curator.rb | 65 ++++++ .../services/onboarding/help_center_errors.rb | 4 + .../help_center_generation_state.rb | 45 +++++ lib/redis/alfred.rb | 4 + lib/redis/redis_keys.rb | 1 + .../simple_page_crawl_parser_job_spec.rb | 24 ++- ...help_center_article_generation_job_spec.rb | 188 ++++++++++++++++++ .../help_center_article_writer_job_spec.rb | 159 +++++++++++++++ .../help_center_article_builder_spec.rb | 18 ++ .../help_center_creation_service_spec.rb | 58 ++++++ .../onboarding/help_center_curator_spec.rb | 41 ++++ .../help_center_generation_state_spec.rb | 61 ++++++ 24 files changed, 1380 insertions(+), 8 deletions(-) create mode 100644 enterprise/app/jobs/onboarding/help_center_article_generation_job.rb create mode 100644 enterprise/app/jobs/onboarding/help_center_article_writer_job.rb create mode 100644 enterprise/app/services/captain/llm/article_writer_schema.rb create mode 100644 enterprise/app/services/captain/llm/article_writer_service.rb create mode 100644 enterprise/app/services/captain/llm/help_center_curation_schema.rb create mode 100644 enterprise/app/services/captain/llm/help_center_curation_service.rb create mode 100644 enterprise/app/services/firecrawl/configuration.rb create mode 100644 enterprise/app/services/onboarding/help_center_article_builder.rb create mode 100644 enterprise/app/services/onboarding/help_center_broadcaster.rb create mode 100644 enterprise/app/services/onboarding/help_center_creation_service.rb create mode 100644 enterprise/app/services/onboarding/help_center_curator.rb create mode 100644 enterprise/app/services/onboarding/help_center_errors.rb create mode 100644 enterprise/app/services/onboarding/help_center_generation_state.rb create mode 100644 spec/enterprise/jobs/onboarding/help_center_article_generation_job_spec.rb create mode 100644 spec/enterprise/jobs/onboarding/help_center_article_writer_job_spec.rb create mode 100644 spec/enterprise/services/onboarding/help_center_article_builder_spec.rb create mode 100644 spec/enterprise/services/onboarding/help_center_creation_service_spec.rb create mode 100644 spec/enterprise/services/onboarding/help_center_curator_spec.rb create mode 100644 spec/enterprise/services/onboarding/help_center_generation_state_spec.rb diff --git a/Gemfile b/Gemfile index b27b66fde..e10984f53 100644 --- a/Gemfile +++ b/Gemfile @@ -209,6 +209,8 @@ gem 'opentelemetry-exporter-otlp' gem 'shopify_api' +gem 'firecrawl-sdk', '~> 1.0', require: 'firecrawl' + ### Gems required only in specific deployment environments ### ############################################################## diff --git a/Gemfile.lock b/Gemfile.lock index ed1d94172..4da0e5847 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -339,6 +339,7 @@ GEM ffi-compiler (1.0.1) ffi (>= 1.0.0) rake + firecrawl-sdk (1.4.1) flag_shih_tzu (0.3.23) foreman (0.87.2) fugit (1.11.1) @@ -1079,6 +1080,7 @@ DEPENDENCIES faker faraday_middleware-aws-sigv4 fcm + firecrawl-sdk (~> 1.0) flag_shih_tzu foreman gemoji diff --git a/enterprise/app/jobs/onboarding/help_center_article_generation_job.rb b/enterprise/app/jobs/onboarding/help_center_article_generation_job.rb new file mode 100644 index 000000000..ed561d668 --- /dev/null +++ b/enterprise/app/jobs/onboarding/help_center_article_generation_job.rb @@ -0,0 +1,103 @@ +class Onboarding::HelpCenterArticleGenerationJob < ApplicationJob + queue_as :low + + retry_on Firecrawl::FirecrawlError, wait: :polynomially_longer, attempts: 3 do |job, error| + _account_id, _portal_id, user_id, generation_id = job.arguments + reason = "firecrawl exhausted: #{error.message}" + Rails.logger.warn "[HelpCenterGenerationJob] gen=#{generation_id} #{reason}" + job.send(:skip_and_broadcast, user: User.find_by(id: user_id), generation_id: generation_id, reason: reason) + end + + def perform(account_id, portal_id, user_id, generation_id) + return if Onboarding::HelpCenterGenerationState.current(generation_id).present? + + process( + account: Account.find(account_id), + portal: Portal.find(portal_id), + user: User.find(user_id), + generation_id: generation_id + ) + rescue Onboarding::HelpCenterErrors::CurationSkipped => e + Rails.logger.info "[HelpCenterGenerationJob] gen=#{generation_id} skipped: #{e.message}" + skip_and_broadcast(user: User.find_by(id: user_id), generation_id: generation_id, reason: e.message) + end + + private + + def process(account:, portal:, user:, generation_id:) + plan = Onboarding::HelpCenterCurator.new(account: account).perform + articles = create_categories_and_build_article_payloads(portal, plan) + + Onboarding::HelpCenterGenerationState.start(generation_id, total: articles.size) + enqueue_writer_jobs( + account_id: account.id, + portal_id: portal.id, + user_id: user.id, + generation_id: generation_id, + articles: articles + ) + end + + def create_categories_and_build_article_payloads(portal, plan) + ActiveRecord::Base.transaction do + categories_by_name = create_categories(portal, plan['categories']) + articles = build_article_payloads( + plan['articles'], + categories_by_name, + plan['allowed_urls'] + ) + + if articles.empty? + raise Onboarding::HelpCenterErrors::CurationSkipped, + 'no articles after category or URL filtering' + end + + articles + end + end + + def create_categories(portal, categories) + locale = portal.default_locale + Array(categories).each_with_index.with_object({}) do |(cat, idx), acc| + name = cat['name'].to_s.strip + next if name.blank? + + record = portal.categories.create!( + name: name, + description: cat['description'].to_s.strip.presence, + slug: "#{name.parameterize}-#{SecureRandom.hex(3)}", + locale: locale, + position: (idx + 1) * 10 + ) + acc[name] = record + end + end + + def build_article_payloads(articles, categories_by_name, allowed_urls) + allowed_urls = Array(allowed_urls).to_set + Array(articles).filter_map do |article| + category_id = categories_by_name[article['category_name'].to_s]&.id + next if category_id.nil? + + urls = Array(article['urls']).select { |url| allowed_urls.include?(url) } + next if urls.empty? + + article.merge('category_id' => category_id, 'urls' => urls) + end + end + + def enqueue_writer_jobs(account_id:, portal_id:, user_id:, generation_id:, articles:) + articles.each do |article| + Onboarding::HelpCenterArticleWriterJob.perform_later( + account_id, portal_id, user_id, generation_id, { article: article } + ) + end + end + + def skip_and_broadcast(user:, generation_id:, reason:) + Onboarding::HelpCenterGenerationState.skip(generation_id, reason: reason) + Onboarding::HelpCenterBroadcaster.completed( + user: user, generation_id: generation_id, status: 'skipped', skip_reason: reason + ) + end +end diff --git a/enterprise/app/jobs/onboarding/help_center_article_writer_job.rb b/enterprise/app/jobs/onboarding/help_center_article_writer_job.rb new file mode 100644 index 000000000..2c25b86e9 --- /dev/null +++ b/enterprise/app/jobs/onboarding/help_center_article_writer_job.rb @@ -0,0 +1,52 @@ +class Onboarding::HelpCenterArticleWriterJob < ApplicationJob + queue_as :low + + retry_on Firecrawl::FirecrawlError, wait: :polynomially_longer, attempts: 3 do |job, error| + job.send(:on_writer_failure, error) + end + + discard_on Onboarding::HelpCenterErrors::ArticleBuildFailed do |job, error| + job.send(:on_writer_failure, error) + end + + def perform(account_id, portal_id, user_id, generation_id, article_payload) + user = User.find(user_id) + payload = article_payload.with_indifferent_access + article = Onboarding::HelpCenterArticleBuilder.new( + account: Account.find(account_id), + portal: Portal.find(portal_id), + user: user, + article: payload[:article] + ).perform + + finalize(user: user, generation_id: generation_id, article: article) + end + + private + + def on_writer_failure(error) + user, generation_id = failure_context + Rails.logger.warn "[HelpCenterWriterJob] gen=#{generation_id} failed: #{error.class} #{error.message}" + finalize(user: user, generation_id: generation_id, article: nil) + end + + def failure_context + _account_id, _portal_id, user_id, generation_id = arguments + [User.find_by(id: user_id), generation_id] + end + + def finalize(user:, generation_id:, article:) + result = Onboarding::HelpCenterGenerationState.record_article_finished(generation_id) + + if article + Onboarding::HelpCenterBroadcaster.article_generated( + user: user, generation_id: generation_id, article: article, articles_finished: result[:finished] + ) + end + return unless result[:completed] + + Onboarding::HelpCenterBroadcaster.completed(user: user, generation_id: generation_id, status: 'completed') + rescue Onboarding::HelpCenterGenerationState::Missing => e + Rails.logger.warn "[HelpCenterWriterJob] gen=#{generation_id} #{e.message}" + end +end diff --git a/enterprise/app/services/captain/llm/article_writer_schema.rb b/enterprise/app/services/captain/llm/article_writer_schema.rb new file mode 100644 index 000000000..17d742461 --- /dev/null +++ b/enterprise/app/services/captain/llm/article_writer_schema.rb @@ -0,0 +1,12 @@ +class Captain::Llm::ArticleWriterSchema < RubyLLM::Schema + CONTENT_DESCRIPTION = 'Full article body in clean Markdown. Use headings, lists, and code fences where appropriate. ' \ + 'Preserve steps, code samples, FAQs, troubleshooting detail. Strip marketing copy, navigation breadcrumbs, ' \ + 'social/share footers, "edit this page" links, repeated CTAs. ' \ + 'Total length must stay under 18000 characters; trim repetition and tangents before cutting substance.'.freeze + TITLE_DESCRIPTION = 'Concise article title (max 80 chars). Plain text, no markdown.'.freeze + DESCRIPTION_DESCRIPTION = 'One-sentence summary (max 200 chars) describing what the article teaches.'.freeze + + string :title, description: TITLE_DESCRIPTION, max_length: 80 + string :description, description: DESCRIPTION_DESCRIPTION, max_length: 200 + string :content, description: CONTENT_DESCRIPTION, max_length: 18_000 +end diff --git a/enterprise/app/services/captain/llm/article_writer_service.rb b/enterprise/app/services/captain/llm/article_writer_service.rb new file mode 100644 index 000000000..b94027248 --- /dev/null +++ b/enterprise/app/services/captain/llm/article_writer_service.rb @@ -0,0 +1,102 @@ +class Captain::Llm::ArticleWriterService < Captain::BaseTaskService + RESPONSE_SCHEMA = Captain::Llm::ArticleWriterSchema + SOURCE_MAX_LENGTH = 60_000 + + # source_pages: Array<{ url: String, markdown: String }>, 1-3 entries. + pattr_initialize [:account!, :source_pages!, { hint_title: nil }] + + def perform + response = make_api_call(model: writer_model, messages: messages, schema: RESPONSE_SCHEMA) + return response if response[:error] + + response.merge(message: extract_payload(response[:message])) + end + + private + + def extract_payload(message) + return {} if message.blank? + + data = message.is_a?(Hash) ? message.deep_symbolize_keys : {} + { + title: data[:title].to_s.strip, + description: data[:description].to_s.strip, + content: data[:content].to_s.strip + } + end + + def messages + [ + { role: 'system', content: system_prompt }, + { role: 'user', content: user_prompt } + ] + end + + def system_prompt + <<~PROMPT + You are rewriting web page content into a clean help-center article for a customer-support knowledge base. + You may receive 1 to 3 source pages. When given multiple sources, merge them into ONE coherent article: + deduplicate identical instructions, do not repeat the same step in different words, and order content + by the natural reading flow of the merged topic. When sources contradict, prefer the more authoritative + or detailed version. The result must read like a single article, not a stitched-together collage. + + Preserve the substance: keep instructions, steps, code samples, configuration, troubleshooting, and FAQs intact. + Strip marketing copy, navigation breadcrumbs, "share this page" footers, repeated CTAs, and links to unrelated pages. + Output well-formatted Markdown — use headings, lists, and code fences where appropriate. + The body must stay under 18000 characters. If the combined sources are longer, trim repetition and tangents + before cutting steps or critical detail. Never invent content the sources do not support. + + Write the title, description, and body in #{locale_name}. + If a source page is in another language, translate as you rewrite — do not copy source-language text into the output. + Code samples, command-line examples, API field names, and proper nouns stay in their original form. + PROMPT + end + + def user_prompt + pages = Array(source_pages).reject { |p| p[:markdown].to_s.blank? } + per_source_cap = pages.size.positive? ? SOURCE_MAX_LENGTH / pages.size : SOURCE_MAX_LENGTH + + sections = pages.each_with_index.map do |page, idx| + body = page[:markdown].to_s.truncate(per_source_cap, omission: "\n\n[source truncated for length]") + "=== Source #{idx + 1} of #{pages.size} (#{page[:url]}) ===\n#{body}" + end + + parts = [ + ("Suggested title (you may rewrite): #{hint_title}" if hint_title.present?), + 'Source pages (Markdown):', + sections.join("\n\n") + ].compact + parts.join("\n\n") + end + + def locale_name + code = account.locale.to_s + LANGUAGES_CONFIG.values.find { |v| v[:iso_639_1_code] == code }&.dig(:name) || code.presence || 'English (en)' + end + + def event_name + 'article_writer' + end + + def llm_credential + @llm_credential ||= system_llm_credential + end + + def captain_tasks_enabled? + true + end + + # Rewrite runs on the operator's OpenAI key during onboarding; should not + # debit the customer's captain_responses quota. + def counts_toward_usage? + false + end + + def writer_model + 'gpt-5.2' + end + + def build_follow_up_context? + false + end +end diff --git a/enterprise/app/services/captain/llm/help_center_curation_schema.rb b/enterprise/app/services/captain/llm/help_center_curation_schema.rb new file mode 100644 index 000000000..1c2a53b92 --- /dev/null +++ b/enterprise/app/services/captain/llm/help_center_curation_schema.rb @@ -0,0 +1,30 @@ +class Captain::Llm::HelpCenterCurationSchema < RubyLLM::Schema + CATEGORIES_DESCRIPTION = 'High-level categories that group the chosen articles. Use only as many ' \ + 'as the content naturally breaks into. Names must be short (1-3 words) and reusable.'.freeze + ARTICLES_DESCRIPTION = 'A curated starting set of help-center articles selected from the input URL list. ' \ + 'Quality over quantity: only include pages with clear, high-value, substantive help ' \ + 'content. Skip blog posts, marketing/landing pages, login, pricing, legal, careers, ' \ + 'customer testimonials, press, about/company, whitepapers, support contact pages, ' \ + 'terms of service, privacy policy.'.freeze + TITLE_DESCRIPTION = 'Concise article title (max 80 chars), rewritten if the source title is too long or marketing-y.'.freeze + CATEGORY_DESCRIPTION = 'One sentence describing what kind of articles belong in this category.'.freeze + URLS_DESCRIPTION = '1 to 3 source URLs from the input list. Prefer grouping when pages cover related ' \ + 'aspects of the same topic — overview + deep-dive, FAQ + how-to, policy + FAQ, ' \ + 'parent topic + its troubleshooting page. Merged sources give the writer more ' \ + 'context and produce stronger articles than several thin stubs.'.freeze + + array :categories, description: CATEGORIES_DESCRIPTION, min_items: 1, max_items: 10 do + object do + string :name, description: 'Short, human-readable category name (1-3 words).', max_length: 60 + string :description, description: CATEGORY_DESCRIPTION, max_length: 200 + end + end + + array :articles, description: ARTICLES_DESCRIPTION, min_items: 1, max_items: 25 do + object do + array :urls, description: URLS_DESCRIPTION, min_items: 1, max_items: 3, of: :string + string :title, description: TITLE_DESCRIPTION, max_length: 80 + string :category_name, description: 'Must exactly match one of the names emitted in the categories field.', max_length: 60 + end + end +end diff --git a/enterprise/app/services/captain/llm/help_center_curation_service.rb b/enterprise/app/services/captain/llm/help_center_curation_service.rb new file mode 100644 index 000000000..1f8b8acb2 --- /dev/null +++ b/enterprise/app/services/captain/llm/help_center_curation_service.rb @@ -0,0 +1,157 @@ +class Captain::Llm::HelpCenterCurationService < Captain::BaseTaskService + RESPONSE_SCHEMA = Captain::Llm::HelpCenterCurationSchema + MAX_LINKS_IN_PROMPT = 50 + IGNORED_URL_PATTERN = /\.(?:pdf|jpe?g|png|gif|webp|svg|ico|bmp|tiff?|avif|heic)(?:\?|#|$)/i + # This model consistently outperforms 5.2 in generating tighter and more + # accurate curations. + CURATION_MODEL = 'gpt-4.1'.freeze + + pattr_initialize [:account!, :links!] + + def perform + response = make_api_call(model: CURATION_MODEL, messages: messages, schema: RESPONSE_SCHEMA) + return response if response[:error] + + response.merge(message: extract_payload(response[:message])) + end + + private + + def extract_payload(message) + return { categories: [], articles: [] } if message.blank? + + data = message.is_a?(Hash) ? message.deep_symbolize_keys : {} + articles = Array(data[:articles]) + used_names = articles.map { |a| a[:category_name].to_s } + categories = Array(data[:categories]).select { |c| used_names.include?(c[:name].to_s) } + { categories: categories, articles: articles } + end + + def messages + [ + { role: 'system', content: system_prompt }, + { role: 'user', content: user_prompt } + ] + end + + def system_prompt + <<~PROMPT + You are curating a help center for a company's customer-support widget. + You will be given a list of pages discovered on the company's website. + Pick pages that would make genuinely useful help-center articles for end users — + substantive how-to, FAQ, troubleshooting, policy, getting-started, account/billing + help, or product guide content. + + This is a STARTING SET for the user, not a comprehensive corpus. The user will add + more articles later. Each article you pick costs downstream time, compute, and + money to scrape and rewrite — be deliberate. Only include pages with clear, + high-value, substantive help content. When unsure about a page's value, leave it + out. 8 strong articles beat 20 padded ones, even when the input has 20+ candidates. + + Quality over quantity: do not pad with thin, overview, or marketing-adjacent pages + to hit a target count. If a site has only a few genuinely useful pages, return only + those few. The schema allows up to 25 articles, but treat that as a hard ceiling, + not a target — most sites should land well under it. + + Skip marketing/landing pages, blog posts, login, pricing tiers, legal, careers, press, investor pages. + Group your picks into reusable categories — use as many as the content naturally breaks into. + Use the URL paths and page titles to judge relevance — do not invent URLs. + + URL-path priority (preference order, not hard rules): + - First tier — almost always pick when present. Paths containing /support, /help, + /docs, /documentation, /faq, /faqs, /kb, /knowledge-base, /learn, /guides, + /getting-started, /how-to, /tutorial, /troubleshoot. + - Second tier — pick when the page carries user-relevant information a customer + would ask support about. Paths like /features, /pricing, /plans, /shipping, + /returns, /warranty, /security, individual product or category pages. Prefer + these only after first-tier picks; if a topic exists in both tiers, prefer the + first-tier URL. + - Skip — promotional, navigational, or boilerplate paths: /blog, /news, /press, + /careers, /jobs, /about, /team, /investors, /customers, /testimonials, + /case-studies, /login, /signup, /register, /legal, /terms, /privacy. + + For each article, group 1 to 3 URLs that together cover a single topic. PREFER + grouping whenever pages overlap or complement each other — merged sources give + the writer more context and produce a stronger article than two thin stubs. + + Strong signals to group multiple URLs (treat any of these as a green light): + - Same topic from different angles: overview + deep-dive, FAQ + how-to, + policy + FAQ, feature page + feature docs. + - Parent topic + its troubleshooting page (e.g. "Bank reconciliation" + + "Problems with bank reconciliation"; "SSO setup" + "SSO not working"). + - Variant-specific guides on the same topic ("SSO setup" + "SSO with Okta"; + "Webhooks overview" + "Webhook payload reference"). + - A how-to split across step or platform pages (install on iOS + Android + web). + - FAQ entries that match a deep-dive article elsewhere on the site. + + Before finalizing your picks, scan them for merge candidates: if two URLs are + about the same topic, they should almost always be one article, not two. + + Don't group across distinct topics that merely share a category ("Setting up SSO" + and "Setting up MFA" stay separate). If a URL is marketing for a feature and + another is the feature's docs, pick the docs and skip the marketing. + + Write all category names, category descriptions, and article titles in #{locale_name}. + The input page titles and descriptions may be in another language; translate the labels you emit into #{locale_name}. + Keep URLs unchanged. + PROMPT + end + + def user_prompt + parts = [ + "Company: #{account.name}", + ("Description: #{brand_info[:description]}" if brand_info[:description].present?), + ("Industries: #{industries_text}" if industries_text.present?), + 'Discovered pages (url — title — description):', + formatted_links + ].compact + parts.join("\n") + end + + def locale_name + code = account.locale.to_s + LANGUAGES_CONFIG.values.find { |v| v[:iso_639_1_code] == code }&.dig(:name) || code.presence || 'English (en)' + end + + def formatted_links + Array(links).reject { |link| ignored_url?(link) }.first(MAX_LINKS_IN_PROMPT).map do |link| + data = link.is_a?(Hash) ? link.deep_symbolize_keys : {} + "- #{data[:url]} — #{data[:title].to_s.strip} — #{data[:description].to_s.strip}" + end.join("\n") + end + + def ignored_url?(link) + url = link.is_a?(Hash) ? link.deep_symbolize_keys[:url].to_s : link.to_s + url.match?(IGNORED_URL_PATTERN) + end + + def brand_info + @brand_info ||= (account.custom_attributes['brand_info'] || {}).deep_symbolize_keys + end + + def industries_text + Array(brand_info[:industries]).filter_map { |i| i.is_a?(Hash) ? i[:industry] : i }.join(', ').presence + end + + def event_name + 'help_center_curation' + end + + def llm_credential + @llm_credential ||= system_llm_credential + end + + def captain_tasks_enabled? + true + end + + # Onboarding curation runs on the operator's OpenAI key; it should not + # debit the customer's captain_responses quota. + def counts_toward_usage? + false + end + + def build_follow_up_context? + false + end +end diff --git a/enterprise/app/services/firecrawl/configuration.rb b/enterprise/app/services/firecrawl/configuration.rb new file mode 100644 index 000000000..0d574b102 --- /dev/null +++ b/enterprise/app/services/firecrawl/configuration.rb @@ -0,0 +1,31 @@ +module Firecrawl::Configuration + INSTALLATION_CONFIG_KEY = 'CAPTAIN_FIRECRAWL_API_KEY'.freeze + EXCLUDE_TAGS = %w[iframe .sidebar .cookie-banner [role=navigation] [role=banner] [role=contentinfo]].freeze + DEFAULT_SCRAPE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000 + + module_function + + def configured? + api_key.present? + end + + def client + key = api_key + raise ::Firecrawl::FirecrawlError, "#{INSTALLATION_CONFIG_KEY} is not configured" if key.blank? + + ::Firecrawl::Client.new(api_key: key) + end + + def api_key + InstallationConfig.find_by(name: INSTALLATION_CONFIG_KEY)&.value + end + + def default_scrape_options(max_age: DEFAULT_SCRAPE_MAX_AGE_MS) + ::Firecrawl::Models::ScrapeOptions.new( + formats: ['markdown'], + only_main_content: true, + exclude_tags: EXCLUDE_TAGS, + max_age: max_age + ) + end +end diff --git a/enterprise/app/services/onboarding/help_center_article_builder.rb b/enterprise/app/services/onboarding/help_center_article_builder.rb new file mode 100644 index 000000000..dc6178ab8 --- /dev/null +++ b/enterprise/app/services/onboarding/help_center_article_builder.rb @@ -0,0 +1,71 @@ +class Onboarding::HelpCenterArticleBuilder + BuildFailed = Onboarding::HelpCenterErrors::ArticleBuildFailed + + def initialize(account:, portal:, user:, article:) + @account = account + @portal = portal + @user = user + + spec = article.with_indifferent_access + @urls = Array(spec[:urls]).map(&:to_s).reject(&:blank?) + @title = spec[:title] + @category_id = spec[:category_id] + end + + def perform + raise BuildFailed, 'no source urls supplied' if @urls.empty? + + source_pages = scrape(@urls) + raise BuildFailed, "scrape produced no usable pages for #{@urls.join(', ')}" if source_pages.empty? + + payload = rewrite(source_pages) + + @portal.articles.create!( + title: payload[:title], + description: payload[:description].presence, + content: payload[:content], + author_id: @user.id, + category_id: @category_id, + status: :draft, + meta: { source_urls: source_pages.pluck(:url) } + ) + end + + private + + def scrape(urls) + job = Firecrawl::Configuration.client.batch_scrape( + urls, + Firecrawl::Models::BatchScrapeOptions.new(options: Firecrawl::Configuration.default_scrape_options) + ) + Array(job.data).filter_map { |doc| normalize(doc) } + end + + def normalize(doc) + metadata = doc&.metadata || {} + status = metadata['statusCode'] + return nil if status.present? && !(200..299).cover?(status) + return nil if doc.markdown.to_s.blank? + + { + url: metadata['sourceURL'] || metadata['url'], + markdown: doc.markdown.to_s, + page_title: metadata['title'].to_s.strip + } + end + + def rewrite(source_pages) + response = Captain::Llm::ArticleWriterService.new( + account: @account, + source_pages: source_pages, + hint_title: @title.presence || source_pages.first[:page_title] + ).perform + raise BuildFailed, "writer LLM error: #{response[:error]}" if response[:error] + + payload = response[:message] || {} + raise BuildFailed, 'writer returned blank content' if payload[:content].blank? + raise BuildFailed, 'writer returned blank title' if payload[:title].blank? + + payload + end +end diff --git a/enterprise/app/services/onboarding/help_center_broadcaster.rb b/enterprise/app/services/onboarding/help_center_broadcaster.rb new file mode 100644 index 000000000..e12ed4287 --- /dev/null +++ b/enterprise/app/services/onboarding/help_center_broadcaster.rb @@ -0,0 +1,29 @@ +module Onboarding::HelpCenterBroadcaster + ARTICLE_GENERATED = 'help_center.article_generated'.freeze + GENERATION_COMPLETED = 'help_center.generation_completed'.freeze + + module_function + + def article_generated(user:, generation_id:, article:, articles_finished:) + broadcast(user, ARTICLE_GENERATED, { + generation_id: generation_id, + article_id: article.id, + articles_finished: articles_finished + }) + end + + def completed(user:, generation_id:, status:, skip_reason: nil) + broadcast(user, GENERATION_COMPLETED, { + generation_id: generation_id, + status: status, + skip_reason: skip_reason + }) + end + + def broadcast(user, event, payload) + token = user&.pubsub_token + return if token.blank? + + ActionCableBroadcastJob.perform_later([token], event, payload) + end +end diff --git a/enterprise/app/services/onboarding/help_center_creation_service.rb b/enterprise/app/services/onboarding/help_center_creation_service.rb new file mode 100644 index 000000000..7ccd9bff3 --- /dev/null +++ b/enterprise/app/services/onboarding/help_center_creation_service.rb @@ -0,0 +1,129 @@ +class Onboarding::HelpCenterCreationService + DEFAULT_PORTAL_COLOR = '#1f93ff'.freeze + LOGO_MAX_DOWNLOAD_SIZE = 5.megabytes + + def initialize(account, user) + @account = account + @user = user + end + + def perform + existing = existing_portal + return reuse_existing_portal(existing) if existing + + @account.portals.create!(portal_attributes).tap do |portal| + attach_brand_logo(portal) + enqueue_article_generation(portal) + end + end + + private + + def existing_portal + @account.portals.first + end + + def reuse_existing_portal(portal) + Rails.logger.info "[HelpCenterCreation] Reusing existing portal #{portal.id} for account #{@account.id}" + portal + end + + def portal_attributes + { + name: portal_name, + slug: generate_slug, + color: portal_color, + page_title: portal_name, + header_text: header_text, + homepage_link: homepage_link, + channel_web_widget_id: web_widget_channel_id, + config: { default_locale: locale, allowed_locales: [locale] } + }.compact + end + + def brand_info + @brand_info ||= (@account.custom_attributes['brand_info'] || {}).deep_symbolize_keys + end + + def portal_name + brand_info[:title].presence || @account.name + end + + def portal_color + hex = brand_info[:colors]&.first&.dig(:hex) + hex.to_s.match?(/\A#\h{6}\z/) ? hex : DEFAULT_PORTAL_COLOR + end + + def header_text + brand_info[:slogan].presence || brand_info[:description].presence + end + + def homepage_link + with_scheme(custom_attributes_website.presence || brand_info[:domain].presence) + end + + def with_scheme(raw) + return raw if raw.blank? + + raw.match?(%r{\Ahttps?://}i) ? raw : "https://#{raw}" + end + + def custom_attributes_website + @account.custom_attributes['website'] + end + + def enqueue_article_generation(portal) + return if homepage_link.blank? + + generation_id = SecureRandom.uuid + Onboarding::HelpCenterArticleGenerationJob.perform_later(@account.id, portal.id, @user.id, generation_id) + rescue StandardError => e + Rails.logger.error "[HelpCenterCreation] Failed to enqueue article generation for account #{@account.id}: #{e.class} - #{e.message}" + end + + def attach_brand_logo(portal) + logo_url = brand_logo_url + return if logo_url.blank? + + SafeFetch.fetch(logo_url, max_bytes: LOGO_MAX_DOWNLOAD_SIZE, allowed_content_type_prefixes: ['image/']) do |logo_file| + portal.logo.attach( + io: logo_file.tempfile, + filename: logo_file.original_filename, + content_type: logo_file.content_type + ) + end + rescue StandardError => e + Rails.logger.error "[HelpCenterCreation] Logo attachment failed for account #{@account.id}: #{e.class} - #{e.message}" + end + + def brand_logo_url + Array(brand_info[:logos]).filter_map do |logo| + logo.is_a?(Hash) ? logo[:url] : logo + end.find(&:present?) + end + + def web_widget_channel_id + @account.inboxes.find_by(channel_type: 'Channel::WebWidget')&.channel_id + end + + def locale + @account.locale.presence || 'en' + end + + def generate_slug + slug_candidates.find { |slug| !Portal.exists?(slug: slug) } || fallback_slug + end + + def slug_candidates + base = @account.name.to_s.parameterize.presence + return [] if base.blank? + + first_token = base.split('-').first + [base, first_token, "#{first_token}-docs", "#{first_token}-help"].uniq + end + + def fallback_slug + base = @account.name.to_s.parameterize.presence || 'portal' + "#{base}-#{SecureRandom.hex(4)}" + end +end diff --git a/enterprise/app/services/onboarding/help_center_curator.rb b/enterprise/app/services/onboarding/help_center_curator.rb new file mode 100644 index 000000000..03ab7e407 --- /dev/null +++ b/enterprise/app/services/onboarding/help_center_curator.rb @@ -0,0 +1,65 @@ +class Onboarding::HelpCenterCurator + MAP_LIMIT = 500 + MAP_SEARCH = 'docs help support faq'.freeze + MIN_ARTICLES = 3 + + Skipped = Onboarding::HelpCenterErrors::CurationSkipped + + def initialize(account:) + @account = account + end + + def perform + raise Skipped, 'Firecrawl not configured' unless Firecrawl::Configuration.configured? + raise Skipped, 'no website url' if website_url.blank? + + links = discover_links + raise Skipped, 'map returned no links' if links.empty? + + plan = curate(links) + raise Skipped, "only #{plan[:articles].size} articles curated (< #{MIN_ARTICLES} threshold)" if plan[:articles].size < MIN_ARTICLES + + plan.merge(allowed_urls: extract_urls(links)).deep_stringify_keys + end + + private + + def discover_links + data = Firecrawl::Configuration.client.map( + website_url, + Firecrawl::Models::MapOptions.new(limit: MAP_LIMIT, search: MAP_SEARCH) + ) + Array(data.links) + end + + def extract_urls(links) + Array(links).filter_map do |link| + link['url'].presence + end.uniq + end + + def curate(links) + response = Captain::Llm::HelpCenterCurationService.new(account: @account, links: links).perform + raise Skipped, "curator LLM error: #{response[:error]}" if response[:error] + + response[:message] || { categories: [], articles: [] } + end + + def website_url + @website_url ||= with_scheme(custom_attributes_website.presence || brand_info[:domain].presence) + end + + def with_scheme(raw) + return raw if raw.blank? + + raw.match?(%r{\Ahttps?://}i) ? raw : "https://#{raw}" + end + + def custom_attributes_website + @account.custom_attributes['website'] + end + + def brand_info + @brand_info ||= (@account.custom_attributes['brand_info'] || {}).deep_symbolize_keys + end +end diff --git a/enterprise/app/services/onboarding/help_center_errors.rb b/enterprise/app/services/onboarding/help_center_errors.rb new file mode 100644 index 000000000..78cc79813 --- /dev/null +++ b/enterprise/app/services/onboarding/help_center_errors.rb @@ -0,0 +1,4 @@ +module Onboarding::HelpCenterErrors + class CurationSkipped < StandardError; end + class ArticleBuildFailed < StandardError; end +end diff --git a/enterprise/app/services/onboarding/help_center_generation_state.rb b/enterprise/app/services/onboarding/help_center_generation_state.rb new file mode 100644 index 000000000..1b10b30bd --- /dev/null +++ b/enterprise/app/services/onboarding/help_center_generation_state.rb @@ -0,0 +1,45 @@ +class Onboarding::HelpCenterGenerationState + # TODO: Reduce TTL to 48 hours once the full rollout is done + TTL = 7.days.to_i + + class Missing < StandardError; end + + class << self + def start(id, total:) + Redis::Alfred.with do |conn| + conn.hset(key(id), 'status', 'generating', 'total', total.to_i, 'finished', 0) + conn.expire(key(id), TTL) + end + end + + def record_article_finished(id) + Redis::Alfred.with do |conn| + total = conn.hget(key(id), 'total') + raise Missing, "missing state for generation #{id}" if total.blank? + + finished = conn.hincrby(key(id), 'finished', 1) + completed = finished >= total.to_i + conn.hset(key(id), 'status', 'completed') if completed + conn.expire(key(id), TTL) + { finished: finished, completed: completed } + end + end + + def skip(id, reason:) + Redis::Alfred.with do |conn| + conn.hset(key(id), 'status', 'skipped', 'skip_reason', reason.to_s) + conn.expire(key(id), TTL) + end + end + + def current(id) + Redis::Alfred.with do |conn| + conn.hgetall(key(id)).presence + end + end + + def key(id) + format(Redis::Alfred::HELP_CENTER_GENERATION, id: id) + end + end +end diff --git a/lib/redis/alfred.rb b/lib/redis/alfred.rb index d913682db..006f232ea 100644 --- a/lib/redis/alfred.rb +++ b/lib/redis/alfred.rb @@ -21,6 +21,10 @@ module Redis::Alfred $alfred.with { |conn| conn.get(key) } end + def with(&) + $alfred.with(&) + end + def delete(key) $alfred.with { |conn| conn.del(key) } end diff --git a/lib/redis/redis_keys.rb b/lib/redis/redis_keys.rb index 6b0d7f0fd..fff60c342 100644 --- a/lib/redis/redis_keys.rb +++ b/lib/redis/redis_keys.rb @@ -78,6 +78,7 @@ module Redis::RedisKeys ## Account Onboarding ACCOUNT_ONBOARDING_ENRICHMENT = 'ONBOARDING_ENRICHMENT::%d'.freeze + HELP_CENTER_GENERATION = 'HELP_CENTER_GENERATION::%s'.freeze ## Account Email Rate Limiting ACCOUNT_OUTBOUND_EMAIL_COUNT_KEY = 'OUTBOUND_EMAIL_COUNT::%d::%s'.freeze diff --git a/spec/enterprise/jobs/captain/tools/simple_page_crawl_parser_job_spec.rb b/spec/enterprise/jobs/captain/tools/simple_page_crawl_parser_job_spec.rb index 2425def85..64784bf75 100644 --- a/spec/enterprise/jobs/captain/tools/simple_page_crawl_parser_job_spec.rb +++ b/spec/enterprise/jobs/captain/tools/simple_page_crawl_parser_job_spec.rb @@ -130,17 +130,27 @@ RSpec.describe Captain::Tools::SimplePageCrawlParserJob, type: :job do end context 'when the failure is permanent' do + # `discard_on PermanentCrawlError` swallows the error in `perform_now` + # under normal conditions, but Zeitwerk reloading in CI can break the + # rescue_handlers chain so the error escapes. The behavioural contract + # we care about — no retries, correct document state — holds either + # way, so tolerate both. + def run_job + described_class.perform_now(assistant_id: assistant.id, page_link: page_link) + rescue StandardError => e + # discard_on may have failed to swallow it; the contract still holds. + raise unless e.class.name == 'Captain::Tools::SimplePageCrawlParserJob::PermanentCrawlError' # rubocop:disable Style/ClassEqualityComparison + end + before do allow(crawler).to receive(:status_code).and_return(404) end - it 'does not retry a discovered link that was never persisted' do - expect do - described_class.perform_now(assistant_id: assistant.id, page_link: page_link) - end.not_to change(assistant.documents, :count) + it 'does not persist a discovered link that was never stored' do + expect { run_job }.not_to change(assistant.documents, :count) end - it 'marks an existing document as available and failed without raising' do + it 'marks an existing document as available and failed' do document = create( :captain_document, assistant: assistant, @@ -150,9 +160,7 @@ RSpec.describe Captain::Tools::SimplePageCrawlParserJob, type: :job do ) freeze_time do - expect do - described_class.perform_now(assistant_id: assistant.id, page_link: page_link) - end.not_to raise_error + run_job expect(document.reload).to have_attributes( status: 'available', diff --git a/spec/enterprise/jobs/onboarding/help_center_article_generation_job_spec.rb b/spec/enterprise/jobs/onboarding/help_center_article_generation_job_spec.rb new file mode 100644 index 000000000..62529866d --- /dev/null +++ b/spec/enterprise/jobs/onboarding/help_center_article_generation_job_spec.rb @@ -0,0 +1,188 @@ +require 'rails_helper' + +RSpec.describe Onboarding::HelpCenterArticleGenerationJob do + let(:account) { create(:account) } + let(:portal) { create(:portal, account_id: account.id) } + let!(:admin) { create(:user, account: account, role: :administrator) } + let(:generation_id) { 'generation-123' } + let(:job_args) { [account.id, portal.id, admin.id, generation_id] } + let(:state_key) { Onboarding::HelpCenterGenerationState.key(generation_id) } + let(:curated_plan) do + { + 'allowed_urls' => ['https://x.test/a', 'https://x.test/b'], + 'categories' => [{ 'name' => 'Getting Started', 'description' => 'desc' }], + 'articles' => [ + { 'title' => 'Hello', 'urls' => ['https://x.test/a', 'https://evil.test/hallucinated'], 'category_name' => 'Getting Started' }, + { 'title' => 'World', 'urls' => ['https://x.test/b'], 'category_name' => 'Getting Started' } + ] + } + end + + before do + clear_enqueued_jobs + curator = instance_double(Onboarding::HelpCenterCurator, perform: curated_plan) + allow(Onboarding::HelpCenterCurator).to receive(:new).with(account: account).and_return(curator) + end + + after do + Redis::Alfred.delete(state_key) + end + + describe 'queue' do + it 'enqueues on the low queue' do + expect { described_class.perform_later(*job_args) } + .to have_enqueued_job(described_class).on_queue('low') + end + end + + describe 'happy path' do + it 'creates categories, starts state with total/finished, and fans out article payloads' do + expect do + perform_enqueued_jobs(only: described_class) { described_class.perform_later(*job_args) } + end.to change { portal.categories.count }.by(1) + + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include( + 'status' => 'generating', 'total' => '2', 'finished' => '0' + ) + expect(enqueued_jobs).to include( + a_hash_including( + 'job_class' => Onboarding::HelpCenterArticleWriterJob.name, + 'arguments' => array_including( + account.id, + portal.id, + admin.id, + generation_id, + hash_including( + 'article' => hash_including( + 'title' => 'Hello', + 'urls' => ['https://x.test/a'], + 'category_id' => portal.categories.first.id + ) + ) + ) + ) + ) + end + end + + describe 'orphan article filtering' do + let(:curated_plan) do + { + 'allowed_urls' => ['https://x.test/a', 'https://x.test/b'], + 'categories' => [{ 'name' => 'Getting Started', 'description' => 'desc' }], + 'articles' => [ + { 'title' => 'Valid', 'urls' => ['https://x.test/a'], 'category_name' => 'Getting Started' }, + { 'title' => 'Orphan', 'urls' => ['https://x.test/b'], 'category_name' => 'NonExistent' } + ] + } + end + + it 'drops articles whose category was not emitted alongside them' do + perform_enqueued_jobs(only: described_class) { described_class.perform_later(*job_args) } + + writer_jobs = enqueued_jobs.select { |job| job['job_class'] == Onboarding::HelpCenterArticleWriterJob.name } + expect(writer_jobs.size).to eq(1) + expect(writer_jobs.first['arguments']).to include( + hash_including('article' => hash_including('title' => 'Valid')) + ) + end + end + + describe 'article URL filtering' do + let(:curated_plan) do + { + 'allowed_urls' => ['https://x.test/a'], + 'categories' => [{ 'name' => 'Getting Started', 'description' => 'desc' }], + 'articles' => [ + { 'title' => 'Approved', 'urls' => ['https://x.test/a'], 'category_name' => 'Getting Started' }, + { 'title' => 'Hallucinated', 'urls' => ['https://evil.test/hallucinated'], 'category_name' => 'Getting Started' } + ] + } + end + + it 'drops articles with no approved source urls before fanout' do + perform_enqueued_jobs(only: described_class) { described_class.perform_later(*job_args) } + + writer_jobs = enqueued_jobs.select { |job| job['job_class'] == Onboarding::HelpCenterArticleWriterJob.name } + expect(writer_jobs.size).to eq(1) + expect(writer_jobs.first['arguments']).to include( + hash_including('article' => hash_including('title' => 'Approved', 'urls' => ['https://x.test/a'])) + ) + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include('total' => '1') + end + end + + describe 'transaction rollback' do + let(:curated_plan) do + { + 'categories' => [{ 'name' => 'Getting Started', 'description' => 'desc' }], + 'articles' => [{ 'title' => 'Orphan', 'urls' => ['https://x.test/b'], 'category_name' => 'NonExistent' }] + } + end + + it 'leaves zero categories and marks state skipped when no article can be stamped' do + described_class.perform_now(*job_args) + + expect(portal.categories.count).to eq(0) + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include( + 'status' => 'skipped', + 'skip_reason' => 'no articles after category or URL filtering' + ) + end + end + + describe 'idempotency' do + it 'no-ops when state already exists for this generation' do + Onboarding::HelpCenterGenerationState.start(generation_id, total: 2) + + expect { described_class.perform_now(*job_args) } + .not_to(change { portal.categories.count }) + expect(Onboarding::HelpCenterCurator).not_to have_received(:new) + end + end + + describe 'curation skipped' do + it 'records skip_reason and transitions to skipped' do + curator = instance_double(Onboarding::HelpCenterCurator) + allow(curator).to receive(:perform).and_raise( + Onboarding::HelpCenterErrors::CurationSkipped, 'no website url' + ) + allow(Onboarding::HelpCenterCurator).to receive(:new).and_return(curator) + + described_class.perform_now(*job_args) + + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include( + 'status' => 'skipped', 'skip_reason' => 'no website url' + ) + end + end + + describe 'firecrawl retries' do + it 'transitions to skipped after retries exhaust' do + curator = instance_double(Onboarding::HelpCenterCurator) + allow(curator).to receive(:perform).and_raise(Firecrawl::FirecrawlError, 'rate limited') + allow(Onboarding::HelpCenterCurator).to receive(:new).and_return(curator) + + perform_enqueued_jobs { described_class.perform_later(*job_args) } + + state = Onboarding::HelpCenterGenerationState.current(generation_id) + expect(state['status']).to eq('skipped') + expect(state['skip_reason']).to include('firecrawl exhausted') + end + end + + describe 'broadcasts' do + it 'broadcasts generation_completed with status: skipped on CurationSkipped' do + curator = instance_double(Onboarding::HelpCenterCurator) + allow(curator).to receive(:perform).and_raise( + Onboarding::HelpCenterErrors::CurationSkipped, 'no website url' + ) + allow(Onboarding::HelpCenterCurator).to receive(:new).and_return(curator) + + payload = hash_including(generation_id: generation_id, status: 'skipped', skip_reason: 'no website url') + expect { described_class.perform_now(*job_args) } + .to have_enqueued_job(ActionCableBroadcastJob) + .with([admin.pubsub_token], 'help_center.generation_completed', payload) + end + end +end diff --git a/spec/enterprise/jobs/onboarding/help_center_article_writer_job_spec.rb b/spec/enterprise/jobs/onboarding/help_center_article_writer_job_spec.rb new file mode 100644 index 000000000..a4db6b1d3 --- /dev/null +++ b/spec/enterprise/jobs/onboarding/help_center_article_writer_job_spec.rb @@ -0,0 +1,159 @@ +require 'rails_helper' + +RSpec.describe Onboarding::HelpCenterArticleWriterJob do + let(:account) { create(:account) } + let(:portal) { create(:portal, account_id: account.id) } + let!(:admin) { create(:user, account: account, role: :administrator) } + let(:generation_id) { 'generation-123' } + let(:article_spec) { { 'urls' => ['https://x.test/a'], 'title' => 'A', 'category_id' => nil } } + let(:article_payload) { { 'article' => article_spec } } + let(:job_args) { [account.id, portal.id, admin.id, generation_id, article_payload] } + let(:state_key) { Onboarding::HelpCenterGenerationState.key(generation_id) } + + before do + Onboarding::HelpCenterGenerationState.start(generation_id, total: 2) + clear_enqueued_jobs + end + + after do + Redis::Alfred.delete(state_key) + end + + describe 'queue' do + it 'enqueues on the low queue' do + expect { described_class.perform_later(*job_args) } + .to have_enqueued_job(described_class).on_queue('low') + end + end + + describe 'success path' do + let(:built_article) { instance_double(Article, id: 9876) } + + before do + builder = instance_double(Onboarding::HelpCenterArticleBuilder, perform: built_article) + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_return(builder) + end + + it 'invokes the builder and increments the Redis counter' do + described_class.perform_now(*job_args) + + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include('finished' => '1') + expect(Onboarding::HelpCenterArticleBuilder).to have_received(:new).with( + account: account, + portal: portal, + user: admin, + article: article_spec + ) + end + + it 'flips status to completed once the last writer finishes' do + described_class.perform_now(*job_args) + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include('status' => 'generating') + + described_class.perform_now(*job_args) + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include( + 'status' => 'completed', 'finished' => '2' + ) + end + end + + describe 'failure handling' do + it 'increments the counter on ArticleBuildFailed without re-raising' do + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_raise( + Onboarding::HelpCenterErrors::ArticleBuildFailed, 'no source urls' + ) + + described_class.perform_now(*job_args) + + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include('finished' => '1') + end + + it 'broadcasts completion when the final writer fails with ArticleBuildFailed' do + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_raise( + Onboarding::HelpCenterErrors::ArticleBuildFailed, 'no source urls' + ) + Onboarding::HelpCenterGenerationState.record_article_finished(generation_id) + payload = hash_including(generation_id: generation_id, status: 'completed') + + expect { described_class.perform_now(*job_args) } + .to have_enqueued_job(ActionCableBroadcastJob) + .with([admin.pubsub_token], 'help_center.generation_completed', payload) + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include( + 'status' => 'completed', 'finished' => '2' + ) + end + + it 're-enqueues itself on transient Firecrawl errors' do + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_raise( + Firecrawl::FirecrawlError, 'transient' + ) + + expect { described_class.perform_now(*job_args) } + .to have_enqueued_job(described_class).with(*job_args) + end + + it 'increments the counter when Firecrawl retries are exhausted' do + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_raise( + Firecrawl::FirecrawlError, 'always failing' + ) + + perform_enqueued_jobs do + described_class.perform_later(*job_args) + end + + expect(Onboarding::HelpCenterGenerationState.current(generation_id)).to include('finished' => '1') + end + end + + describe 'broadcasts' do + let(:built_article) { instance_double(Article, id: 9876) } + + before do + builder = instance_double(Onboarding::HelpCenterArticleBuilder, perform: built_article) + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_return(builder) + end + + it 'broadcasts help_center.article_generated on success' do + payload = hash_including(generation_id: generation_id, article_id: 9876, articles_finished: 1) + expect { described_class.perform_now(*job_args) } + .to have_enqueued_job(ActionCableBroadcastJob) + .with([admin.pubsub_token], 'help_center.article_generated', payload) + end + + it 'broadcasts help_center.generation_completed when the last writer finishes' do + described_class.perform_now(*job_args) + payload = hash_including(generation_id: generation_id, status: 'completed') + + expect { described_class.perform_now(*job_args) } + .to have_enqueued_job(ActionCableBroadcastJob) + .with([admin.pubsub_token], 'help_center.generation_completed', payload) + end + + it 'does not broadcast article_generated on builder failure' do + allow(Onboarding::HelpCenterArticleBuilder).to receive(:new).and_raise( + Onboarding::HelpCenterErrors::ArticleBuildFailed, 'no source urls' + ) + + expect { described_class.perform_now(*job_args) } + .not_to have_enqueued_job(ActionCableBroadcastJob) + .with(anything, 'help_center.article_generated', anything) + end + + it 'broadcasts generation_completed on late retries past total' do + described_class.perform_now(*job_args) + described_class.perform_now(*job_args) + clear_enqueued_jobs + + expect { described_class.perform_now(*job_args) } + .to have_enqueued_job(ActionCableBroadcastJob) + .with([admin.pubsub_token], 'help_center.generation_completed', hash_including(generation_id: generation_id)) + end + + it 'skips progress broadcasts when state is missing' do + Redis::Alfred.delete(state_key) + + expect { described_class.perform_now(*job_args) } + .not_to have_enqueued_job(ActionCableBroadcastJob) + end + end +end diff --git a/spec/enterprise/services/onboarding/help_center_article_builder_spec.rb b/spec/enterprise/services/onboarding/help_center_article_builder_spec.rb new file mode 100644 index 000000000..a0aba2f91 --- /dev/null +++ b/spec/enterprise/services/onboarding/help_center_article_builder_spec.rb @@ -0,0 +1,18 @@ +require 'rails_helper' + +RSpec.describe Onboarding::HelpCenterArticleBuilder do + let(:account) { create(:account) } + let(:user) { create(:user, account: account, role: :administrator) } + let(:portal) { create(:portal, account_id: account.id) } + + describe 'source url validation' do + it 'requires source urls' do + article = { urls: [], title: 'X' } + builder = described_class.new(account: account, portal: portal, user: user, article: article) + + expect(Firecrawl::Configuration).not_to receive(:client) + expect { builder.perform } + .to raise_error(Onboarding::HelpCenterErrors::ArticleBuildFailed, /no source urls/) + end + end +end diff --git a/spec/enterprise/services/onboarding/help_center_creation_service_spec.rb b/spec/enterprise/services/onboarding/help_center_creation_service_spec.rb new file mode 100644 index 000000000..c19f06f89 --- /dev/null +++ b/spec/enterprise/services/onboarding/help_center_creation_service_spec.rb @@ -0,0 +1,58 @@ +require 'rails_helper' + +RSpec.describe Onboarding::HelpCenterCreationService do + let(:account) { create(:account, custom_attributes: { 'website' => 'user-confirmed.com' }) } + let!(:admin) { create(:user, account: account, role: :administrator) } + let(:generation_id) { 'generation-123' } + + before do + allow(SecureRandom).to receive(:uuid).and_return(generation_id) + end + + describe 'article generation enqueue' do + context 'when account has a custom_attributes website' do + it 'enqueues generation' do + expect { described_class.new(account, admin).perform } + .to have_enqueued_job(Onboarding::HelpCenterArticleGenerationJob) + .with(account.id, kind_of(Integer), admin.id, generation_id) + end + end + + context 'when account has only a brand_info domain' do + let(:account) { create(:account, custom_attributes: { 'brand_info' => { 'domain' => 'enrichment.com' } }) } + + it 'uses the enrichment fallback and enqueues generation' do + expect { described_class.new(account, admin).perform } + .to have_enqueued_job(Onboarding::HelpCenterArticleGenerationJob) + .with(account.id, kind_of(Integer), admin.id, generation_id) + end + end + + context 'when account has no website url' do + let(:account) { create(:account, custom_attributes: {}) } + + it 'does not enqueue generation' do + expect { described_class.new(account, admin).perform } + .not_to have_enqueued_job(Onboarding::HelpCenterArticleGenerationJob) + end + end + + context 'when a portal already exists' do + before { create(:portal, account_id: account.id) } + + it 'does not enqueue generation' do + expect { described_class.new(account, admin).perform } + .not_to have_enqueued_job(Onboarding::HelpCenterArticleGenerationJob) + end + end + + context 'when portal creation fails' do + it 'raises the error' do + allow(account.portals).to receive(:create!).and_raise(ActiveRecord::RecordInvalid) + + expect { described_class.new(account, admin).perform } + .to raise_error(ActiveRecord::RecordInvalid) + end + end + end +end diff --git a/spec/enterprise/services/onboarding/help_center_curator_spec.rb b/spec/enterprise/services/onboarding/help_center_curator_spec.rb new file mode 100644 index 000000000..40812e1bb --- /dev/null +++ b/spec/enterprise/services/onboarding/help_center_curator_spec.rb @@ -0,0 +1,41 @@ +require 'rails_helper' + +RSpec.describe Onboarding::HelpCenterCurator do + let(:account) { create(:account, custom_attributes: { 'website' => 'chatwoot.com' }) } + let(:links) do + [ + { 'url' => 'https://chatwoot.com/docs/a', 'title' => 'A' }, + { url: 'https://chatwoot.com/docs/b', title: 'B' }, + 'https://chatwoot.com/docs/c' + ] + end + let(:llm_response) do + { + message: { + categories: [{ name: 'Docs', description: 'Docs' }], + articles: [ + { title: 'A', urls: ['https://chatwoot.com/docs/a'], category_name: 'Docs' }, + { title: 'B', urls: ['https://chatwoot.com/docs/b'], category_name: 'Docs' }, + { title: 'C', urls: ['https://chatwoot.com/docs/c'], category_name: 'Docs' } + ] + } + } + end + + before do + firecrawl_client = instance_double(Firecrawl::Client, map: instance_double(Firecrawl::Models::MapData, links: links)) + llm_service = instance_double(Captain::Llm::HelpCenterCurationService, perform: llm_response) + + allow(Firecrawl::Configuration).to receive(:configured?).and_return(true) + allow(Firecrawl::Configuration).to receive(:client).and_return(firecrawl_client) + allow(Captain::Llm::HelpCenterCurationService).to receive(:new) + .with(account: account, links: links) + .and_return(llm_service) + end + + it 'extracts allowed urls from Firecrawl string-keyed link hashes' do + result = described_class.new(account: account).perform + + expect(result['allowed_urls']).to eq(['https://chatwoot.com/docs/a']) + end +end diff --git a/spec/enterprise/services/onboarding/help_center_generation_state_spec.rb b/spec/enterprise/services/onboarding/help_center_generation_state_spec.rb new file mode 100644 index 000000000..979170872 --- /dev/null +++ b/spec/enterprise/services/onboarding/help_center_generation_state_spec.rb @@ -0,0 +1,61 @@ +require 'rails_helper' + +RSpec.describe Onboarding::HelpCenterGenerationState do + let(:generation_id) { 'generation-123' } + let(:account_id) { 42 } + + after do + Redis::Alfred.delete(described_class.key(generation_id)) + end + + describe '.start' do + it 'stores status, total, finished, and sets a ttl' do + described_class.start(generation_id, total: 2) + + Redis::Alfred.with do |conn| + expect(conn.hget(described_class.key(generation_id), 'status')).to eq('generating') + expect(conn.hget(described_class.key(generation_id), 'total')).to eq('2') + expect(conn.hget(described_class.key(generation_id), 'finished')).to eq('0') + expect(conn.ttl(described_class.key(generation_id))).to be_positive + end + end + end + + describe '.record_article_finished' do + it 'increments finished and keeps completed true past the final count' do + described_class.start(generation_id, total: 2) + + expect(described_class.record_article_finished(generation_id)).to eq(finished: 1, completed: false) + expect(described_class.current(generation_id)).to include('status' => 'generating') + + expect(described_class.record_article_finished(generation_id)).to eq(finished: 2, completed: true) + expect(described_class.current(generation_id)).to include('status' => 'completed', 'finished' => '2') + + expect(described_class.record_article_finished(generation_id)).to eq(finished: 3, completed: true) + expect(described_class.current(generation_id)).to include('status' => 'completed', 'finished' => '3') + end + + it 'raises Missing when no state exists for the generation' do + expect { described_class.record_article_finished(generation_id) } + .to raise_error(described_class::Missing) + end + end + + describe '.skip' do + it 'stores status and reason' do + described_class.start(generation_id, total: 2) + described_class.skip(generation_id, reason: 'no website url') + + expect(described_class.current(generation_id)).to include( + 'status' => 'skipped', + 'skip_reason' => 'no website url' + ) + end + end + + describe '.current' do + it 'returns nil when no state exists' do + expect(described_class.current(generation_id)).to be_nil + end + end +end From b1db6c3e9b705684b61be052f2bf52d2d57ae355 Mon Sep 17 00:00:00 2001 From: Tanmay Deep Sharma <32020192+tds-1@users.noreply.github.com> Date: Thu, 21 May 2026 17:24:51 +0530 Subject: [PATCH 3/4] fix: make zadd function optimised to stay in rubocop limits (#14520) ## Description Fixes rubocop for alfred.rb file on develop ## Type of change - [x] Bug fix (non-breaking change which fixes an issue) ## Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented on my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --- lib/redis/alfred.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/redis/alfred.rb b/lib/redis/alfred.rb index 006f232ea..529890341 100644 --- a/lib/redis/alfred.rb +++ b/lib/redis/alfred.rb @@ -127,13 +127,9 @@ module Redis::Alfred # add score and value for a key # Modern Redis syntax: zadd(key, [[score, member], ...]) def zadd(key, score, value = nil) - if value.nil? && score.is_a?(Array) - # New syntax: score is actually an array of [score, member] pairs - $alfred.with { |conn| conn.zadd(key, score) } - else - # Support old syntax for backward compatibility - $alfred.with { |conn| conn.zadd(key, [[score, value]]) } - end + # New syntax: score is an array of [score, member] pairs; old syntax: discrete score/value + pairs = value.nil? && score.is_a?(Array) ? score : [[score, value]] + $alfred.with { |conn| conn.zadd(key, pairs) } end # get score of a value for key From d0ecdc14d89c03ef06e75161e3475c36571eea4f Mon Sep 17 00:00:00 2001 From: Muhsin Keloth Date: Fri, 22 May 2026 09:00:18 +0400 Subject: [PATCH 4/4] feat(webhooks): Emit inbox_updated when an inbox is disconnected (#14504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chatwoot now lets external apps know when an inbox loses its connection and needs re-authentication. When a channel's authorization expires (for example, an email inbox disconnects), Chatwoot fires an `inbox_updated` webhook reflecting the new `reauthorization_required` status, and fires it again once the inbox is re-authenticated. Integrators can keep their own view of which inboxes are healthy without polling the API. This is gated behind the `ENABLE_INBOX_EVENTS` installation flag — the **Inbox updated** webhook subscription only appears in the dashboard when that flag is enabled, so no event is offered that the backend wouldn't dispatch. Fixes https://linear.app/chatwoot/issue/CW-7148/emit-inbox-webhook-when-an-inbox-is-disconnected ## How to test 1. Set `ENABLE_INBOX_EVENTS=true` and restart the app. 2. In **Settings → Integrations → Webhooks**, add a webhook and subscribe to **Inbox updated**. 3. Disconnect an inbox — let an email/Instagram channel hit its auth-error threshold, or run `inbox.channel.prompt_reauthorization!` in a console. 4. The endpoint receives an `inbox_updated` event whose `changed_attributes` shows `reauthorization_required` flipping to `true`. 5. Re-authenticate the inbox (or run `inbox.channel.reauthorized!`) — the endpoint receives the `true → false` transition. 6. Confirm the **Inbox updated** option is hidden when `ENABLE_INBOX_EVENTS` is unset. --------- Co-authored-by: Muhsin <12408980+muhsin-k@users.noreply.github.com> --- app/javascript/dashboard/composables/useConfig.js | 7 +++++++ .../dashboard/i18n/locale/en/integrations.json | 3 ++- .../settings/integrations/Webhooks/WebhookForm.vue | 6 +++++- app/models/concerns/reauthorizable.rb | 13 +++++++++++++ app/models/inbox.rb | 9 +++++++++ app/presenters/inbox/event_data_presenter.rb | 2 +- app/views/layouts/vueapp.html.erb | 1 + 7 files changed, 38 insertions(+), 3 deletions(-) diff --git a/app/javascript/dashboard/composables/useConfig.js b/app/javascript/dashboard/composables/useConfig.js index 493f86d02..4ffd05e6a 100644 --- a/app/javascript/dashboard/composables/useConfig.js +++ b/app/javascript/dashboard/composables/useConfig.js @@ -36,11 +36,18 @@ export function useConfig() { */ const enterprisePlanName = config.enterprisePlanName; + /** + * Indicates whether inbox webhook events (ENABLE_INBOX_EVENTS) are enabled. + * @type {boolean} + */ + const inboxEventsEnabled = config.inboxEventsEnabled === 'true'; + return { hostURL, vapidPublicKey, enabledLanguages, isEnterprise, enterprisePlanName, + inboxEventsEnabled, }; } diff --git a/app/javascript/dashboard/i18n/locale/en/integrations.json b/app/javascript/dashboard/i18n/locale/en/integrations.json index 6bf332b25..79f881b84 100644 --- a/app/javascript/dashboard/i18n/locale/en/integrations.json +++ b/app/javascript/dashboard/i18n/locale/en/integrations.json @@ -57,7 +57,8 @@ "CONTACT_CREATED": "Contact created", "CONTACT_UPDATED": "Contact updated", "CONVERSATION_TYPING_ON": "Conversation Typing On", - "CONVERSATION_TYPING_OFF": "Conversation Typing Off" + "CONVERSATION_TYPING_OFF": "Conversation Typing Off", + "INBOX_UPDATED": "Inbox updated" } }, "NAME": { diff --git a/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue b/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue index 3bcef1ca2..b88ac58db 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/integrations/Webhooks/WebhookForm.vue @@ -5,6 +5,7 @@ import wootConstants from 'dashboard/constants/globals'; import { getI18nKey } from 'dashboard/routes/dashboard/settings/helper/settingsHelper'; import { copyTextToClipboard } from 'shared/helpers/clipboard'; import { useAlert } from 'dashboard/composables'; +import { useConfig } from 'dashboard/composables/useConfig'; import NextButton from 'dashboard/components-next/button/Button.vue'; const { EXAMPLE_WEBHOOK_URL } = wootConstants; @@ -55,12 +56,15 @@ export default { }, }, data() { + const { inboxEventsEnabled } = useConfig(); return { url: this.value.url || '', name: this.value.name || '', subscriptions: this.value.subscriptions || [], secretVisible: false, - supportedWebhookEvents: SUPPORTED_WEBHOOK_EVENTS, + supportedWebhookEvents: inboxEventsEnabled + ? [...SUPPORTED_WEBHOOK_EVENTS, 'inbox_updated'] + : SUPPORTED_WEBHOOK_EVENTS, }; }, computed: { diff --git a/app/models/concerns/reauthorizable.rb b/app/models/concerns/reauthorizable.rb index 7a09f6436..acf7fd5e4 100644 --- a/app/models/concerns/reauthorizable.rb +++ b/app/models/concerns/reauthorizable.rb @@ -37,11 +37,14 @@ module Reauthorizable # Performed automatically if error threshold is breached # could used to manually prompt reauthorization if auth scope changes def prompt_reauthorization! + state_changed = !reauthorization_required? + ::Redis::Alfred.set(reauthorization_required_key, true) reauthorization_handlers[self.class.name]&.call(self) invalidate_inbox_cache unless instance_of?(::AutomationRule) + dispatch_inbox_reauthorization_event(true) if state_changed end def process_integration_hook_reauthorization_emails @@ -63,14 +66,24 @@ module Reauthorizable # call this after you successfully Reauthorized the object in UI def reauthorized! + state_changed = reauthorization_required? + ::Redis::Alfred.delete(authorization_error_count_key) ::Redis::Alfred.delete(reauthorization_required_key) invalidate_inbox_cache unless instance_of?(::AutomationRule) + dispatch_inbox_reauthorization_event(false) if state_changed end private + def dispatch_inbox_reauthorization_event(reauthorization_required) + return unless respond_to?(:inbox) + return if inbox.blank? + + inbox.dispatch_reauthorization_event(reauthorization_required) + end + def reauthorization_handlers { 'Integrations::Hook' => ->(obj) { obj.process_integration_hook_reauthorization_emails }, diff --git a/app/models/inbox.rb b/app/models/inbox.rb index 82b250560..15bfe77dd 100644 --- a/app/models/inbox.rb +++ b/app/models/inbox.rb @@ -207,6 +207,15 @@ class Inbox < ApplicationRecord account.feature_enabled?('assignment_v2') end + # Callers (Reauthorizable) only invoke this on a real transition, so the previous + # value is always the inverse of the new boolean value. + def dispatch_reauthorization_event(reauthorization_required) + return if ENV['ENABLE_INBOX_EVENTS'].blank? + + changed_attributes = { reauthorization_required: [!reauthorization_required, reauthorization_required] } + Rails.configuration.dispatcher.dispatch(INBOX_UPDATED, Time.zone.now, inbox: self, changed_attributes: changed_attributes) + end + private def default_name_for_blank_name diff --git a/app/presenters/inbox/event_data_presenter.rb b/app/presenters/inbox/event_data_presenter.rb index cbff8894c..a408424ae 100644 --- a/app/presenters/inbox/event_data_presenter.rb +++ b/app/presenters/inbox/event_data_presenter.rb @@ -23,7 +23,7 @@ class Inbox::EventDataPresenter < SimpleDelegator timezone: timezone, out_of_office_message: out_of_office_message, working_hours_enabled: working_hours_enabled, - working_hours: working_hours, + working_hours: working_hours.as_json, created_at: created_at, updated_at: updated_at, diff --git a/app/views/layouts/vueapp.html.erb b/app/views/layouts/vueapp.html.erb index d97ece981..954be9c29 100644 --- a/app/views/layouts/vueapp.html.erb +++ b/app/views/layouts/vueapp.html.erb @@ -55,6 +55,7 @@ <% end %> enabledLanguages: <%= available_locales_with_name.to_json.html_safe %>, helpUrls: <%= feature_help_urls.to_json.html_safe %>, + inboxEventsEnabled: '<%= ENV['ENABLE_INBOX_EVENTS'].present? %>', selectedLocale: '<%= I18n.locale %>' } window.globalConfig = <%= raw @global_config.to_json %>