From 8fc5c7a5c890193f142c5ffa8945f5a4a2b24ce0 Mon Sep 17 00:00:00 2001 From: Muhsin Keloth Date: Mon, 13 Jul 2026 13:26:09 +0400 Subject: [PATCH 1/2] feat: add captain general guidelines migration helpers (#14909) This PR adds internal tooling and planning docs for migrating existing Captain assistant instructions into the new General Guidelines structure. **Summary** This PR adds a controlled migration path for moving existing Captain V1 assistant instructions into the structured Captain architecture. It introduces a classifier that reads the current `config.instructions` and produces reviewed migration drafts with separate sections for: - assistant description / business context - response guidelines - guardrails - scenario candidates - conversation messages - FAQ/document candidates - needs-review items The migration is intentionally staged. It only targets V1-style assistants that still have custom instructions, are connected to inboxes, and do not already have structured response guidelines, guardrails, or scenario records. When applied, the task writes the extracted business context to the assistant description, response guidelines to `response_guidelines`, guardrails to `guardrails`, and stores scenario candidates / FAQ candidates / review notes under `config["assistant_migration"]`. Scenario candidates are also flattened into response guidelines for now so customer behavior is preserved before we create real `Captain::Scenario` records in a later rollout. The applier stores the original assistant values under migration metadata so conversation message config can be restored if needed. It does not create scenario records yet. **How to generate drafts** For specific assistant IDs: ```bash bundle exec rake captain:assistant_migration:generate \ IDS=546,636,819 \ LIMIT=0 \ OUTPUT=tmp/captain_migration_drafts.jsonl ``` For the first 50 eligible assistants: ```bash bundle exec rake captain:assistant_migration:generate \ OUTPUT=tmp/captain_migration_drafts.jsonl ``` For all eligible assistants: ```bash bundle exec rake captain:assistant_migration:generate \ LIMIT=0 \ OUTPUT=tmp/captain_migration_drafts.jsonl ``` **How to apply drafts** Dry run first: ```bash bundle exec rake captain:assistant_migration:apply \ INPUT=tmp/captain_migration_drafts.jsonl \ DRY_RUN=true ``` Apply changes: ```bash bundle exec rake captain:assistant_migration:apply \ INPUT=tmp/captain_migration_drafts.jsonl \ DRY_RUN=false ``` **How to restore conversation messages** If extracted `welcome_message`, `handoff_message`, or `resolution_message` need to be reverted to their pre-migration values: ```bash bundle exec rake captain:assistant_migration:restore_messages \ IDS=546,636,819 \ DRY_RUN=true ``` ```bash bundle exec rake captain:assistant_migration:restore_messages \ IDS=546,636,819 \ DRY_RUN=false ``` **Notes** - `LIMIT=0` means no limit. - `generate` overwrites the output file. - The apply task skips assistants that are no longer V1 migration candidates. - This PR does not create `Captain::Scenario` records; scenario candidates are staged in assistant config for a future migration. --------- Co-authored-by: Muhsin <12408980+muhsin-k@users.noreply.github.com> Co-authored-by: aakashb95 Co-authored-by: Aakash Bakhle <48802744+aakashb95@users.noreply.github.com> --- .../assistant_migration/draft_applier.rb | 199 +++++++++++++ .../instruction_classifier.rb | 148 ++++++++++ .../instruction_classifier_schema.rb | 91 ++++++ .../prompts/instruction_classifier.liquid | 137 +++++++++ lib/tasks/captain_assistant_migration.rake | 278 ++++++++++++++++++ .../assistant_migration/draft_applier_spec.rb | 116 ++++++++ 6 files changed, 969 insertions(+) create mode 100644 enterprise/app/services/captain/assistant_migration/draft_applier.rb create mode 100644 enterprise/app/services/captain/assistant_migration/instruction_classifier.rb create mode 100644 enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb create mode 100644 enterprise/lib/captain/prompts/instruction_classifier.liquid create mode 100644 lib/tasks/captain_assistant_migration.rake create mode 100644 spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb diff --git a/enterprise/app/services/captain/assistant_migration/draft_applier.rb b/enterprise/app/services/captain/assistant_migration/draft_applier.rb new file mode 100644 index 000000000..df03624b4 --- /dev/null +++ b/enterprise/app/services/captain/assistant_migration/draft_applier.rb @@ -0,0 +1,199 @@ +class Captain::AssistantMigration::DraftApplier + ASSISTANT_DESCRIPTION_LIMIT = 500 + CONFIG_KEY = 'assistant_migration'.freeze + SCENARIO_DESCRIPTION_LIMIT = 500 + ORIGINAL_VALUES_KEY = 'original_values'.freeze + + pattr_initialize [:assistant!, :draft!, { dry_run: true }] + + def perform + changes = build_changes + apply_changes(changes) unless dry_run + + { + assistant_id: assistant.id, + dry_run: dry_run, + changes: changes + } + end + + private + + def build_changes + { + description: description_change, + response_guidelines: array_change(:response_guidelines, response_guidelines), + guardrails: array_change(:guardrails, guardrails), + config: config_change + }.compact + end + + def apply_changes(changes) + assistant.transaction do + assistant.update!(assistant_update_attributes(changes)) if assistant_update_attributes(changes).present? + end + end + + def assistant_update_attributes(changes) + {}.tap do |attributes| + attributes[:description] = changes.dig(:description, :to) if changes[:description].present? + attributes[:response_guidelines] = changes.dig(:response_guidelines, :to) if changes[:response_guidelines].present? + attributes[:guardrails] = changes.dig(:guardrails, :to) if changes[:guardrails].present? + attributes[:config] = changes.dig(:config, :to) if changes[:config].present? + end + end + + def description_change + value = assistant_description_value + return if value.blank? || value == assistant.description + + { from: assistant.description, to: value } + end + + def assistant_description_value + value = item_values(:business_product_context).join(' ').presence + return if value.blank? + + raise ArgumentError, "Assistant description exceeds #{ASSISTANT_DESCRIPTION_LIMIT} characters" if value.length > ASSISTANT_DESCRIPTION_LIMIT + + value + end + + def response_guidelines + (item_values(:response_guidelines) + scenario_response_guidelines).uniq + end + + def guardrails + item_values(:guardrails) + end + + def array_change(field, values) + return if values.blank? + + current = Array(assistant.public_send(field)).map(&:to_s) + return if current == values + + { from: current, to: values } + end + + def config_change + updated_config = assistant.config.deep_dup + conversation_messages.each do |key, value| + next if value.blank? + next if updated_config[key].present? + + updated_config[key] = value + end + updated_config[CONFIG_KEY] = migration_config + + return if updated_config == assistant.config + + { from: assistant.config, to: updated_config } + end + + def migration_config + existing_migration_config.merge( + ORIGINAL_VALUES_KEY => existing_original_values, + 'scenario_candidates' => staged_scenario_candidates, + 'faq_document_candidates' => normalized_faq_document_candidates, + 'needs_review' => normalized_instruction_items(:needs_review) + ) + end + + def existing_migration_config + config = assistant.config[CONFIG_KEY] + config.is_a?(Hash) ? config : {} + end + + def existing_original_values + existing_migration_config[ORIGINAL_VALUES_KEY].presence || original_values + end + + def original_values + { + 'name' => assistant.name, + 'description' => assistant.description, + 'config' => original_config, + 'response_guidelines' => Array(assistant.response_guidelines), + 'guardrails' => Array(assistant.guardrails) + } + end + + def original_config + assistant.config.except(CONFIG_KEY) + end + + def conversation_messages + messages = draft_hash.fetch(:conversation_messages, {}) + messages = messages.deep_stringify_keys + + { + 'welcome_message' => messages['welcome_message'].to_s.strip, + 'handoff_message' => messages['handoff_message'].to_s.strip, + 'resolution_message' => messages['resolution_message'].to_s.strip + } + end + + def staged_scenario_candidates + scenario_candidates.map do |candidate| + candidate.transform_keys(&:to_s) + end + end + + def scenario_response_guidelines + scenario_candidates.filter_map { |candidate| candidate[:response_guideline].presence } + end + + def scenario_tool_ids(tool_ids) + Array(tool_ids).filter_map { |tool_id| tool_id.to_s.squish.presence }.uniq + end + + def scenario_candidates + Array(draft_hash[:scenario_candidates]).filter_map do |candidate| + normalized_scenario_candidate(candidate) + end + end + + def normalized_scenario_candidate(candidate) + return unless candidate.is_a?(Hash) + + candidate = candidate.deep_symbolize_keys + normalized_candidate = { + title: candidate[:title].to_s.squish, + description: candidate[:description].to_s.squish.truncate(SCENARIO_DESCRIPTION_LIMIT), + instruction: candidate[:instruction].to_s.squish, + response_guideline: candidate[:response_guideline].to_s.squish, + tool_ids: scenario_tool_ids(candidate[:tool_ids]) + } + return if normalized_candidate.values_at(:title, :description, :instruction).any?(&:blank?) + + normalized_candidate + end + + def item_values(key) + Array(draft_hash[key]).filter_map do |item| + item.to_s.squish.presence + end.uniq + end + + def normalized_instruction_items(key) + item_values(key) + end + + def normalized_faq_document_candidates + Array(draft_hash[:faq_document_candidates]).map do |candidate| + raise ArgumentError, 'FAQ document candidates must be question and answer objects' unless candidate.is_a?(Hash) + + candidate = candidate.deep_symbolize_keys + question = candidate[:question].to_s.squish + answer = candidate[:answer].to_s.squish + raise ArgumentError, 'FAQ document candidates must include a question and answer' if question.blank? || answer.blank? + + { 'question' => question, 'answer' => answer } + end.uniq + end + + def draft_hash + @draft_hash ||= draft.deep_symbolize_keys + end +end diff --git a/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb b/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb new file mode 100644 index 000000000..989989855 --- /dev/null +++ b/enterprise/app/services/captain/assistant_migration/instruction_classifier.rb @@ -0,0 +1,148 @@ +class Captain::AssistantMigration::InstructionClassifier < Captain::BaseTaskService + RESPONSE_SCHEMA = Captain::AssistantMigration::InstructionClassifierSchema + CLASSIFIER_MODEL = 'gpt-5.2'.freeze + MAX_INSTRUCTIONS_LENGTH = 20_000 + + pattr_initialize [:assistant!] + + def perform + response = make_api_call(model: CLASSIFIER_MODEL, messages: messages, schema: RESPONSE_SCHEMA) + return error_response(response) if response[:error] + + { + assistant: assistant_metadata, + draft: normalized_payload(response[:message]), + usage: response[:usage], + request_messages: response[:request_messages] + } + end + + private + + def account + assistant.account + end + + def messages + [ + { role: 'system', content: system_prompt }, + { role: 'user', content: user_prompt } + ] + end + + def system_prompt + Captain::PromptRenderer.render('instruction_classifier') + end + + def user_prompt + JSON.pretty_generate(assistant_payload) + end + + def assistant_payload # rubocop:disable Metrics/AbcSize + { + assistant_id: assistant.id, + account_id: assistant.account_id, + account_name: assistant.account.name, + name: assistant.name, + description: assistant.description, + product_name: assistant.config['product_name'], + instructions: truncated_instructions, + welcome_message: assistant.config['welcome_message'], + handoff_message: assistant.config['handoff_message'], + resolution_message: assistant.config['resolution_message'], + existing_response_guidelines: assistant.response_guidelines || [], + existing_guardrails: assistant.guardrails || [], + existing_scenarios: existing_scenarios, + available_agent_tools: available_agent_tools, + feature_settings: feature_settings + } + end + + def truncated_instructions + instructions = assistant.config['instructions'].to_s + return instructions if instructions.length <= MAX_INSTRUCTIONS_LENGTH + + "#{instructions.first(MAX_INSTRUCTIONS_LENGTH)}\n\n[TRUNCATED]" + end + + def existing_scenarios + assistant.scenarios.map do |scenario| + { + id: scenario.id, + title: scenario.title, + description: scenario.description, + instruction: scenario.instruction, + enabled: scenario.enabled + } + end + end + + def available_agent_tools + tools = assistant.respond_to?(:available_agent_tools) ? assistant.available_agent_tools : Captain::Assistant.built_in_agent_tools + tools.map { |tool| tool.slice(:id, :title, :description) } + end + + def feature_settings + assistant.config.slice( + 'feature_faq', + 'feature_memory', + 'feature_citation', + 'feature_contact_attributes', + 'temperature' + ) + end + + def normalized_payload(message) + payload = message.is_a?(Hash) ? message.deep_symbolize_keys : {} + payload.reverse_merge( + business_product_context: [], + response_guidelines: [], + guardrails: [], + scenario_candidates: [], + conversation_messages: {}, + faq_document_candidates: [], + needs_review: [], + classification_notes: [] + ) + end + + def assistant_metadata # rubocop:disable Metrics/AbcSize + { + id: assistant.id, + name: assistant.name, + account_id: assistant.account_id, + account_name: assistant.account.name, + inbox_count: assistant.captain_inboxes.size, + instruction_length: assistant.config['instructions'].to_s.length, + original_instructions: assistant.config['instructions'].to_s, + welcome_message: assistant.config['welcome_message'].to_s, + handoff_message: assistant.config['handoff_message'].to_s, + resolution_message: assistant.config['resolution_message'].to_s + } + end + + def error_response(response) + { + assistant: assistant_metadata, + error: response[:error], + error_code: response[:error_code], + request_messages: response[:request_messages] + } + end + + def event_name + 'assistant_migration_instruction_classifier' + end + + def captain_tasks_enabled? + true + end + + def counts_toward_usage? + false + end + + def build_follow_up_context? + false + end +end diff --git a/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb b/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb new file mode 100644 index 000000000..3e42bb49d --- /dev/null +++ b/enterprise/app/services/captain/assistant_migration/instruction_classifier_schema.rb @@ -0,0 +1,91 @@ +class Captain::AssistantMigration::InstructionClassifierSchema < RubyLLM::Schema + DESCRIPTION_LENGTH_LIMIT = 500 + + def self.instruction_items(field_name, description:, max_items: 20) + array field_name, + description: "#{description} Return plain standalone sentences without numbering, bullets, or section labels.", + max_items: max_items, + of: :string + end + + array :business_product_context, + description: "Single compact root assistant description for the root orchestrator prompt, maximum #{DESCRIPTION_LENGTH_LIMIT} characters: " \ + 'preserve the existing assistant description and enrich it only with relevant business/product context from the ' \ + 'custom instructions. Include assistant identity, product scope, high-level mission, and high-level source/routing ' \ + 'priorities only. Do not include workflows, procedures, attribute glossaries, policy details, or long inventories. ' \ + 'Return complete plain prose without numbering, bullets, section labels, or a truncated final sentence.', + min_items: 1, + max_items: 1 do + string max_length: DESCRIPTION_LENGTH_LIMIT + end + + instruction_items :response_guidelines, + description: 'Tone, language, answer length, formatting, and clarification behavior.', + max_items: 20 + + instruction_items :guardrails, + description: 'Refusal rules, escalation boundaries, source boundaries, safety limits, and things the assistant must not do.', + max_items: 20 + + array :scenario_candidates, + description: 'Review-stage specialized-agent candidates. These are also temporarily flattened into response guidelines.', + max_items: 15 do + object do + string :title, + description: 'Short scenario agent title for a distinct user-intent workflow.', + max_length: 80 + string :description, + description: 'When this specialized scenario should be used. This is shown to the orchestrator for routing.', + max_length: 500 + string :instruction, + description: 'How the specialized agent should handle the workflow. Include only evidence-backed markdown tool links. ' \ + 'Do not include confidence labels or review notes.', + max_length: 2000 + string :response_guideline, + description: 'Same-language, customer-visible response guideline that preserves this scenario behavior when flattened. ' \ + 'Do not include tool syntax, tool names, labels, private-note instructions, or internal implementation details.', + max_length: 1000 + array :tool_ids, + description: 'Available tool IDs explicitly referenced in instruction using markdown links. Empty when no tools are required.', + max_items: 10, + of: :string + end + end + + object :conversation_messages, description: 'Exact globally reusable customer-facing message copy found in instructions. ' \ + 'Leave empty for conditional, placeholder, or workflow-specific copy.' do + string :welcome_message, description: 'Exact globally reusable initial greeting copy from instructions, or empty string. ' \ + 'Do not convert an instruction about greeting into message copy.', + max_length: 1000 + string :handoff_message, + description: 'Exact globally reusable human-handoff message copy from instructions, or empty string. ' \ + 'Do not use scenario-specific, team-specific, placeholder, or conditional handoff copy.', + max_length: 1000 + string :resolution_message, + description: 'Exact globally reusable resolution/closing message copy from instructions, or empty string. ' \ + 'Do not use conditional or placeholder closing copy.', + max_length: 1000 + end + + array :faq_document_candidates, + description: 'Pending FAQ candidates for factual or product-specific knowledge such as pricing, policy, setup, troubleshooting, ' \ + 'or operational details. These candidates remain inactive until reviewed and approved.', + max_items: 25 do + object do + string :question, + description: 'Natural, standalone customer question about factual product or business knowledge.', + max_length: 255 + string :answer, + description: 'Self-contained factual answer using only the existing instructions. Do not include assistant behavior, ' \ + 'tool use, or message copy. Preserve exact values, conditions, and exceptions.', + max_length: 2000 + end + end + + instruction_items :needs_review, + description: 'Unclear, conflicting, risky, duplicated, or uncertain content that needs human review. ' \ + 'Include the reason in the item text.', + max_items: 20 + + array :classification_notes, description: 'Short notes about important migration decisions or risks.', max_items: 10, of: :string +end diff --git a/enterprise/lib/captain/prompts/instruction_classifier.liquid b/enterprise/lib/captain/prompts/instruction_classifier.liquid new file mode 100644 index 000000000..abc58ff60 --- /dev/null +++ b/enterprise/lib/captain/prompts/instruction_classifier.liquid @@ -0,0 +1,137 @@ +You are migrating Captain assistant instructions into a structured configuration. + +Classify the existing assistant instructions into these sections: +1. Business/Product Context +2. Response Guidelines +3. Guardrails +4. Scenario Candidates +5. Conversation Messages +6. FAQs/Documents Candidates +7. Needs Review + +## General Rules + +- Preserve behavior as closely as possible. +- Do not duplicate the same content across sections. +- Return clean migrated values only. Do not include source excerpts, source labels, citations, or "Source:" text in any migrated field. +- Do not rewrite customer-facing message copy unless necessary to classify an exact copy from instructions. +- Do not include confidence labels, review labels, bracketed reviewer comments, or schema labels inside migrated values. +- For Business/Product Context, Response Guidelines, and Guardrails, return each item as a plain standalone sentence. + Do not prefix items with numbers, bullets, section labels, or list markers such as "1.", "-", or "*". +- When several instructions share the same trigger, condition, or subject, combine them into one concise item instead + of repeating the same trigger across multiple items. Preserve every required action, prohibition, and routing + outcome from the source instruction when combining. +- If unsure, place content in Needs Review and include the reason in that item. +- Return data that matches the provided schema. + +## Business/Product Context + +- Business/Product Context maps to the root assistant description and is injected into the root orchestrator prompt. +- Return exactly one Business/Product Context item. +- Start with the existing assistant description and preserve its meaning. +- Enrich it only with relevant business or product context found in the custom instructions. +- Produce one coherent description rather than appending a second context block or repeating the existing description. +- Keep it at most 500 characters because that is the assistant description limit in the UI and model. +- Prefer roughly 300-450 characters when the source needs detail, leaving room below the hard limit. +- Finish the description cleanly. Never end mid-word, mid-clause, after an opening bracket, or with a dangling separator. +- Make it a compact summary of assistant identity, product scope, high-level mission, and high-level source or routing priorities. +- Do not include detailed workflows, step-by-step procedures, long support-scope inventories, attribute glossaries, + policy details, scenario-specific handling, tool instructions, or customer-facing message copy. + +## Conversation Messages + +- Existing welcome_message, handoff_message, and resolution_message config values are provided separately. +- Treat welcome_message, handoff_message, and resolution_message as conversation message config fields. +- Extract exact welcome, handoff, or resolution message copy from instructions into conversation_messages when present. +- Only classify handoff copy as conversation_messages.handoff_message when it is generic enough to reuse for any human handoff. +- If handoff copy is scenario-specific, keep it inside that scenario instruction; if it is only a rule about when or how to hand off, classify it as a Response Guideline or Guardrail. +- Do not extract a conversation message from an instruction about what to say, from a placeholder template, + from conditional copy, from role/team-specific copy, or from text that only applies inside one workflow. +- If a message contains placeholders such as a blank name, team name, bracketed variable, business-hours state, + or dynamic runtime condition, do not place it in conversation_messages. Keep it in the relevant workflow or + Needs Review. +- Do not copy message values from existing config into conversation_messages. +- Do not decide whether existing config values should be overwritten. Migration code handles applying extracted + conversation_messages only when the corresponding config value is blank. + +## Scenario Candidates + +- In the current architecture, a scenario becomes a specialized sub-agent with its own title, description, + instructions, and optional tools. +- During this migration, scenario candidates are also temporarily flattened into response guidelines so existing + assistant behavior is preserved before scenario records are created. +- For every scenario candidate, write a response_guideline that is the flattened version of that scenario for + the root assistant's response guidelines. +- The response_guideline must be in the same language as the original scenario or source instruction. +- The response_guideline must preserve the intended customer-visible behavior, trigger, information to collect, + and routing/escalation outcome. +- The response_guideline must not include tool syntax, tool:// links, markdown tool links, tool names, label + updates, priority updates, private-note instructions, custom-tool instructions, or internal implementation details. +- If the scenario uses internal tools such as labels, priorities, private notes, or custom tools, describe only + the customer-visible behavior and expected routing/escalation outcome in response_guideline. +- If human handoff is needed, describe it in natural language such as route/escalate/transfer to a human; do not + mention the handoff tool in response_guideline. +- Keep scenario titles, descriptions, instructions, and response_guidelines clear, self-contained, and reviewable. +- Only create scenario candidates for distinct user-intent workflows that should be routed to a specialized agent. + A candidate must be narrow enough to become a named specialist assistant with domain-specific handling instructions. +- Good scenario candidates include multi-step intake workflows, qualification flows, specialized troubleshooting + workflows, booking flows, lead-capture flows, recommendation flows, fulfillment workflows, or tool-use procedures + for a specific user intent. +- A scenario candidate should answer "yes" to this test: would a named specialist sub-agent improve handling + beyond the base assistant's global FAQ, guardrail, response-guideline, and human-handoff behavior? +- Do not create scenario candidates for global escalation rules, generic handoff policy, missing-information + behavior, source-boundary rules, refusal rules, tone, formatting, answer length, or one-step fallback behavior. +- Do not create scenario candidates whose main purpose is to escalate or hand off. "Identify the trigger, avoid + guessing, tell the user support will review, and hand off" is a guardrail/handoff boundary, not a scenario, + even though it contains multiple statements. +- Do create scenario candidates when the instructions define a concrete intake, qualification, troubleshooting, + booking, lead-capture, recommendation, or fulfillment workflow, even when the workflow eventually hands off + to a human. +- Do not create scenario candidates for simple routing triggers such as "user asks for a human", "immediately + hand off this category", or "route sales questions to the sales team" when there is no concrete workflow to run. +- Handoff behavior is a scenario candidate only when part of a larger intake, qualification, or specialized handling workflow. +- Global rules like "if not in docs, escalate", "ask one clarifying question", "do not answer account-specific + questions", or "tell the user support will review" belong in Guardrails or Response Guidelines, not Scenario Candidates. +- Broad buckets like "account-specific issue escalation", "unknown question escalation", "contact support", + "fallback to human", or "documentation unavailable" are not scenario candidates. + +## Tool Use + +- If a scenario candidate requires tools, reference the available tool explicitly inside the scenario instruction + using markdown tool links such as [Handoff to Human](tool://handoff). +- Use only tool IDs listed in available_agent_tools. If a needed tool is unavailable or the workflow depends on + unavailable runtime data such as FAQ relevance scores or business-hours status, place it in Needs Review instead. +- Do not map an unavailable named tool to a different available tool. For example, do not treat FAQ Lookup as + Product Search, Order Status, website browsing, pricing lookup, agent availability, business-hours detection, + ticket creation, or custom-attribute assignment unless the instructions explicitly say that the available + tool provides that behavior. +- If a workflow cannot run without an unavailable tool or runtime signal, do not create a tool-backed scenario + for it. Preserve the instruction in Needs Review with the missing capability named. + +## FAQs/Documents Candidates + +- Convert factual or product-specific knowledge into pending FAQ candidates with a natural customer question and a self-contained answer. +- FAQ candidates are review-stage data only. They are not active assistant knowledge until a human reviews and approves them. +- Use only facts stated in the existing instructions. Do not invent, generalize, update, or fill in missing details. +- Preserve exact prices, limits, dates, time zones, conditions, exceptions, product names, and operational details in the answer. +- Write each question as a standalone question a customer might naturally ask. Make it specific enough to retrieve the corresponding answer. +- Write each answer so it fully answers its question without relying on another FAQ candidate or surrounding context. +- Split unrelated facts into separate candidates. Keep related conditions and exceptions together when separating them would make an answer incomplete. +- Do not create FAQ candidates about what the assistant should say or do, how it should use sources or tools, when it should route or escalate, + or which exact message it should send. Classify those as Response Guidelines, Guardrails, Scenario Candidates, Conversation Messages, + or Needs Review as appropriate. +- FAQ questions must ask about the product or business, not about the assistant. Do not write questions such as "What should the assistant answer?", + "What should I say?", "Which source should the assistant use?", or "Which tool should be called?". +- FAQ answers must contain customer-facing knowledge, not instructions to call tools, inspect internal data, update records, transfer conversations, + or follow internal workflows. +- When factual sources conflict and the instructions do not explicitly establish which fact overrides the others, put the conflict in Needs Review + instead of creating an FAQ candidate. Use an explicitly stated override or superseding fact when one is present. +- Only factual or product-specific knowledge should become FAQs/Documents candidates. +- Generic capability statements such as "answer product questions", "help with billing", + "troubleshoot common issues", or "direct to documentation" are not FAQ/document candidates. + Put them in Business/Product Context or Response Guidelines when useful. +- Product facts, pricing, policies, setup steps, troubleshooting facts, support hours, emergency contacts, + and operational details should become pending FAQ candidates, not Response Guidelines or trusted approved knowledge. +- Do not create FAQ/document candidates for topic labels or unsupported capabilities when the factual content is + missing. Put "pricing details are needed", "same-day delivery schedule details are needed", or similar gaps in + Needs Review instead. diff --git a/lib/tasks/captain_assistant_migration.rake b/lib/tasks/captain_assistant_migration.rake new file mode 100644 index 000000000..7cb8c2e1c --- /dev/null +++ b/lib/tasks/captain_assistant_migration.rake @@ -0,0 +1,278 @@ +require 'json' +require 'fileutils' +require 'csv' + +# rubocop:disable Metrics/BlockLength +namespace :captain do + namespace :assistant_migration do + desc 'Generate structured migration drafts. Usage: rake captain:assistant_migration:generate IDS=1,2,3 LIMIT=50 ' \ + 'OUTPUT=tmp/captain_migration.jsonl' + task generate: :environment do + assistants = CaptainAssistantMigrationTask.assistants + output_path = ENV.fetch('OUTPUT', Rails.root.join('tmp/captain_assistant_migration_drafts.jsonl').to_s) + + FileUtils.mkdir_p(File.dirname(output_path)) + processed = 0 + + File.open(output_path, 'w') do |file| + CaptainAssistantMigrationTask.each_assistant(assistants) do |assistant| + result = Captain::AssistantMigration::InstructionClassifier.new(assistant: assistant).perform + file.puts(JSON.generate(result)) + processed += 1 + puts "Generated migration draft for assistant #{assistant.id} (#{processed}/#{CaptainAssistantMigrationTask.assistant_count(assistants)})" + end + end + + puts "Wrote #{processed} migration drafts to #{output_path}" + end + + desc 'Apply reviewed migration drafts. Usage: rake captain:assistant_migration:apply INPUT=tmp/reviewed.jsonl DRY_RUN=true' + task apply: :environment do + input_path = ENV.fetch('INPUT') + dry_run = CaptainAssistantMigrationTask.truthy?('DRY_RUN', default: true) + + results = CaptainAssistantMigrationTask.apply_drafts( + input_path: input_path, + dry_run: dry_run + ) + + results.each { |result| puts(JSON.generate(result)) } + puts "Processed #{results.size} migration drafts from #{input_path}" + puts 'Dry run only. Re-run with DRY_RUN=false to write changes.' if dry_run + end + + desc 'Restore conversation message config from migration backup. Usage: rake captain:assistant_migration:restore_messages IDS=1,2 DRY_RUN=true' + task restore_messages: :environment do + dry_run = CaptainAssistantMigrationTask.truthy?('DRY_RUN', default: true) + results = CaptainAssistantMigrationTask.restore_conversation_messages(dry_run: dry_run) + + results.each { |result| puts(JSON.generate(result)) } + puts "Processed #{results.size} assistant message restores" + puts 'Dry run only. Re-run with DRY_RUN=false to restore conversation messages.' if dry_run + end + end +end +# rubocop:enable Metrics/BlockLength + +# rubocop:disable Style/OneClassPerFile +class CaptainAssistantMigrationTask + CsvAccount = Struct.new(:id, :name, keyword_init: true) do + def captain_models + {} + end + + def conversations + CsvRelation.new + end + end + + CsvAssociation = Struct.new(:inbox_count, keyword_init: true) do + def size + inbox_count + end + end + + class CsvRelation + def find_by(*) + nil + end + + def exists? + false + end + end + + CsvAssistant = Struct.new( + :id, + :name, + :account_id, + :account, + :description, + :config, + :response_guidelines, + :guardrails, + :captain_inboxes, + :scenarios, + keyword_init: true + ) + + class << self + def assistants + return csv_assistants if ENV['CSV_INPUT'].present? + + scope = Captain::Assistant.includes(:account, :captain_inboxes, :scenarios) + + ids = ENV.fetch('IDS', '').split(',').filter_map { |id| id.strip.presence } + scope = scope.where(id: ids) if ids.any? + + scope = migration_eligible_scope(scope).order(:id) + + limit = ENV.fetch('LIMIT', 50).to_i + limit.positive? ? scope.limit(limit) : scope + end + + def each_assistant(assistants, &) + return assistants.find_each(&) if assistants.respond_to?(:find_each) + + assistants.each(&) + end + + def assistant_count(assistants) + assistants.respond_to?(:size) ? assistants.size : assistants.count + end + + def restore_conversation_messages(dry_run:) + ENV.fetch('IDS').split(',').filter_map { |id| id.strip.presence }.map do |assistant_id| + assistant = Captain::Assistant.find(assistant_id) + restore_conversation_messages_for(assistant, dry_run: dry_run) + rescue ActiveRecord::RecordNotFound + { assistant_id: assistant_id, error: 'Assistant not found' } + end + end + + def apply_drafts(input_path:, dry_run:) + File.readlines(input_path, chomp: true).filter_map.with_index(1) do |line, line_number| + next if line.blank? + + apply_draft(JSON.parse(line), line_number: line_number, dry_run: dry_run) + rescue JSON::ParserError => e + { line_number: line_number, error: "Invalid JSON: #{e.message}" } + end + end + + def apply_draft(payload, line_number:, dry_run:) + return { line_number: line_number, skipped: true, reason: payload['error'] } if payload['error'].present? + + assistant_id = payload.dig('assistant', 'id') || payload['assistant_id'] + assistant = Captain::Assistant.find(assistant_id) + return skipped_result(line_number, assistant_id, 'Assistant is not a V1 migration candidate') unless migration_candidate?(assistant) + + draft = payload['draft'] || payload + + Captain::AssistantMigration::DraftApplier.new( + assistant: assistant, + draft: draft, + dry_run: dry_run + ).perform.merge(line_number: line_number) + rescue ActiveRecord::RecordNotFound + { line_number: line_number, assistant_id: assistant_id, error: 'Assistant not found' } + end + + def truthy?(key, default:) + value = ENV.fetch(key, nil) + return default if value.nil? + + value.to_s.downcase.in?(%w[1 true yes y]) + end + + private + + def restore_conversation_messages_for(assistant, dry_run:) + original_config = assistant.config.dig( + Captain::AssistantMigration::DraftApplier::CONFIG_KEY, + Captain::AssistantMigration::DraftApplier::ORIGINAL_VALUES_KEY, + 'config' + ) + return skipped_result(nil, assistant.id, 'No stored migration original config found') if original_config.nil? + + config, changes = restored_message_config(assistant.config.deep_dup, original_config) + assistant.update!(config: config) if !dry_run && changes.present? + + { assistant_id: assistant.id, dry_run: dry_run, changes: changes } + end + + def restored_message_config(config, original_config) + changes = {} + %w[welcome_message handoff_message resolution_message].each do |key| + original_present = original_config.key?(key) + next if config[key] == original_config[key] && config.key?(key) == original_present + + changes[key] = { from: config[key], to: original_config[key] } + original_present ? config[key] = original_config[key] : config.delete(key) + end + [config, changes] + end + + def skipped_result(line_number, assistant_id, reason) + { + line_number: line_number, + assistant_id: assistant_id, + skipped: true, + reason: reason + } + end + + def migration_eligible_scope(scope) + scope.left_outer_joins(:scenarios) + .joins(:captain_inboxes) + .where("NULLIF(captain_assistants.config->>'instructions', '') IS NOT NULL") + .where("captain_assistants.response_guidelines IS NULL OR captain_assistants.response_guidelines = '[]'::jsonb") + .where("captain_assistants.guardrails IS NULL OR captain_assistants.guardrails = '[]'::jsonb") + .where(captain_scenarios: { id: nil }) + .distinct + end + + def migration_candidate?(assistant) + assistant.config['instructions'].present? && + assistant.captain_inboxes.size.positive? && + Array(assistant.response_guidelines).blank? && + Array(assistant.guardrails).blank? && + !scenarios_exist?(assistant) + end + + def scenarios_exist?(assistant) + scenarios = assistant.scenarios + return scenarios.exists? if scenarios.respond_to?(:exists?) + + scenarios.present? + end + + def csv_assistants # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + rows = CSV.read(ENV.fetch('CSV_INPUT'), headers: true) + ids = ENV.fetch('IDS', '').split(',').filter_map { |id| id.strip.presence } + status = ENV.fetch('STATUS', '').presence + + assistants = rows.filter_map do |row| + next if ids.any? && ids.exclude?(row['id'].to_s) + next if status.present? && row['status'].to_s != status + + assistant = csv_assistant(row) + next unless migration_candidate?(assistant) + + assistant + end + + limit = ENV.fetch('LIMIT', 50).to_i + limit.positive? ? assistants.first(limit) : assistants + end + + def csv_assistant(row) + config = parse_json(row['config'], fallback: {}) + CsvAssistant.new( + id: normalize_integer(row['id']), + name: row['name'].to_s, + account_id: normalize_integer(row['account_id']), + account: CsvAccount.new(id: normalize_integer(row['account_id']), name: row['account_name'].to_s), + description: row['description'].to_s, + config: config, + response_guidelines: parse_json(row['response_guidelines'], fallback: []), + guardrails: parse_json(row['guardrails'], fallback: []), + captain_inboxes: CsvAssociation.new(inbox_count: normalize_integer(row['inbox_count'])), + scenarios: [] + ) + end + + def parse_json(value, fallback:) + return fallback if value.blank? + + JSON.parse(value) + rescue JSON::ParserError + fallback + end + + def normalize_integer(value) + value.to_s.delete(',').to_i + end + end +end +# rubocop:enable Style/OneClassPerFile diff --git a/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb b/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb new file mode 100644 index 000000000..0e2f420ac --- /dev/null +++ b/spec/enterprise/services/captain/assistant_migration/draft_applier_spec.rb @@ -0,0 +1,116 @@ +require 'rails_helper' + +RSpec.describe Captain::AssistantMigration::DraftApplier do + let(:account) { create(:account) } + let(:assistant) do + create( + :captain_assistant, + account: account, + config: { 'product_name' => 'Test Product', 'instructions' => 'Legacy V1 custom instructions.' }, + response_guidelines: [], + guardrails: [] + ) + end + let(:scenario_candidate) do + { + 'title' => 'Billing Investigation', + 'description' => 'Use when a customer reports an account-specific billing issue.', + 'instruction' => 'Collect the invoice number and summarize the issue before escalating.', + 'response_guideline' => 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.', + 'tool_ids' => [] + } + end + let(:faq_document_candidate) do + { + 'question' => 'When is support available?', + 'answer' => 'Support is available Monday to Friday.' + } + end + let(:draft) do + { + business_product_context: ['Support assistant for Test Product.'], + response_guidelines: ['Be concise.'], + guardrails: ['Do not guess.'], + conversation_messages: {}, + scenario_candidates: [scenario_candidate], + faq_document_candidates: [faq_document_candidate], + needs_review: ['Pricing details are missing because factual details are absent from the source instructions.'] + } + end + + describe '#perform' do + it 'reports staged scenario candidates in dry run without writing to the assistant' do + result = described_class.new(assistant: assistant, draft: draft, dry_run: true).perform + + expect(result.dig(:changes, :config, :to, 'assistant_migration', 'scenario_candidates')).to eq([scenario_candidate]) + expect(result.dig(:changes, :response_guidelines, :to)).to include( + 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.' + ) + expect(assistant.reload.config).not_to have_key('assistant_migration') + expect(assistant.scenarios.count).to eq(0) + end + + it 'stores scenario candidates in assistant config and flattens them into response guidelines' do + described_class.new(assistant: assistant, draft: draft, dry_run: false).perform + + assistant.reload + expect(assistant.config.dig('assistant_migration', 'scenario_candidates')).to eq([scenario_candidate]) + expect(assistant.config.dig('assistant_migration', 'faq_document_candidates')).to contain_exactly(faq_document_candidate) + expect(assistant.config.dig('assistant_migration', 'needs_review')).to contain_exactly( + 'Pricing details are missing because factual details are absent from the source instructions.' + ) + expect(assistant.response_guidelines).to include( + 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.' + ) + expect(assistant.response_guidelines).not_to include(faq_document_candidate['answer']) + expect(assistant.scenarios.count).to eq(0) + end + + it 'rejects stale drafts whose FAQ candidates use the old string format' do + stale_draft = draft.merge(faq_document_candidates: ['Support is available Monday to Friday.']) + + expect do + described_class.new(assistant: assistant, draft: stale_draft, dry_run: false).perform + end.to raise_error(ArgumentError, 'FAQ document candidates must be question and answer objects') + + expect(assistant.reload.config).not_to have_key('assistant_migration') + end + + it 'preserves original values in migration config before applying classifier output' do + assistant.update!( + description: 'Existing assistant description.', + response_guidelines: ['Use plain language.'], + guardrails: ['Do not disclose internal notes.'] + ) + + described_class.new(assistant: assistant, draft: draft, dry_run: false).perform + + assistant.reload + expect(assistant.description).to eq('Support assistant for Test Product.') + expect(assistant.response_guidelines).to include('Be concise.') + expect(assistant.guardrails).to eq(['Do not guess.']) + expect(assistant.config.dig('assistant_migration', 'original_values')).to include( + 'name' => assistant.name, + 'description' => 'Existing assistant description.', + 'config' => { 'product_name' => 'Test Product', 'instructions' => 'Legacy V1 custom instructions.' }, + 'response_guidelines' => ['Use plain language.'], + 'guardrails' => ['Do not disclose internal notes.'] + ) + end + + it 'rejects an oversized assistant description from a stale draft' do + long_context = 'This assistant supports a very broad product surface with many long details. ' * 10 + original_description = assistant.description + + expect do + described_class.new( + assistant: assistant, + draft: draft.merge(business_product_context: [long_context]), + dry_run: false + ).perform + end.to raise_error(ArgumentError, 'Assistant description exceeds 500 characters') + + expect(assistant.reload.description).to eq(original_description) + end + end +end From 11c65f3b9a86b18e2ae599abe0028034054fc12a Mon Sep 17 00:00:00 2001 From: Sony Mathew Date: Mon, 13 Jul 2026 15:25:07 +0530 Subject: [PATCH 2/2] feat: Intercom import workflow (#14922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Adds an admin-only Intercom import workflow under Settings > Data. Admins can connect an Intercom access token, start named historical contact/conversation imports, monitor active and previous import runs, review paginated skip/error logs, download skip logs, and route imported conversations into source-bucket API inboxes that can be renamed later. The import path stores durable source mappings, batches Intercom contact/conversation pages through Sidekiq, records already-imported records as skipped, and writes historical messages without normal outbound delivery callbacks. The PR also includes the Intercom import PRD/TDD document for review context. Closes [CW-7519](https://linear.app/chatwoot/issue/CW-7519/explore-intercom-import) ## Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality not to work as expected) - [x] This change requires a documentation update ## How Has This Been Tested? Tested importing using actual data through integration. Screenshots: Screenshot 2026-07-02 at 10 48 48 PM Screenshot 2026-07-02 at 10 49 03 PM Screenshot 2026-07-02 at 10 49 21 PM Screenshot 2026-07-02 at 10 49 38 PM Passed locally: ```sh eval "$(rbenv init -)" && bundle exec rspec spec/models/data_import_spec.rb spec/jobs/data_import_job_spec.rb spec/requests/api/v1/accounts/data_imports_spec.rb spec/requests/api/v1/accounts/integrations/intercom_spec.rb spec/jobs/data_imports/intercom/import_jobs_spec.rb spec/services/data_imports/intercom/importer_spec.rb spec/services/data_imports/intercom/placeholder_inbox_builder_spec.rb spec/services/data_imports/intercom/source_bucket_spec.rb ``` ```sh eval "$(rbenv init -)" && bundle exec rubocop app/controllers/api/v1/accounts/data_imports_controller.rb app/controllers/api/v1/accounts/integrations/intercom_controller.rb app/jobs/data_imports/intercom app/models/data_import.rb app/models/data_import_error.rb app/models/data_import_item.rb app/models/data_import_mapping.rb app/models/integrations/hook.rb app/policies/data_import_policy.rb app/policies/hook_policy.rb app/services/data_imports/intercom db/migrate/20260702000000_expand_data_imports_for_intercom_imports.rb db/migrate/20260702000001_create_data_import_items.rb db/migrate/20260702000002_create_data_import_mappings.rb db/migrate/20260702000003_create_data_import_errors.rb spec/jobs/data_imports/intercom spec/requests/api/v1/accounts/data_imports_spec.rb spec/requests/api/v1/accounts/integrations/intercom_spec.rb spec/services/data_imports/intercom ``` ```sh pnpm exec eslint app/javascript/dashboard/api/dataImports.js app/javascript/dashboard/api/integrations.js app/javascript/dashboard/routes/dashboard/settings/data/Index.vue app/javascript/dashboard/routes/dashboard/settings/data/Show.vue app/javascript/dashboard/routes/dashboard/settings/data/data.routes.js app/javascript/dashboard/routes/dashboard/settings/data/importStatus.js app/javascript/dashboard/routes/dashboard/settings/integrations/Intercom.vue app/javascript/dashboard/routes/dashboard/settings/integrations/integrations.routes.js app/javascript/dashboard/routes/dashboard/settings/settings.routes.js app/javascript/dashboard/components-next/sidebar/Sidebar.vue app/javascript/dashboard/routes/dashboard/settings/inbox/Index.vue ``` ```sh git diff --check ``` Note: the RSpec boot logs the existing local `chatwoot_dev` purge warning because other database sessions are open, then continues and completes with 52 examples, 0 failures. ## Checklist: - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code - [ ] I have commented on my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published in downstream modules --------- Co-authored-by: Shivam Mishra Co-authored-by: Sivin Varghese <64252451+iamsivin@users.noreply.github.com> Co-authored-by: iamsivin --- .../v1/accounts/data_imports_controller.rb | 159 +++ app/finders/data_import_error_finder.rb | 11 + app/finders/data_import_skip_log_finder.rb | 35 + app/javascript/dashboard/api/dataImports.js | 39 + .../components-next/sidebar/Sidebar.vue | 17 + app/javascript/dashboard/components/Modal.vue | 4 +- app/javascript/dashboard/featureFlags.js | 1 + .../dashboard/i18n/locale/en/settings.json | 99 ++ .../components/BaseSettingsHeader.vue | 8 +- .../routes/dashboard/settings/data/Index.vue | 378 +++++++ .../settings/data/NewImportDialog.vue | 210 ++++ .../routes/dashboard/settings/data/Show.vue | 238 +++++ .../data/components/ImportDetailHeader.vue | 128 +++ .../data/components/ImportErrorsSection.vue | 78 ++ .../data/components/ImportLogSection.vue | 105 ++ .../data/components/ImportProgress.vue | 100 ++ .../data/components/ImportSkipLogsSection.vue | 127 +++ .../data/components/ImportSummaryTiles.vue | 110 ++ .../dashboard/settings/data/data.routes.js | 34 + .../dashboard/settings/data/importSources.js | 17 + .../dashboard/settings/data/importStatus.js | 84 ++ .../settings/data/specs/importStatus.spec.js | 92 ++ .../data/specs/pollingLifecycle.spec.js | 121 +++ .../routes/dashboard/settings/inbox/Index.vue | 6 +- .../dashboard/settings/settings.routes.js | 2 + app/jobs/data_imports/intercom/base_job.rb | 43 + .../intercom/contacts_page_job.rb | 26 + .../intercom/conversations_page_job.rb | 16 + app/jobs/data_imports/intercom/import_job.rb | 24 + app/models/data_import.rb | 88 +- app/models/data_import_error.rb | 33 + app/models/data_import_item.rb | 35 + app/models/data_import_mapping.rb | 33 + app/policies/data_import_policy.rb | 41 + .../intercom/activity_content_builder.rb | 84 ++ app/services/data_imports/intercom/client.rb | 107 ++ .../data_imports/intercom/creation_service.rb | 67 ++ .../intercom/credentials_validator.rb | 36 + .../data_imports/intercom/importer.rb | 991 ++++++++++++++++++ .../intercom/placeholder_inbox_builder.rb | 41 + .../data_imports/intercom/restart_service.rb | 55 + .../data_imports/intercom/source_bucket.rb | 23 + .../data_imports/_data_import.json.jbuilder | 24 + .../accounts/data_imports/index.json.jbuilder | 5 + .../accounts/data_imports/show.json.jbuilder | 31 + config/features.yml | 4 + config/locales/en.yml | 22 + config/routes.rb | 11 + ...xpand_data_imports_for_intercom_imports.rb | 31 + ...20260702000001_create_data_import_items.rb | 35 + ...60702000002_create_data_import_mappings.rb | 22 + ...0260702000003_create_data_import_errors.rb | 17 + db/schema.rb | 66 ++ .../images/integrations/intercom.png | Bin 0 -> 3016 bytes spec/factories/data_import.rb | 9 + spec/finders/data_import_error_finder_spec.rb | 26 + .../data_import_skip_log_finder_spec.rb | 38 + .../data_imports/intercom/import_jobs_spec.rb | 218 ++++ spec/models/account_spec.rb | 8 +- spec/models/data_import_spec.rb | 44 + .../api/v1/accounts/data_imports_spec.rb | 438 ++++++++ .../intercom/activity_content_builder_spec.rb | 51 + .../data_imports/intercom/client_spec.rb | 16 + .../intercom/creation_service_spec.rb | 52 + .../intercom/credentials_validator_spec.rb | 54 + .../data_imports/intercom/importer_spec.rb | 968 +++++++++++++++++ .../placeholder_inbox_builder_spec.rb | 33 + .../intercom/restart_service_spec.rb | 64 ++ .../intercom/source_bucket_spec.rb | 17 + 69 files changed, 6239 insertions(+), 11 deletions(-) create mode 100644 app/controllers/api/v1/accounts/data_imports_controller.rb create mode 100644 app/finders/data_import_error_finder.rb create mode 100644 app/finders/data_import_skip_log_finder.rb create mode 100644 app/javascript/dashboard/api/dataImports.js create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/Index.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/NewImportDialog.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/Show.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/components/ImportDetailHeader.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/components/ImportErrorsSection.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/components/ImportLogSection.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/components/ImportProgress.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSkipLogsSection.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSummaryTiles.vue create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/data.routes.js create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/importSources.js create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/importStatus.js create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/specs/importStatus.spec.js create mode 100644 app/javascript/dashboard/routes/dashboard/settings/data/specs/pollingLifecycle.spec.js create mode 100644 app/jobs/data_imports/intercom/base_job.rb create mode 100644 app/jobs/data_imports/intercom/contacts_page_job.rb create mode 100644 app/jobs/data_imports/intercom/conversations_page_job.rb create mode 100644 app/jobs/data_imports/intercom/import_job.rb create mode 100644 app/models/data_import_error.rb create mode 100644 app/models/data_import_item.rb create mode 100644 app/models/data_import_mapping.rb create mode 100644 app/policies/data_import_policy.rb create mode 100644 app/services/data_imports/intercom/activity_content_builder.rb create mode 100644 app/services/data_imports/intercom/client.rb create mode 100644 app/services/data_imports/intercom/creation_service.rb create mode 100644 app/services/data_imports/intercom/credentials_validator.rb create mode 100644 app/services/data_imports/intercom/importer.rb create mode 100644 app/services/data_imports/intercom/placeholder_inbox_builder.rb create mode 100644 app/services/data_imports/intercom/restart_service.rb create mode 100644 app/services/data_imports/intercom/source_bucket.rb create mode 100644 app/views/api/v1/accounts/data_imports/_data_import.json.jbuilder create mode 100644 app/views/api/v1/accounts/data_imports/index.json.jbuilder create mode 100644 app/views/api/v1/accounts/data_imports/show.json.jbuilder create mode 100644 db/migrate/20260702000000_expand_data_imports_for_intercom_imports.rb create mode 100644 db/migrate/20260702000001_create_data_import_items.rb create mode 100644 db/migrate/20260702000002_create_data_import_mappings.rb create mode 100644 db/migrate/20260702000003_create_data_import_errors.rb create mode 100644 public/dashboard/images/integrations/intercom.png create mode 100644 spec/finders/data_import_error_finder_spec.rb create mode 100644 spec/finders/data_import_skip_log_finder_spec.rb create mode 100644 spec/jobs/data_imports/intercom/import_jobs_spec.rb create mode 100644 spec/requests/api/v1/accounts/data_imports_spec.rb create mode 100644 spec/services/data_imports/intercom/activity_content_builder_spec.rb create mode 100644 spec/services/data_imports/intercom/client_spec.rb create mode 100644 spec/services/data_imports/intercom/creation_service_spec.rb create mode 100644 spec/services/data_imports/intercom/credentials_validator_spec.rb create mode 100644 spec/services/data_imports/intercom/importer_spec.rb create mode 100644 spec/services/data_imports/intercom/placeholder_inbox_builder_spec.rb create mode 100644 spec/services/data_imports/intercom/restart_service_spec.rb create mode 100644 spec/services/data_imports/intercom/source_bucket_spec.rb diff --git a/app/controllers/api/v1/accounts/data_imports_controller.rb b/app/controllers/api/v1/accounts/data_imports_controller.rb new file mode 100644 index 000000000..7f0d28e81 --- /dev/null +++ b/app/controllers/api/v1/accounts/data_imports_controller.rb @@ -0,0 +1,159 @@ +require 'csv' + +class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseController + DATA_IMPORT_FEATURE = 'data_import'.freeze + + before_action :ensure_data_import_feature_enabled + before_action :set_data_import, only: [:show, :start, :abandon, :error_logs, :skip_logs] + before_action :check_authorization + + def index + @data_imports = policy_scope(Current.account.data_imports).includes(:initiated_by).order(created_at: :desc) + data_import_ids = @data_imports.map(&:id) + @import_errors_counts = DataImportError.non_skip_logs.where(data_import_id: data_import_ids).group(:data_import_id).count + @skip_logs_counts = DataImportError.skip_logs.where(data_import_id: data_import_ids).group(:data_import_id).count + end + + def show + render_show + end + + def validate_source + totals = validate_intercom_source + render json: { valid: true, totals: totals } + rescue DataImports::Intercom::Client::AuthenticationError + render_source_validation_error('We could not validate this Intercom access key. Check the key and its permissions.') + rescue DataImports::Intercom::Client::Error + render_source_validation_error('Intercom could not be reached. Please try again.') + rescue ArgumentError => e + render_source_validation_error(e.message) + end + + def create + @data_import = creation_service.perform + unless @data_import + render json: { message: 'Another data import is already in progress.' }, status: :unprocessable_entity + return + end + + DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id) + render_show + rescue DataImports::Intercom::Client::AuthenticationError + render_source_validation_error('We could not validate this Intercom access key. Check the key and its permissions.') + rescue DataImports::Intercom::Client::Error + render_source_validation_error('Intercom could not be reached. Please try again.') + rescue ArgumentError => e + render_source_validation_error(e.message) + end + + def start + restart_service = DataImports::Intercom::RestartService.new(account: Current.account, data_import: @data_import) + restart_result = restart_service.perform + @data_import = restart_service.data_import + if restart_result == :access_token_missing + render json: { message: 'The Intercom access key for this import is unavailable.' }, status: :unprocessable_entity + return + end + + DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id) if restart_result == :enqueue + render_show + end + + def abandon + @data_import.abandon! + render_show + end + + def skip_logs + send_data( + skip_logs_csv, + filename: "data-import-#{@data_import.id}-skip-logs.csv", + type: 'text/csv' + ) + end + + def error_logs + send_data( + error_logs_csv, + filename: "data-import-#{@data_import.id}-error-logs.csv", + type: 'text/csv' + ) + end + + private + + def ensure_data_import_feature_enabled + raise Pundit::NotAuthorizedError unless Current.account.feature_enabled?(DATA_IMPORT_FEATURE) + end + + def set_data_import + @data_import = Current.account.data_imports.find(params[:id]) + end + + def check_authorization + authorize(@data_import || DataImport) + end + + def permitted_params + params.permit(:name, :source_provider, :access_token, import_types: []) + end + + def creation_service + DataImports::Intercom::CreationService.new( + account: Current.account, + initiated_by: Current.user, + source_params: permitted_params.to_h + ) + end + + def import_types + return DataImports::Intercom::Importer::DEFAULT_IMPORT_TYPES unless permitted_params.key?(:import_types) + + Array(permitted_params[:import_types]).compact_blank + end + + def validate_intercom_source + raise ArgumentError, 'Unsupported import source.' unless permitted_params[:source_provider] == 'intercom' + + DataImports::Intercom::CredentialsValidator.new( + access_token: permitted_params[:access_token], + import_types: import_types + ).perform + end + + def render_source_validation_error(message) + render json: { valid: false, message: message }, status: :unprocessable_entity + end + + def render_show + @import_errors_finder = DataImportErrorFinder.new(@data_import) + @skip_logs_finder = DataImportSkipLogFinder.new(@data_import, params) + render :show + end + + def skip_logs_csv + logs_csv(@data_import.import_errors.skip_logs) + end + + def error_logs_csv + logs_csv(@data_import.import_errors.non_skip_logs) + end + + def logs_csv(logs) + CSV.generate(headers: true) do |csv| + csv << %w[created_at kind source_object_type source_object_id error_code message details] + + logs.order(:created_at).find_each do |log| + csv << [ + log.created_at.iso8601, + log.details['kind'], + log.source_object_type, + log.source_object_id, + log.error_code, + log.message, + log.details.to_json + ] + end + end + end +end diff --git a/app/finders/data_import_error_finder.rb b/app/finders/data_import_error_finder.rb new file mode 100644 index 000000000..69d85b23c --- /dev/null +++ b/app/finders/data_import_error_finder.rb @@ -0,0 +1,11 @@ +class DataImportErrorFinder + RESULTS_LIMIT = 5 + + def initialize(data_import) + @data_import = data_import + end + + def import_errors + @data_import.import_errors.non_skip_logs.order(created_at: :desc).limit(RESULTS_LIMIT) + end +end diff --git a/app/finders/data_import_skip_log_finder.rb b/app/finders/data_import_skip_log_finder.rb new file mode 100644 index 000000000..ee932f1da --- /dev/null +++ b/app/finders/data_import_skip_log_finder.rb @@ -0,0 +1,35 @@ +class DataImportSkipLogFinder + RESULTS_LIMIT = 5 + SOURCE_OBJECT_TYPES = %w[contact conversation message].freeze + + attr_reader :selected_source_object_type + + def initialize(data_import, params = {}) + @data_import = data_import + @selected_source_object_type = valid_source_object_type(params[:skip_logs_type]) + end + + def skip_logs + filtered_scope.order(created_at: :desc).limit(RESULTS_LIMIT) + end + + def counts_by_type + base_scope.group(:source_object_type).count + end + + private + + def base_scope + @base_scope ||= @data_import.import_errors.skip_logs + end + + def filtered_scope + return base_scope if selected_source_object_type.blank? + + base_scope.where(source_object_type: selected_source_object_type) + end + + def valid_source_object_type(source_object_type) + source_object_type if SOURCE_OBJECT_TYPES.include?(source_object_type) + end +end diff --git a/app/javascript/dashboard/api/dataImports.js b/app/javascript/dashboard/api/dataImports.js new file mode 100644 index 000000000..b4c15b98a --- /dev/null +++ b/app/javascript/dashboard/api/dataImports.js @@ -0,0 +1,39 @@ +/* global axios */ + +import ApiClient from './ApiClient'; + +class DataImportsAPI extends ApiClient { + constructor() { + super('data_imports', { accountScoped: true }); + } + + start(id) { + return axios.post(`${this.url}/${id}/start`); + } + + abandon(id) { + return axios.post(`${this.url}/${id}/abandon`); + } + + show(id, params = {}) { + return axios.get(`${this.url}/${id}`, { params }); + } + + validateSource(payload) { + return axios.post(`${this.url}/validate_source`, payload); + } + + downloadSkipLogs(id) { + return axios.get(`${this.url}/${id}/skip_logs.csv`, { + responseType: 'blob', + }); + } + + downloadErrorLogs(id) { + return axios.get(`${this.url}/${id}/error_logs.csv`, { + responseType: 'blob', + }); + } +} + +export default new DataImportsAPI(); diff --git a/app/javascript/dashboard/components-next/sidebar/Sidebar.vue b/app/javascript/dashboard/components-next/sidebar/Sidebar.vue index a78460ca2..294e0dd5d 100644 --- a/app/javascript/dashboard/components-next/sidebar/Sidebar.vue +++ b/app/javascript/dashboard/components-next/sidebar/Sidebar.vue @@ -85,6 +85,13 @@ const hasFilteredUnreadCounts = computed(() => { ); }); +const hasDataImport = computed(() => { + return isFeatureEnabledonAccount.value( + accountId.value, + FEATURE_FLAGS.DATA_IMPORT + ); +}); + const fetchConversationUnreadCounts = ([currentAccountId, isEnabled]) => { if (!currentAccountId) return; @@ -856,6 +863,16 @@ const menuItems = computed(() => { icon: 'i-lucide-blocks', to: accountScopedRoute('settings_applications'), }, + ...(hasDataImport.value + ? [ + { + name: 'Settings Data', + label: t('SIDEBAR.DATA'), + icon: 'i-lucide-database', + to: accountScopedRoute('settings_data_imports'), + }, + ] + : []), { name: 'Settings Audit Logs', label: t('SIDEBAR.AUDIT_LOGS'), diff --git a/app/javascript/dashboard/components/Modal.vue b/app/javascript/dashboard/components/Modal.vue index 48d4e8d6b..1936a5c3c 100644 --- a/app/javascript/dashboard/components/Modal.vue +++ b/app/javascript/dashboard/components/Modal.vue @@ -7,7 +7,7 @@ import Button from 'dashboard/components-next/button/Button.vue'; const { modalType, closeOnBackdropClick, onClose } = defineProps({ closeOnBackdropClick: { type: Boolean, default: true }, showCloseButton: { type: Boolean, default: true }, - onClose: { type: Function, required: true }, + onClose: { type: Function, default: null }, fullWidth: { type: Boolean, default: false }, modalType: { type: String, default: 'centered' }, size: { type: String, default: '' }, @@ -35,7 +35,7 @@ const handleMouseDown = () => { const close = () => { show.value = false; emit('close'); - onClose(); + onClose?.(); }; const onMouseUp = () => { diff --git a/app/javascript/dashboard/featureFlags.js b/app/javascript/dashboard/featureFlags.js index 34404f0eb..058921eea 100644 --- a/app/javascript/dashboard/featureFlags.js +++ b/app/javascript/dashboard/featureFlags.js @@ -11,6 +11,7 @@ export const FEATURE_FLAGS = { CANNED_RESPONSES: 'canned_responses', CRM: 'crm', CUSTOM_ATTRIBUTES: 'custom_attributes', + DATA_IMPORT: 'data_import', INBOX_MANAGEMENT: 'inbox_management', INTEGRATIONS: 'integrations', LABELS: 'labels', diff --git a/app/javascript/dashboard/i18n/locale/en/settings.json b/app/javascript/dashboard/i18n/locale/en/settings.json index b621e63b1..eaecd7b80 100644 --- a/app/javascript/dashboard/i18n/locale/en/settings.json +++ b/app/javascript/dashboard/i18n/locale/en/settings.json @@ -340,6 +340,7 @@ "NOTIFICATIONS": "Notifications", "CANNED_RESPONSES": "Canned Responses", "INTEGRATIONS": "Integrations", + "DATA": "Data", "PROFILE_SETTINGS": "Profile Settings", "ACCOUNT_SETTINGS": "Account Settings", "APPLICATIONS": "Applications", @@ -411,6 +412,104 @@ "CAPTAIN_AI": "Captain", "CONVERSATION_WORKFLOW": "Conversation Workflow" }, + "DATA_IMPORTS": { + "HEADER": "Data", + "DESCRIPTION": "Bring your existing contacts and past conversations into this account from another support tool. Each import runs in the background, so you can keep working while it finishes, track its progress, and review anything that was skipped along the way.", + "LOADING": "Fetching imports", + "DEFAULT_IMPORT_NAME": "Intercom import", + "TABS": { + "IMPORT": "Import", + "EXPORT": "Export" + }, + "TYPES": { + "CONTACTS": "Contacts", + "CONVERSATIONS": "Conversations", + "MESSAGES": "Messages" + }, + "DRAWER": { + "TITLE": "New import", + "SOURCE": "Source", + "NAME": "Import name", + "NAME_PLACEHOLDER": "July Intercom migration", + "ACCESS_KEY": "Intercom access key", + "ACCESS_KEY_PLACEHOLDER": "Paste your Intercom access key", + "DATA_TYPES": "Data to import", + "VALIDATING": "Validating access key...", + "VALID_KEY": "Access key validated.", + "INVALID_KEY": "Could not validate this access key.", + "ACTIVE_IMPORT": "Wait for the active import to finish before starting another one.", + "CANCEL": "Cancel", + "IMPORT": "Import" + }, + "EXPORT": { + "TITLE": "Exports are on the way", + "DESCRIPTION": "Export your contacts and conversations out of this account. This workflow is coming soon.", + "COMING_SOON": "Coming soon" + }, + "TABLE": { + "TITLE": "Recent imports", + "EMPTY": "No imports yet", + "EMPTY_DESCRIPTION": "Start an import to bring your existing customer history into this account.", + "NEW_IMPORT": "Import", + "COUNT": "{count} imports", + "UNNAMED": "Untitled import", + "IMPORTED_COUNT": "{count} imported", + "VIEW": "View import", + "NAME": "Name", + "TYPE": "Type", + "STATUS": "Status", + "IMPORTED": "Imported", + "CREATED": "Created", + "ABANDON": "Abandon" + }, + "DETAIL": { + "BACK": "Back to imports", + "ERRORS": "Errors", + "SKIP_LOGS": "Skip logs", + "SOURCE": "Source", + "IMPORT_TYPES": "Import types", + "CREATED": "Created", + "DURATION": "Duration", + "INITIATED_BY": "Started by", + "PROGRESS": "Import progress", + "PROGRESS_WITH_TOTAL": "{imported} of {total} imported", + "PROGRESS_WITHOUT_TOTAL": "{imported} imported", + "PROGRESS_OF_TOTAL": "of {total} imported", + "PROGRESS_IMPORTED": "imported", + "LAST_UPDATED_TOOLTIP": "Last updated {time}", + "NO_SKIP_LOGS": "No skipped or failed records recorded.", + "DOWNLOAD_SKIP_LOGS": "Download CSV", + "DOWNLOAD_ERROR_LOGS": "Download CSV", + "ALL_SKIP_LOGS": "All", + "KIND": "Kind", + "NO_ERRORS": "No errors recorded.", + "ERROR_CODE": "Code", + "SOURCE_OBJECT": "Source object", + "MESSAGE": "Message" + }, + "MONITOR": { + "LIVE": "Live updates every {seconds}s", + "LAST_UPDATED": "Last updated {time}", + "REFRESH": "Refresh", + "REFRESHING": "Refreshing", + "STAGES": { + "unknown": "Waiting for update", + "queued": "Queued", + "contacts": "Importing contacts", + "conversations": "Importing conversations", + "finalizing": "Finalizing import", + "completed": "Completed", + "completed_with_errors": "Completed with errors", + "failed": "Failed", + "abandoned": "Abandoned" + } + }, + "ALERTS": { + "IMPORT_STARTED": "Intercom import has started.", + "IMPORT_ABANDONED": "Intercom import has been abandoned.", + "IMPORT_FAILED": "Could not start the Intercom import." + } + }, "CAPTAIN_SETTINGS": { "TITLE": "Captain Settings", "DESCRIPTION": "Configure your AI models and features for Captain. Captain follows a credit based billing, you will be charged credits for every action Captain takes based on the model selected.", diff --git a/app/javascript/dashboard/routes/dashboard/settings/components/BaseSettingsHeader.vue b/app/javascript/dashboard/routes/dashboard/settings/components/BaseSettingsHeader.vue index 167d13cf5..2fa3bf0b8 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/components/BaseSettingsHeader.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/components/BaseSettingsHeader.vue @@ -52,9 +52,11 @@ const helpURL = getHelpUrlForFeature(props.featureName); v-if="title" class="flex items-center justify-between w-full gap-4 min-h-8 mb-2" > -

- {{ title }} -

+ +

+ {{ title }} +

+
+import { + computed, + onActivated, + onBeforeUnmount, + onDeactivated, + ref, +} from 'vue'; +import { useI18n } from 'vue-i18n'; +import { useRouter } from 'vue-router'; +import { useStoreGetters } from 'dashboard/composables/store'; + +import Button from 'dashboard/components-next/button/Button.vue'; +import Icon from 'dashboard/components-next/icon/Icon.vue'; +import TabBar from 'dashboard/components-next/tabbar/TabBar.vue'; +import SettingsLayout from '../SettingsLayout.vue'; +import BaseSettingsHeader from '../components/BaseSettingsHeader.vue'; +import DataImportsAPI from 'dashboard/api/dataImports'; +import NewImportDialog from './NewImportDialog.vue'; +import { importSourceFor } from './importSources'; +import { + POLL_INTERVAL_MS, + formatDate, + formatStatus, + importedCount, + isActiveImport, + isActiveIntercomImport, + statusDotClass, +} from './importStatus'; + +const { t } = useI18n(); +const getters = useStoreGetters(); +const router = useRouter(); + +const dataImports = ref([]); +const isLoading = ref(true); +const isRefreshing = ref(false); +const isPolling = ref(false); +const showImportDrawer = ref(false); +const activeTab = ref('import'); +let pollTimer; +let isPageActive = false; + +const accountId = getters.getCurrentAccountId; + +const tabs = computed(() => [ + { key: 'import', label: t('DATA_IMPORTS.TABS.IMPORT') }, + { key: 'export', label: t('DATA_IMPORTS.TABS.EXPORT') }, +]); + +const activeTabIndex = computed(() => + tabs.value.findIndex(tab => tab.key === activeTab.value) +); + +const hasActiveImport = computed(() => dataImports.value.some(isActiveImport)); +const hasActiveIntercomImport = computed(() => + dataImports.value.some(isActiveIntercomImport) +); + +const dataImportRoute = dataImport => ({ + name: 'settings_data_import_show', + params: { accountId: accountId.value, dataImportId: dataImport.id }, +}); + +const importTypesFor = dataImport => + dataImport.import_types?.length + ? dataImport.import_types + : [dataImport.data_type]; + +const importTypeLabel = dataImport => + importTypesFor(dataImport) + .map(type => { + if (type === 'contacts') return t('DATA_IMPORTS.TYPES.CONTACTS'); + if (type === 'conversations') { + return t('DATA_IMPORTS.TYPES.CONVERSATIONS'); + } + return type; + }) + .join(', '); + +const fetchImports = async () => { + const response = await DataImportsAPI.get(); + dataImports.value = response.data.payload || []; +}; + +const stopPolling = () => { + if (!pollTimer) return; + + window.clearInterval(pollTimer); + pollTimer = null; +}; + +const refreshImportsInBackground = async () => { + if ( + !isPageActive || + isPolling.value || + !hasActiveImport.value || + document.hidden + ) { + return; + } + + isPolling.value = true; + try { + await fetchImports(); + } finally { + isPolling.value = false; + if (!hasActiveImport.value) stopPolling(); + } +}; + +const startPolling = () => { + stopPolling(); + if (!isPageActive || !hasActiveImport.value) return; + + pollTimer = window.setInterval(refreshImportsInBackground, POLL_INTERVAL_MS); +}; + +const refresh = async ({ showLoader = true } = {}) => { + if (showLoader) isLoading.value = true; + else isRefreshing.value = true; + + try { + await fetchImports(); + } finally { + isLoading.value = false; + isRefreshing.value = false; + if (isPageActive) { + if (hasActiveImport.value && !pollTimer) startPolling(); + if (!hasActiveImport.value) stopPolling(); + } + } +}; + +const openImport = dataImport => { + router.push(dataImportRoute(dataImport)); +}; + +const openImportDrawer = () => { + if (!hasActiveIntercomImport.value) showImportDrawer.value = true; +}; + +const onImportCreated = dataImportId => { + showImportDrawer.value = false; + router.push({ + name: 'settings_data_import_show', + params: { accountId: accountId.value, dataImportId }, + }); +}; + +const onTabChanged = tab => { + activeTab.value = tab.key; +}; + +const handleVisibilityChange = () => { + if (isPageActive && !document.hidden && hasActiveImport.value) { + refreshImportsInBackground(); + } +}; + +onActivated(async () => { + isPageActive = true; + await refresh(); + if (!isPageActive) return; + + startPolling(); + document.addEventListener('visibilitychange', handleVisibilityChange); +}); + +onDeactivated(() => { + isPageActive = false; + stopPolling(); + document.removeEventListener('visibilitychange', handleVisibilityChange); +}); + +onBeforeUnmount(() => { + isPageActive = false; + stopPolling(); + document.removeEventListener('visibilitychange', handleVisibilityChange); +}); + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/NewImportDialog.vue b/app/javascript/dashboard/routes/dashboard/settings/data/NewImportDialog.vue new file mode 100644 index 000000000..56fda08f3 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/NewImportDialog.vue @@ -0,0 +1,210 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/Show.vue b/app/javascript/dashboard/routes/dashboard/settings/data/Show.vue new file mode 100644 index 000000000..ed89c66a9 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/Show.vue @@ -0,0 +1,238 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportDetailHeader.vue b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportDetailHeader.vue new file mode 100644 index 000000000..935d63b87 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportDetailHeader.vue @@ -0,0 +1,128 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportErrorsSection.vue b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportErrorsSection.vue new file mode 100644 index 000000000..0cbfd9d2b --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportErrorsSection.vue @@ -0,0 +1,78 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportLogSection.vue b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportLogSection.vue new file mode 100644 index 000000000..8ccbaf5fe --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportLogSection.vue @@ -0,0 +1,105 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportProgress.vue b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportProgress.vue new file mode 100644 index 000000000..fb2cd003c --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportProgress.vue @@ -0,0 +1,100 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSkipLogsSection.vue b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSkipLogsSection.vue new file mode 100644 index 000000000..2b3d9c467 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSkipLogsSection.vue @@ -0,0 +1,127 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSummaryTiles.vue b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSummaryTiles.vue new file mode 100644 index 000000000..01e239b32 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/components/ImportSummaryTiles.vue @@ -0,0 +1,110 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/data.routes.js b/app/javascript/dashboard/routes/dashboard/settings/data/data.routes.js new file mode 100644 index 000000000..1e81d3e9a --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/data.routes.js @@ -0,0 +1,34 @@ +import { FEATURE_FLAGS } from '../../../../featureFlags'; +import { frontendURL } from '../../../../helper/URLHelper'; +import SettingsWrapper from '../SettingsWrapper.vue'; +import Index from './Index.vue'; +import Show from './Show.vue'; + +export default { + routes: [ + { + path: frontendURL('accounts/:accountId/settings/data'), + component: SettingsWrapper, + children: [ + { + path: '', + name: 'settings_data_imports', + component: Index, + meta: { + featureFlag: FEATURE_FLAGS.DATA_IMPORT, + permissions: ['administrator'], + }, + }, + { + path: ':dataImportId', + name: 'settings_data_import_show', + component: Show, + meta: { + featureFlag: FEATURE_FLAGS.DATA_IMPORT, + permissions: ['administrator'], + }, + }, + ], + }, + ], +}; diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/importSources.js b/app/javascript/dashboard/routes/dashboard/settings/data/importSources.js new file mode 100644 index 000000000..85e832a6c --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/importSources.js @@ -0,0 +1,17 @@ +export const IMPORT_SOURCES = [ + { + value: 'intercom', + label: 'Intercom', + icon: '/dashboard/images/integrations/intercom.png', + }, +]; + +const DEFAULT_IMPORT_SOURCE = { + value: 'file', + label: 'File import', + iconClass: 'i-lucide-file-text', +}; + +export const importSourceFor = dataImport => + IMPORT_SOURCES.find(source => source.value === dataImport?.source_provider) || + DEFAULT_IMPORT_SOURCE; diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/importStatus.js b/app/javascript/dashboard/routes/dashboard/settings/data/importStatus.js new file mode 100644 index 000000000..f658ee04c --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/importStatus.js @@ -0,0 +1,84 @@ +export const POLL_INTERVAL_MS = 5000; + +export const ACTIVE_IMPORT_STATUSES = ['pending', 'processing']; + +export const isActiveImport = dataImport => + ACTIVE_IMPORT_STATUSES.includes(dataImport?.status); + +export const isIntercomImport = dataImport => + dataImport?.data_type === 'intercom' && + dataImport?.source_provider === 'intercom'; + +export const isActiveIntercomImport = dataImport => + isIntercomImport(dataImport) && isActiveImport(dataImport); + +export const isAbandonableImport = dataImport => + isActiveIntercomImport(dataImport); + +export const importedCount = dataImport => { + if (!isIntercomImport(dataImport)) { + return Number(dataImport?.processed_records || 0); + } + + return ['contacts', 'conversations', 'messages'].reduce( + (total, key) => total + Number(dataImport?.stats?.[key]?.imported || 0), + 0 + ); +}; + +export const importStageKey = dataImport => { + if (!dataImport) return 'unknown'; + + if (dataImport.status === 'completed') return 'completed'; + if (dataImport.status === 'completed_with_errors') { + return 'completed_with_errors'; + } + if (dataImport.status === 'failed') return 'failed'; + if (dataImport.status === 'abandoned') return 'abandoned'; + if (dataImport.status === 'pending') return 'queued'; + + const importTypes = dataImport.import_types?.length + ? dataImport.import_types + : [dataImport.data_type]; + const cursor = dataImport.cursor || {}; + + if (importTypes.includes('contacts') && !cursor.contacts?.completed) { + return 'contacts'; + } + + if ( + importTypes.includes('conversations') && + !cursor.conversations?.completed + ) { + return 'conversations'; + } + + return 'finalizing'; +}; + +export const formatStatus = value => value?.replaceAll('_', ' ') || '-'; + +export const sourceObjectLabel = record => + [record.source_object_type, record.source_object_id] + .filter(Boolean) + .join(': ') || '-'; + +export const formatDate = value => { + if (!value) return '-'; + return new Intl.DateTimeFormat(undefined, { + dateStyle: 'medium', + timeStyle: 'short', + }).format(new Date(value)); +}; + +const STATUS_DOT_CLASS = { + pending: 'bg-n-amber-9', + processing: 'bg-n-blue-9', + completed: 'bg-n-teal-9', + completed_with_errors: 'bg-n-amber-9', + failed: 'bg-n-ruby-9', + abandoned: 'bg-n-slate-9', +}; + +export const statusDotClass = status => + STATUS_DOT_CLASS[status] || 'bg-n-slate-9'; diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/specs/importStatus.spec.js b/app/javascript/dashboard/routes/dashboard/settings/data/specs/importStatus.spec.js new file mode 100644 index 000000000..dcfe7d9bd --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/specs/importStatus.spec.js @@ -0,0 +1,92 @@ +import { + formatDate, + importedCount, + isActiveIntercomImport, + statusDotClass, +} from '../importStatus'; + +describe('importStatus', () => { + describe('isActiveIntercomImport', () => { + it('only treats pending or processing Intercom imports as active', () => { + expect( + isActiveIntercomImport({ + data_type: 'intercom', + source_provider: 'intercom', + status: 'processing', + }) + ).toBe(true); + expect( + isActiveIntercomImport({ + data_type: 'contacts', + source_provider: null, + status: 'processing', + }) + ).toBe(false); + expect( + isActiveIntercomImport({ + data_type: 'intercom', + source_provider: 'intercom', + status: 'completed', + }) + ).toBe(false); + }); + }); + + describe('importedCount', () => { + it('sums Intercom imported stats', () => { + expect( + importedCount({ + data_type: 'intercom', + source_provider: 'intercom', + processed_records: 20, + stats: { + contacts: { imported: 2 }, + conversations: { imported: 3 }, + messages: { imported: 10 }, + }, + }) + ).toBe(15); + }); + + it('uses processed records for legacy imports', () => { + expect( + importedCount({ + data_type: 'contacts', + source_provider: null, + processed_records: 7, + stats: {}, + }) + ).toBe(7); + }); + }); + + describe('statusDotClass', () => { + it('maps each status to its dot color class', () => { + expect(statusDotClass('pending')).toBe('bg-n-amber-9'); + expect(statusDotClass('processing')).toBe('bg-n-blue-9'); + expect(statusDotClass('completed')).toBe('bg-n-teal-9'); + expect(statusDotClass('completed_with_errors')).toBe('bg-n-amber-9'); + expect(statusDotClass('failed')).toBe('bg-n-ruby-9'); + expect(statusDotClass('abandoned')).toBe('bg-n-slate-9'); + }); + + it('falls back to slate for unknown or missing status', () => { + expect(statusDotClass('unknown')).toBe('bg-n-slate-9'); + expect(statusDotClass(undefined)).toBe('bg-n-slate-9'); + }); + }); + + describe('formatDate', () => { + it('returns a dash for empty values', () => { + expect(formatDate(null)).toBe('-'); + expect(formatDate('')).toBe('-'); + expect(formatDate(undefined)).toBe('-'); + }); + + it('formats a valid date into a readable string', () => { + const formatted = formatDate('2026-07-10T18:09:00Z'); + expect(formatted).not.toBe('-'); + expect(formatted).toContain('2026'); + }); + }); +}); diff --git a/app/javascript/dashboard/routes/dashboard/settings/data/specs/pollingLifecycle.spec.js b/app/javascript/dashboard/routes/dashboard/settings/data/specs/pollingLifecycle.spec.js new file mode 100644 index 000000000..7e74bd565 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/data/specs/pollingLifecycle.spec.js @@ -0,0 +1,121 @@ +import { flushPromises, mount } from '@vue/test-utils'; +import { KeepAlive, defineComponent, h, nextTick, ref } from 'vue'; +import DataImportsAPI from 'dashboard/api/dataImports'; +import Index from '../Index.vue'; +import Show from '../Show.vue'; + +vi.mock('dashboard/api/dataImports', () => ({ + default: { + get: vi.fn(), + show: vi.fn(), + }, +})); + +vi.mock('dashboard/composables/store', () => ({ + useStoreGetters: () => ({ getCurrentAccountId: { value: 1 } }), +})); + +vi.mock('dashboard/composables', () => ({ + useAlert: vi.fn(), +})); + +vi.mock('vue-i18n', () => ({ + useI18n: () => ({ t: key => key }), +})); + +vi.mock('vue-router', async importOriginal => ({ + ...(await importOriginal()), + useRoute: () => ({ params: { dataImportId: 1 } }), + useRouter: () => ({ push: vi.fn() }), +})); + +const deferredRequest = () => { + let resolve; + const promise = new Promise(resolvePromise => { + resolve = resolvePromise; + }); + return { promise, resolve }; +}; + +const mountKeptAlive = component => { + const Host = defineComponent({ + setup() { + const visible = ref(true); + return { visible }; + }, + render() { + return h(KeepAlive, null, { + default: () => (this.visible ? h(component) : null), + }); + }, + }); + + return mount(Host, { + global: { + stubs: { + SettingsLayout: true, + BaseSettingsHeader: true, + Button: true, + Icon: true, + TabBar: true, + NewImportDialog: true, + ImportDetailHeader: true, + ImportSummaryTiles: true, + ImportProgress: true, + ImportErrorsSection: true, + ImportSkipLogsSection: true, + }, + mocks: { + $t: key => key, + }, + }, + }); +}; + +describe('data import polling lifecycle', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + }); + + it('does not start list polling after the page deactivates', async () => { + const request = deferredRequest(); + DataImportsAPI.get.mockReturnValue(request.promise); + const wrapper = mountKeptAlive(Index); + await nextTick(); + + wrapper.vm.visible = false; + await nextTick(); + request.resolve({ data: { payload: [{ status: 'processing' }] } }); + await flushPromises(); + await vi.advanceTimersByTimeAsync(5000); + + expect(DataImportsAPI.get).toHaveBeenCalledTimes(1); + wrapper.unmount(); + }); + + it('does not start detail polling after the page deactivates', async () => { + const request = deferredRequest(); + DataImportsAPI.show.mockReturnValue(request.promise); + const wrapper = mountKeptAlive(Show); + await nextTick(); + + wrapper.vm.visible = false; + await nextTick(); + request.resolve({ + data: { + status: 'processing', + skip_logs_filters: {}, + }, + }); + await flushPromises(); + await vi.advanceTimersByTimeAsync(5000); + + expect(DataImportsAPI.show).toHaveBeenCalledTimes(1); + wrapper.unmount(); + }); +}); diff --git a/app/javascript/dashboard/routes/dashboard/settings/inbox/Index.vue b/app/javascript/dashboard/routes/dashboard/settings/inbox/Index.vue index 884c198c4..0026cd8a9 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/inbox/Index.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/inbox/Index.vue @@ -1,5 +1,5 @@ ' + } + + expect(described_class.new(part).perform).to eq( + 'Intercom teammate closed the conversation: Customer confirmed resolution' + ) + end +end diff --git a/spec/services/data_imports/intercom/client_spec.rb b/spec/services/data_imports/intercom/client_spec.rb new file mode 100644 index 000000000..78498121a --- /dev/null +++ b/spec/services/data_imports/intercom/client_spec.rb @@ -0,0 +1,16 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::Client do + let(:client) { described_class.new(access_token: 'intercom-token') } + + describe '#list_contacts' do + it 'wraps transport failures in a retryable client error', :aggregate_failures do + allow(HTTParty).to receive(:get).and_raise(SocketError, 'getaddrinfo failed') + + expect { client.list_contacts }.to raise_error(DataImports::Intercom::Client::Error) do |error| + expect(error.message).to eq('Intercom API request failed before receiving a response: getaddrinfo failed') + expect(error.body).to include(transport_error_class: 'SocketError') + end + end + end +end diff --git a/spec/services/data_imports/intercom/creation_service_spec.rb b/spec/services/data_imports/intercom/creation_service_spec.rb new file mode 100644 index 000000000..4345b3715 --- /dev/null +++ b/spec/services/data_imports/intercom/creation_service_spec.rb @@ -0,0 +1,52 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::CreationService do + let(:account) { create(:account) } + let(:user) { create(:user, account: account) } + let(:validator) { instance_double(DataImports::Intercom::CredentialsValidator, perform: { 'contacts' => 12 }) } + + before do + allow(DataImports::Intercom::CredentialsValidator).to receive(:new).and_return(validator) + end + + it 'validates and creates an import with its credentials and totals', :aggregate_failures do + data_import = described_class.new( + account: account, + initiated_by: user, + source_params: { + name: 'Migration run', + source_provider: 'intercom', + access_token: ' intercom-token ', + import_types: %w[contacts] + } + ).perform + + expect(data_import).to have_attributes( + name: 'Migration run', + source_type: 'api', + source_provider: 'intercom', + import_types: %w[contacts], + access_token: 'intercom-token', + initiated_by_id: user.id + ) + expect(data_import.stats.dig('contacts', 'total')).to eq(12) + expect(data_import.active_intercom_import_run_id).to be_present + end + + it 'returns no import without validating when another import is active' do + create(:data_import, :intercom, account: account, status: :processing) + + data_import = described_class.new( + account: account, + initiated_by: user, + source_params: { + name: 'Second run', + source_provider: 'intercom', + access_token: 'intercom-token' + } + ).perform + + expect(data_import).to be_nil + expect(validator).not_to have_received(:perform) + end +end diff --git a/spec/services/data_imports/intercom/credentials_validator_spec.rb b/spec/services/data_imports/intercom/credentials_validator_spec.rb new file mode 100644 index 000000000..3d3f9698b --- /dev/null +++ b/spec/services/data_imports/intercom/credentials_validator_spec.rb @@ -0,0 +1,54 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::CredentialsValidator do + let(:client) { instance_double(DataImports::Intercom::Client) } + + before do + allow(DataImports::Intercom::Client).to receive(:new).with(access_token: 'intercom-token').and_return(client) + allow(client).to receive(:list_contacts) + allow(client).to receive(:list_conversations) + end + + it 'validates and counts only contacts when conversations are not selected' do + allow(client).to receive(:list_contacts).with(per_page: 1).and_return('total_count' => 42) + + totals = described_class.new(access_token: ' intercom-token ', import_types: %w[contacts]).perform + + expect(totals).to eq('contacts' => 42) + expect(client).not_to have_received(:list_conversations) + end + + it 'validates contact access and counts only conversations when contacts are not selected' do + allow(client).to receive(:list_contacts).with(per_page: 1).and_return('total_count' => 42) + allow(client).to receive(:list_conversations).with(per_page: 1).and_return('total_count' => 17) + + totals = described_class.new(access_token: 'intercom-token', import_types: %w[conversations]).perform + + expect(totals).to eq('conversations' => 17) + expect(client).to have_received(:list_contacts).with(per_page: 1) + end + + it 'keeps an undiscovered total absent' do + allow(client).to receive(:list_contacts).with(per_page: 1).and_return('data' => []) + + totals = described_class.new(access_token: 'intercom-token', import_types: %w[contacts]).perform + + expect(totals).to be_empty + end + + it 'preserves a known zero total' do + allow(client).to receive(:list_contacts).with(per_page: 1).and_return('total_count' => 0) + + totals = described_class.new(access_token: 'intercom-token', import_types: %w[contacts]).perform + + expect(totals).to eq('contacts' => 0) + end + + it 'rejects an empty access key before calling Intercom' do + expect do + described_class.new(access_token: '', import_types: %w[contacts]).perform + end.to raise_error(ArgumentError, 'Intercom access key is required.') + + expect(DataImports::Intercom::Client).not_to have_received(:new) + end +end diff --git a/spec/services/data_imports/intercom/importer_spec.rb b/spec/services/data_imports/intercom/importer_spec.rb new file mode 100644 index 000000000..67c2ec576 --- /dev/null +++ b/spec/services/data_imports/intercom/importer_spec.rb @@ -0,0 +1,968 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::Importer do + let(:account) { create(:account) } + let(:data_import) do + create( + :data_import, :intercom, + account: account + ) + end + let(:client) { instance_double(DataImports::Intercom::Client) } + let(:contact_payload) do + { + 'id' => 'contact_1', + 'external_id' => 'external_1', + 'email' => 'CUSTOMER@Example.com', + 'phone' => '15551234567', + 'name' => 'Customer One', + 'created_at' => 1_700_000_000, + 'updated_at' => 1_700_000_100 + } + end + let(:conversation_payload) do + { + 'id' => 'conversation_1', + 'created_at' => 1_700_000_000, + 'updated_at' => 1_700_000_200, + 'state' => 'closed', + 'open' => false, + 'admin_assignee_id' => 123, + 'team_assignee_id' => 456, + 'contacts' => { 'contacts' => [{ 'id' => 'contact_1' }] }, + 'source' => { + 'id' => 'source_1', + 'type' => 'email', + 'delivered_as' => 'customer_initiated', + 'subject' => 'Need help', + 'body' => '

Hello there

', + 'author' => { 'type' => 'user', 'id' => 'contact_1', 'email' => 'CUSTOMER@example.com' } + }, + 'conversation_parts' => { + 'conversation_parts' => [ + { + 'id' => 'part_1', + 'part_type' => 'comment', + 'body' => '

Admin reply

', + 'created_at' => 1_700_000_100, + 'updated_at' => 1_700_000_100, + 'author' => { 'type' => 'admin', 'id' => 'admin_1' }, + 'attachments' => [] + }, + { + 'id' => 'part_2', + 'part_type' => 'note', + 'body' => 'Internal note', + 'created_at' => 1_700_000_150, + 'updated_at' => 1_700_000_150, + 'author' => { 'type' => 'admin', 'id' => 'admin_1' }, + 'attachments' => [] + } + ] + } + } + end + + before do + account.enable_features!('data_import') + allow(DataImports::Intercom::Client).to receive(:new).with(access_token: 'intercom-token').and_return(client) + allow(client).to receive(:list_contacts).with(starting_after: nil).and_return( + 'data' => [contact_payload], + 'total_count' => 1, + 'pages' => { 'next' => nil } + ) + allow(client).to receive(:list_conversations).with(starting_after: nil).and_return( + 'conversations' => [{ 'id' => 'conversation_1' }], + 'total_count' => 1, + 'pages' => { 'next' => nil } + ) + allow(client).to receive(:retrieve_conversation).with('conversation_1').and_return(conversation_payload) + allow(client).to receive(:retrieve_contact).with('contact_1').and_return(contact_payload) + end + + it 'imports contacts, conversations, messages, and source-bucket inboxes without normal message creation callbacks', :aggregate_failures do + described_class.new(data_import: data_import).perform + + contact = account.contacts.find_by!(email: 'customer@example.com') + expect(contact.name).to eq('Customer One') + expect(contact.phone_number).to eq('+15551234567') + expect(contact).to be_lead + expect(contact.custom_attributes).to include('intercom_contact_id' => 'contact_1') + + inbox = account.inboxes.find_by!(name: 'Intercom Import - Email') + expect(inbox.channel.additional_attributes).to include('source_bucket' => 'email', 'import_placeholder' => true) + + conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1') + expect(conversation).to have_attributes( + status: 'resolved', + inbox_id: inbox.id, + contact_id: contact.id + ) + expect(conversation.additional_attributes.dig('source', 'routing_method')).to eq('source_bucket_api_inbox') + + expect(conversation.messages.order(:created_at).pluck(:content)).to eq(["Need help\n\nHello there", 'Admin reply', 'Internal note']) + expect(conversation.messages.order(:created_at).map(&:message_type)).to eq(%w[incoming outgoing outgoing]) + expect(conversation.messages.order(:created_at).last.private).to be(true) + + expect(data_import.reload).to be_completed + expect(data_import.stats).to include( + 'contacts' => include('imported' => 1, 'skipped' => 0, 'total' => 1), + 'conversations' => include('imported' => 1, 'skipped' => 0, 'total' => 1), + 'messages' => include('imported' => 3, 'skipped' => 0, 'total' => 3), + 'errors' => { 'count' => 0 } + ) + expect(data_import.processed_records).to eq(5) + expect(data_import.items.imported.count).to eq(2) + expect(DataImportMapping.where(data_import: data_import).count).to eq(5) + end + + it 'imports historical records without dispatching record events or outbound side effects', :aggregate_failures do + dispatched_events = [] + allow(Rails.configuration.dispatcher).to receive(:dispatch) do |event_name, *_args| + dispatched_events << event_name + end + clear_enqueued_jobs + + described_class.new(data_import: data_import).perform + + record_events = [ + Events::Types::CONTACT_CREATED, + Events::Types::CONTACT_UPDATED, + Events::Types::CONVERSATION_CREATED, + Events::Types::CONVERSATION_UPDATED, + Events::Types::CONVERSATION_STATUS_CHANGED, + Events::Types::ASSIGNEE_CHANGED, + Events::Types::TEAM_CHANGED, + Events::Types::MESSAGE_CREATED, + Events::Types::FIRST_REPLY_CREATED, + Events::Types::REPLY_CREATED + ] + side_effect_jobs = [SendReplyJob, EventDispatcherJob, ActionCableBroadcastJob, WebhookJob, HookJob] + + expect(dispatched_events & record_events).to be_empty + expect(enqueued_jobs.pluck(:job) & side_effect_jobs).to be_empty + expect(Notification.where(account: account)).to be_empty + end + + context 'when Intercom contact activity timestamps are available' do + let(:contact_payload) do + super().merge('last_seen_at' => 1_700_000_050, 'last_replied_at' => 1_700_000_090) + end + + it 'prefers last_seen_at for contact activity' do + described_class.new(data_import: data_import).import_contacts_page + + contact = account.contacts.find_by!(email: 'customer@example.com') + expect(contact.last_activity_at).to eq(Time.zone.at(1_700_000_050)) + end + end + + context 'when Intercom contact last_seen_at is unavailable' do + let(:contact_payload) do + super().merge('last_seen_at' => nil, 'last_replied_at' => 1_700_000_090) + end + + it 'falls back to last_replied_at for contact activity' do + described_class.new(data_import: data_import).import_contacts_page + + contact = account.contacts.find_by!(email: 'customer@example.com') + expect(contact.last_activity_at).to eq(Time.zone.at(1_700_000_090)) + end + end + + it 'leaves contact activity blank when Intercom activity timestamps are unavailable' do + described_class.new(data_import: data_import).import_contacts_page + + contact = account.contacts.find_by!(email: 'customer@example.com') + expect(contact.last_activity_at).to be_nil + end + + it 'updates message totals by delta when a conversation page is retried' do + importer = described_class.new(data_import: data_import) + + importer.import_conversations_page + importer.import_conversations_page + + expect(data_import.reload.stats.dig('messages', 'total')).to eq(3) + item = data_import.items.find_by!(source_object_type: 'conversation', source_object_id: 'conversation_1') + expect(item.metadata['message_total_contribution']).to eq(3) + end + + it 'reconciles imported message stats from same-run mappings on retry' do + described_class.new(data_import: data_import).import_conversations_page + stats = data_import.reload.stats.deep_dup + stats['messages']['imported'] = 0 + data_import.update!(stats: stats) + + described_class.new(data_import: data_import).import_conversations_page + + expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3) + end + + it 'indexes imported messages for advanced search' do + allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(true) + allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(false) + reindexed_message_ids = [] + original_reindex_for_search = Message.instance_method(:reindex_for_search) + Message.define_method(:reindex_for_search) { reindexed_message_ids << id } + Message.__send__(:private, :reindex_for_search) + + described_class.new(data_import: data_import).perform + + expect(reindexed_message_ids).to match_array(Message.where(account_id: account.id).pluck(:id)) + ensure + Message.define_method(:reindex_for_search, original_reindex_for_search) + Message.__send__(:private, :reindex_for_search) + end + + it 'keeps imported messages successful when search reindexing fails', :aggregate_failures do + allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(true) + allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(false) + # rubocop:disable RSpec/AnyInstance + allow_any_instance_of(Message).to receive(:reindex_for_search).and_raise(StandardError, 'search unavailable') + # rubocop:enable RSpec/AnyInstance + + described_class.new(data_import: data_import).perform + + message = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:source:source_1') + mapping = data_import.mappings.find_by!(source_object_type: 'message', source_object_id: 'conversation:conversation_1:source:source_1') + expect(mapping.chatwoot_record).to eq(message) + expect(data_import.reload).to be_completed + expect(data_import.import_errors.exists?).to be(false) + expect(data_import.stats.dig('messages', 'imported')).to eq(3) + end + + describe '#start!' do + it 'does not overwrite an import abandoned by another process', :aggregate_failures do + importer = described_class.new(data_import: data_import) + + DataImport.find(data_import.id).update!( + status: :abandoned, + abandoned_at: Time.current + ) + + expect(importer.start!).to be_nil + expect(data_import.reload).to be_abandoned + expect(data_import.started_at).to be_nil + end + end + + describe '#perform' do + it 'stops when the import was abandoned before processing starts' do + importer = described_class.new(data_import: data_import) + DataImport.find(data_import.id).update!( + status: :abandoned, + abandoned_at: Time.current + ) + + expect(client).not_to receive(:list_contacts) + + importer.perform + + expect(data_import.reload).to be_abandoned + end + end + + describe '#import_conversations_page' do + it 'stops an in-flight page when a newer import run takes over', :aggregate_failures do + run_id = 'intercom-run-1' + data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id }) + allow(client).to receive(:list_conversations).with(starting_after: nil).and_return( + 'conversations' => [{ 'id' => 'conversation_1' }, { 'id' => 'conversation_2' }], + 'pages' => { 'next' => { 'starting_after' => 'next-conversation-cursor' } } + ) + allow(client).to receive(:retrieve_conversation).with('conversation_1') do + data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' }) + conversation_payload + end + + result = described_class.new(data_import: data_import, run_id: run_id).import_conversations_page + + expect(result).to be_done + expect(client).not_to have_received(:retrieve_conversation).with('conversation_2') + expect(account.conversations.where(identifier: 'intercom:conversation_1')).to be_empty + expect(account.contacts.where(email: 'customer@example.com')).to be_empty + expect(data_import.reload.cursor.dig('conversations', 'starting_after')).to be_nil + end + + it 'rolls back a newly inserted conversation when mapping persistence fails', :aggregate_failures do + importer = described_class.new(data_import: data_import) + allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:| + raise StandardError, 'mapping failed' if object_type == 'conversation' + + method.call(object_type, source_id, record, metadata: metadata) + end + + importer.import_conversations_page + + expect(account.conversations.where(identifier: 'intercom:conversation_1')).to be_empty + item = data_import.items.find_by!(source_object_type: 'conversation', source_object_id: 'conversation_1') + expect(item).to be_failed + expect(item.last_error_message).to eq('mapping failed') + end + + it 'rolls back a newly inserted contact when mapping persistence fails', :aggregate_failures do + sparse_contact = contact_payload.slice('id', 'name', 'created_at', 'updated_at') + allow(client).to receive(:retrieve_contact).with('contact_1').and_return(sparse_contact) + importer = described_class.new(data_import: data_import) + allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:| + raise StandardError, 'mapping failed' if object_type == 'contact' + + method.call(object_type, source_id, record, metadata: metadata) + end + + importer.import_conversations_page + + expect(account.contacts.where(name: 'Customer One')).to be_empty + expect(data_import.mappings.where(source_object_type: 'contact', source_object_id: 'contact_1')).to be_empty + contact_item = data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1') + expect(contact_item).to be_failed + expect(contact_item.last_error_message).to eq('mapping failed') + end + + it 'rolls back a newly inserted message when mapping persistence fails', :aggregate_failures do + importer = described_class.new(data_import: data_import) + allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:| + raise StandardError, 'mapping failed' if object_type == 'message' + + method.call(object_type, source_id, record, metadata: metadata) + end + + importer.import_conversations_page + + conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1') + expect(conversation.messages.where(source_id: 'intercom:conversation:conversation_1:source:source_1')).to be_empty + error = data_import.import_errors.find_by!( + source_object_type: 'message', + source_object_id: 'conversation:conversation_1:source:source_1' + ) + expect(error).to have_attributes(error_code: 'StandardError', message: 'mapping failed') + end + end + + describe '#finish!' do + it 'does not overwrite an import abandoned by another process' do + data_import.update!(status: :processing) + importer = described_class.new(data_import: data_import) + + DataImport.find(data_import.id).update!( + status: :abandoned, + abandoned_at: Time.current + ) + + importer.finish! + + expect(data_import.reload).to be_abandoned + expect(data_import.completed_at).to be_nil + end + end + + describe '#fail!' do + it 'does not overwrite an import abandoned by another process', :aggregate_failures do + data_import.update!(status: :processing) + importer = described_class.new(data_import: data_import) + + DataImport.find(data_import.id).update!( + status: :abandoned, + abandoned_at: Time.current + ) + + importer.fail!(StandardError.new('boom')) + + expect(data_import.reload).to be_abandoned + expect(data_import.last_error_at).to be_nil + expect(data_import.import_errors.exists?).to be(false) + end + end + + context 'when the Intercom records were imported by an earlier run' do + let(:next_data_import) do + create( + :data_import, :intercom, + account: account + ) + end + + it 'records the already mapped records as skipped for the current import run', :aggregate_failures do + described_class.new(data_import: data_import).perform + + described_class.new(data_import: next_data_import).perform + + expect(next_data_import.reload.stats).to include( + 'contacts' => include('imported' => 0, 'skipped' => 1, 'total' => 1), + 'conversations' => include('imported' => 0, 'skipped' => 1, 'total' => 1), + 'messages' => include('imported' => 0, 'skipped' => 3, 'total' => 3), + 'errors' => { 'count' => 0 } + ) + expect(next_data_import).to be_completed + expect(next_data_import.total_records).to eq(5) + expect(next_data_import.processed_records).to eq(0) + expect(next_data_import.items.skipped.count).to eq(2) + expect(next_data_import.import_errors.skip_logs.group(:source_object_type).count).to eq( + 'contact' => 1, + 'conversation' => 1, + 'message' => 3 + ) + expect(next_data_import.import_errors.skip_logs.pluck(:details).map { |details| details['reason'] }.uniq).to eq(['already_imported']) + end + + it 'recreates messages when existing message mappings point to deleted records', :aggregate_failures do + described_class.new(data_import: data_import).perform + conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1') + Message.where(conversation_id: conversation.id).delete_all + + described_class.new(data_import: next_data_import).perform + + expect(conversation.reload.messages.pluck(:source_id)).to match_array( + %w[ + intercom:conversation:conversation_1:source:source_1 + intercom:conversation:conversation_1:part:part_1 + intercom:conversation:conversation_1:part:part_2 + ] + ) + expect(next_data_import.reload.stats).to include( + 'contacts' => include('imported' => 0, 'skipped' => 1, 'total' => 1), + 'conversations' => include('imported' => 0, 'skipped' => 1, 'total' => 1), + 'messages' => include('imported' => 3, 'skipped' => 0, 'total' => 3), + 'errors' => { 'count' => 0 } + ) + expect(next_data_import.import_errors.skip_logs.where(source_object_type: 'message')).to be_empty + message_mappings = DataImportMapping.where(account: account, source_provider: 'intercom', source_object_type: 'message') + expect(message_mappings.filter_map(&:chatwoot_record).count).to eq(3) + end + + it 'updates conversation activity when a later import adds new messages to the mapped conversation', :aggregate_failures do + new_part = { + 'id' => 'part_3', + 'part_type' => 'comment', + 'body' => '

Follow-up reply

', + 'created_at' => 1_700_000_300, + 'updated_at' => 1_700_000_300, + 'author' => { 'type' => 'admin', 'id' => 'admin_1' }, + 'attachments' => [] + } + updated_conversation_payload = conversation_payload.deep_dup + updated_conversation_payload['updated_at'] = 1_700_000_300 + updated_conversation_payload['conversation_parts']['conversation_parts'] << new_part + allow(client).to receive(:retrieve_conversation).with('conversation_1').and_return( + conversation_payload, + updated_conversation_payload + ) + + described_class.new(data_import: data_import).perform + conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1') + + described_class.new(data_import: next_data_import).perform + + expect(conversation.reload.last_activity_at).to eq(Time.zone.at(1_700_000_300)) + expect(conversation.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:part_3').content).to eq('Follow-up reply') + end + end + + context 'when a conversation references an already mapped contact' do + it 'reuses the mapped contact without hydrating the sparse reference' do + described_class.new(data_import: data_import).import_contacts_page + + expect(client).not_to receive(:retrieve_contact) + + described_class.new(data_import: data_import).import_conversations_page + end + end + + context 'when a same-run contact mapping outlives its item progress' do + let!(:mapped_contact) { create(:contact, account: account) } + + before do + DataImportMapping.create!( + account: account, + data_import: data_import, + source_provider: 'intercom', + source_object_type: 'contact', + source_object_id: 'contact_1', + chatwoot_record_type: 'Contact', + chatwoot_record_id: mapped_contact.id, + metadata: {} + ) + data_import.items.create!( + source_provider: 'intercom', + source_object_type: 'contact', + source_object_id: 'contact_1', + status: :processing, + metadata: contact_payload + ) + end + + it 'repairs the item and imported count on retry', :aggregate_failures do + described_class.new(data_import: data_import).import_contacts_page + + item = data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1') + expect(item).to be_imported + expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: mapped_contact.id) + expect(data_import.reload.stats.dig('contacts', 'imported')).to eq(1) + end + end + + context 'when an existing contact has the same email but a different external id' do + let(:contact_payload) do + super().merge('last_replied_at' => 1_700_000_090) + end + let!(:existing_contact) { create(:contact, account: account, email: 'customer@example.com', identifier: nil) } + + it 'updates the existing contact instead of creating a duplicate', :aggregate_failures do + described_class.new(data_import: data_import).import_contacts_page + + expect(existing_contact.reload.identifier).to eq('external_1') + expect(existing_contact.last_activity_at).to eq(Time.zone.at(1_700_000_090)) + expect(account.contacts.where(email: 'customer@example.com').count).to eq(1) + item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1') + expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id) + end + end + + context 'when an existing contact has the same phone but a different external id' do + let(:contact_payload) do + super().merge('email' => nil) + end + let!(:existing_contact) { create(:contact, account: account, phone_number: '+15551234567', identifier: nil) } + + it 'updates the existing contact instead of creating a duplicate', :aggregate_failures do + described_class.new(data_import: data_import).import_contacts_page + + expect(existing_contact.reload.identifier).to eq('external_1') + expect(account.contacts.where(phone_number: '+15551234567').count).to eq(1) + item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1') + expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id) + end + end + + context 'when an existing contact has the same phone but Intercom sends a new email' do + let!(:existing_contact) { create(:contact, account: account, phone_number: '+15551234567', identifier: nil) } + + it 'falls through to the phone match after the email lookup misses', :aggregate_failures do + described_class.new(data_import: data_import).import_contacts_page + + expect(existing_contact.reload.email).to eq('customer@example.com') + expect(existing_contact.identifier).to eq('external_1') + expect(account.contacts.where(phone_number: '+15551234567').count).to eq(1) + item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1') + expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id) + end + end + + context 'when an existing visitor contact matches the Intercom external id' do + let!(:existing_contact) { create(:contact, account: account, identifier: 'external_1') } + + it 'promotes the contact to a lead when adding email or phone', :aggregate_failures do + expect(existing_contact).to be_visitor + + described_class.new(data_import: data_import).import_contacts_page + + expect(existing_contact.reload).to be_lead + expect(existing_contact.email).to eq('customer@example.com') + expect(existing_contact.phone_number).to eq('+15551234567') + end + end + + context 'when an identifier match has contact details owned by another contact' do + let!(:existing_contact) { create(:contact, account: account, identifier: 'external_1') } + let!(:email_owner) { create(:contact, account: account, email: 'customer@example.com') } + let!(:phone_owner) { create(:contact, account: account, phone_number: '+15551234567') } + + it 'does not copy the conflicting email or phone number', :aggregate_failures do + described_class.new(data_import: data_import).import_contacts_page + + expect(existing_contact.reload.email).to be_nil + expect(existing_contact.phone_number).to be_nil + expect(existing_contact).to be_visitor + expect(email_owner.reload.email).to eq('customer@example.com') + expect(phone_owner.reload.phone_number).to eq('+15551234567') + expect(account.contacts.where(email: 'customer@example.com').count).to eq(1) + expect(account.contacts.where(phone_number: '+15551234567').count).to eq(1) + + item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1') + expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id) + end + end + + context 'when Intercom rate limits a conversation detail request' do + before do + allow(client).to receive(:retrieve_conversation).with('conversation_1').and_raise( + DataImports::Intercom::Client::RateLimitError.new('rate limited', status: 429) + ) + end + + it 're-raises the provider error so the page job can retry', :aggregate_failures do + expect { described_class.new(data_import: data_import).import_conversations_page } + .to raise_error(DataImports::Intercom::Client::RateLimitError) + + item = data_import.items.find_by!(source_object_type: 'conversation', source_object_id: 'conversation_1') + expect(item).to be_processing + expect(data_import.import_errors.exists?).to be(false) + end + end + + context 'when Intercom rate limits a contact hydration request' do + before do + allow(client).to receive(:retrieve_contact).with('contact_1').and_raise( + DataImports::Intercom::Client::RateLimitError.new('rate limited', status: 429) + ) + end + + it 're-raises the provider error instead of importing a sparse contact', :aggregate_failures do + expect { described_class.new(data_import: data_import).import_conversations_page } + .to raise_error(DataImports::Intercom::Client::RateLimitError) + + expect(data_import.items.exists?(source_object_type: 'contact')).to be(false) + expect(data_import.import_errors.exists?).to be(false) + end + end + + context 'when Intercom no longer has a sparse contact referenced by a conversation' do + before do + allow(client).to receive(:retrieve_contact).with('contact_1').and_raise( + DataImports::Intercom::Client::Error.new('not found', status: 404) + ) + end + + it 'falls back to the conversation contact reference', :aggregate_failures do + expect { described_class.new(data_import: data_import).import_conversations_page }.not_to raise_error + + expect(data_import.items.imported.exists?(source_object_type: 'contact', source_object_id: 'contact_1')).to be(true) + expect(data_import.import_errors.exists?).to be(false) + end + end + + context 'when the Intercom source message only has attachments' do + let(:conversation_payload) do + super().deep_merge( + 'source' => { + 'subject' => nil, + 'body' => nil, + 'attachments' => [{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }] + }, + 'conversation_parts' => { + 'conversation_parts' => [] + } + ) + end + + it 'imports the source message attachment placeholder', :aggregate_failures do + described_class.new(data_import: data_import).perform + + conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1') + expect(conversation.messages.pluck(:content)).to eq(['[Intercom attachment skipped: 1]']) + expect(conversation.messages.first.additional_attributes.dig('source', 'attachments')).to eq( + [{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }] + ) + expect(data_import.reload.stats.dig('messages', 'imported')).to eq(1) + end + end + + context 'when the Intercom source message has text and attachments' do + let(:conversation_payload) do + super().deep_merge( + 'source' => { + 'attachments' => [{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }] + } + ) + end + + it 'adds an attachment omission marker to the imported message', :aggregate_failures do + described_class.new(data_import: data_import).perform + + message = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:source:source_1') + expect(message.content).to eq("Need help\n\nHello there\n\n[Intercom attachment skipped: 1]") + expect(message.additional_attributes.dig('source', 'attachments')).to eq( + [{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }] + ) + expect(data_import.reload.stats.dig('messages', 'skipped')).to eq(0) + end + end + + context 'when Intercom omits the conversation source' do + let(:conversation_payload) do + super().merge( + 'source' => nil, + 'first_contact_reply' => { + 'type' => 'whatsapp', + 'created_at' => 1_700_000_000, + 'url' => nil + } + ) + end + + it 'routes the conversation from the first contact reply type', :aggregate_failures do + described_class.new(data_import: data_import).perform + + inbox = account.inboxes.find_by!(name: 'Intercom Import - WhatsApp') + conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1') + + expect(conversation.inbox).to eq(inbox) + expect(conversation.additional_attributes.dig('source', 'source_type')).to eq('whatsapp') + end + end + + context 'when an Intercom chat message part cannot be imported' do + let(:conversation_payload) do + super().deep_merge( + 'conversation_parts' => { + 'conversation_parts' => [ + { + 'id' => 'blank_part', + 'part_type' => 'comment', + 'body' => nil, + 'created_at' => 1_700_000_175, + 'updated_at' => 1_700_000_175, + 'author' => { 'type' => 'admin', 'id' => 'admin_1' }, + 'attachments' => [] + } + ] + } + ) + end + + it 'records a skip log with the Intercom message source id', :aggregate_failures do + described_class.new(data_import: data_import).perform + + skip_log = data_import.import_errors.skip_logs.find_by!(source_object_type: 'message') + expect(skip_log).to have_attributes( + source_object_id: 'conversation:conversation_1:part:blank_part', + error_code: 'DataImports::Intercom::SkippedMessage', + message: 'Skipped Intercom comment event blank_part: no message body or attachments to import.' + ) + expect(skip_log.details).to include( + 'kind' => 'skipped', + 'reason' => 'blank_or_unsupported_intercom_part', + 'reason_details' => 'no message body or attachments to import', + 'event_name' => 'comment', + 'event_type' => 'comment', + 'author_type' => 'admin' + ) + expect(data_import.reload.stats.dig('messages', 'skipped')).to eq(1) + end + + it 'records the skip log again for a later import run', :aggregate_failures do + described_class.new(data_import: data_import).perform + next_data_import = create( + :data_import, :intercom, + account: account + ) + + described_class.new(data_import: next_data_import).perform + + skip_log = next_data_import.import_errors.skip_logs.find_by!( + source_object_type: 'message', + source_object_id: 'conversation:conversation_1:part:blank_part', + error_code: 'DataImports::Intercom::SkippedMessage' + ) + expect(skip_log).to have_attributes( + source_object_id: 'conversation:conversation_1:part:blank_part', + error_code: 'DataImports::Intercom::SkippedMessage' + ) + expect(next_data_import.reload.stats.dig('messages', 'skipped')).to eq(2) + end + + it 'reconciles a same-run skipped mapping and missing skip log on retry', :aggregate_failures do + described_class.new(data_import: data_import).import_conversations_page + data_import.import_errors.where(source_object_type: 'message').delete_all + stats = data_import.reload.stats.deep_dup + stats['messages']['skipped'] = 0 + data_import.update!(stats: stats) + + described_class.new(data_import: data_import).import_conversations_page + + expect(data_import.reload.stats.dig('messages', 'skipped')).to eq(1) + expect(data_import.import_errors.skip_logs.exists?(source_object_id: 'conversation:conversation_1:part:blank_part')).to be(true) + end + + it 'repairs a previously skipped mapping when the part is now an activity', :aggregate_failures do + described_class.new(data_import: data_import).perform + previous_skip_log = data_import.import_errors.skip_logs.find_by!(source_object_id: 'conversation:conversation_1:part:blank_part') + conversation_payload.dig('conversation_parts', 'conversation_parts').first.merge!( + 'part_type' => 'assignment', + 'assigned_to' => { 'name' => 'Support' } + ) + next_data_import = create(:data_import, :intercom, account: account) + + described_class.new(data_import: next_data_import).perform + + activity = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:blank_part') + mapping = DataImportMapping.find_by!( + account: account, + source_provider: 'intercom', + source_object_type: 'message', + source_object_id: 'conversation:conversation_1:part:blank_part' + ) + expect(activity).to be_activity + expect(activity.content).to eq('Intercom teammate assigned the conversation to Support') + expect(mapping.chatwoot_record).to eq(activity) + expect(data_import.import_errors.skip_logs).to include(previous_skip_log) + expect(next_data_import.import_errors.skip_logs.where(source_object_id: mapping.source_object_id)).to be_empty + end + end + + context 'when Intercom returns bodyless lifecycle events' do + let(:conversation_payload) do + super().deep_merge( + 'conversation_parts' => { + 'total_count' => 1, + 'conversation_parts' => [ + { + 'id' => 'assignment_part', + 'part_type' => 'assignment', + 'body' => nil, + 'created_at' => 1_700_000_175, + 'author' => { 'type' => 'admin', 'name' => 'Avery' }, + 'assigned_to' => { 'type' => 'team', 'name' => 'Support' }, + 'state' => 'open', + 'tags' => { 'tags' => [{ 'name' => 'priority' }] }, + 'event_details' => { 'source' => 'workflow' }, + 'app_package_code' => 'workflow' + } + ] + } + ) + end + + it 'imports events as public activity messages with source metadata', :aggregate_failures do + described_class.new(data_import: data_import).perform + + activity = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:assignment_part') + expect(activity).to have_attributes( + message_type: 'activity', + content: 'Avery assigned the conversation to Support', + private: false, + sender: nil, + created_at: Time.zone.at(1_700_000_175) + ) + expect(activity.additional_attributes['source']).to include( + 'part_type' => 'assignment', + 'assigned_to' => include('name' => 'Support'), + 'state' => 'open', + 'event_details' => include('source' => 'workflow'), + 'app_package_code' => 'workflow' + ) + expect(data_import.reload.stats['messages']).to include('imported' => 2, 'skipped' => 0, 'total' => 2) + expect(data_import.import_errors.skip_logs).to be_empty + end + end + + context 'when Intercom omits older conversation parts from the retrieved conversation' do + let(:conversation_payload) do + super().deep_merge( + 'conversation_parts' => { + 'total_count' => 503 + }, + 'statistics' => { + 'count_conversation_parts' => 503 + } + ) + end + + it 'records an incomplete import error and completes with errors', :aggregate_failures do + described_class.new(data_import: data_import).perform + + error = data_import.import_errors.non_skip_logs.find_by!( + source_object_type: 'conversation', + source_object_id: 'conversation_1', + error_code: 'DataImports::Intercom::TruncatedConversationParts' + ) + expect(error.message).to eq('Intercom returned 2 of 503 conversation parts.') + expect(error.details).to include( + 'kind' => 'incomplete', + 'imported_parts_count' => 2, + 'total_parts_count' => 503 + ) + expect(data_import.reload).to be_completed_with_errors + expect(data_import.stats.dig('errors', 'count')).to eq(1) + end + end + + context 'when the conversation parts total matches the returned parts' do + let(:conversation_payload) do + super().deep_merge( + 'conversation_parts' => { + 'total_count' => 2 + }, + 'statistics' => { + 'count_conversation_parts' => 2 + } + ) + end + + it 'does not record a truncated parts error', :aggregate_failures do + described_class.new(data_import: data_import).perform + + expect(data_import.import_errors.non_skip_logs).to be_empty + expect(data_import.reload).to be_completed + expect(data_import.stats.dig('errors', 'count')).to eq(0) + end + end + + context 'when Intercom statistics count is higher than the conversation parts total' do + let(:conversation_payload) do + super().deep_merge( + 'source' => {}, + 'conversation_parts' => { + 'total_count' => 2 + }, + 'statistics' => { + 'count_conversation_parts' => 3 + } + ) + end + + it 'trusts the returned conversation parts total over the statistics counter', :aggregate_failures do + described_class.new(data_import: data_import).perform + + expect(data_import.import_errors.non_skip_logs).to be_empty + expect(data_import.reload).to be_completed + expect(data_import.stats.dig('errors', 'count')).to eq(0) + end + end + + context 'when a specific Intercom message part fails to persist' do + let(:conversation_payload) do + super().deep_merge( + 'conversation_parts' => { + 'conversation_parts' => [ + { + 'id' => 'bad_part', + 'part_type' => 'comment', + 'body' => '

Message that cannot be stored

', + 'created_at' => 1_700_000_175, + 'updated_at' => 1_700_000_175, + 'author' => { 'type' => 'admin', 'id' => 'admin_1' }, + 'attachments' => [] + } + ] + } + ) + end + + before do + allow(Message).to receive(:insert_all!).and_wrap_original do |method, records, **kwargs| + raise ActiveRecord::StatementInvalid, 'bad message' if records.first[:source_id] == 'intercom:conversation:conversation_1:part:bad_part' + + method.call(records, **kwargs) + end + end + + it 'records a skip log with the Intercom message part id', :aggregate_failures do + described_class.new(data_import: data_import).perform + + skip_log = data_import.import_errors.skip_logs.find_by!(source_object_type: 'message') + expect(skip_log).to have_attributes( + source_object_id: 'conversation:conversation_1:part:bad_part', + error_code: 'ActiveRecord::StatementInvalid', + message: 'bad message' + ) + expect(skip_log.details).to include( + 'kind' => 'failed', + 'conversation_id' => 'intercom:conversation_1' + ) + expect(data_import.reload).to be_completed_with_errors + expect(data_import.stats.dig('errors', 'count')).to eq(1) + end + end +end diff --git a/spec/services/data_imports/intercom/placeholder_inbox_builder_spec.rb b/spec/services/data_imports/intercom/placeholder_inbox_builder_spec.rb new file mode 100644 index 000000000..c1a7baab5 --- /dev/null +++ b/spec/services/data_imports/intercom/placeholder_inbox_builder_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::PlaceholderInboxBuilder do + let(:account) { create(:account) } + + describe '#inbox_for' do + it 'creates a source-bucket API inbox for an Intercom conversation source' do + inbox = described_class.new(account: account).inbox_for('email') + + expect(inbox.name).to eq('Intercom Import - Email') + expect(inbox.channel).to be_a(Channel::Api) + expect(inbox.enable_auto_assignment).to be(false) + expect(inbox.allow_messages_after_resolved).to be(false) + expect(inbox.channel.additional_attributes).to include( + 'source_provider' => 'intercom', + 'source_bucket' => 'email', + 'import_placeholder' => true, + 'agent_reply_time_window' => 1 + ) + end + + it 'reuses an existing placeholder inbox for the same source bucket' do + builder = described_class.new(account: account) + + first_inbox = builder.inbox_for('phone_call') + expect(account).not_to receive(:inboxes) + second_inbox = builder.inbox_for('phone_switch') + + expect(second_inbox).to eq(first_inbox) + expect(Inbox.where(account: account, channel_type: 'Channel::Api').count).to eq(1) + end + end +end diff --git a/spec/services/data_imports/intercom/restart_service_spec.rb b/spec/services/data_imports/intercom/restart_service_spec.rb new file mode 100644 index 000000000..5dbc8f5ab --- /dev/null +++ b/spec/services/data_imports/intercom/restart_service_spec.rb @@ -0,0 +1,64 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::RestartService do + let(:account) { create(:account) } + let(:data_import) { create(:data_import, :intercom, account: account, status: :abandoned, abandoned_at: 1.hour.ago) } + + it 'prepares a failed or abandoned import for another run', :aggregate_failures do + data_import.update!( + stats: { + 'contacts' => { 'imported' => 1, 'skipped' => 9, 'total' => 10 }, + 'conversations' => { 'imported' => 2, 'skipped' => 8, 'total' => 10 }, + 'messages' => { 'imported' => 3, 'skipped' => 7, 'total' => 10 }, + 'errors' => { 'count' => 6 } + } + ) + data_import.import_errors.create!(error_code: 'StandardError', message: 'old run error') + data_import.import_errors.create!( + error_code: 'ContactFailed', + message: 'old contact error', + source_object_type: 'contact', + details: { kind: 'failed' } + ) + retained_skip_log = data_import.import_errors.create!( + error_code: DataImports::Intercom::Importer::ALREADY_IMPORTED_ERROR_CODE, + message: 'old skip log', + source_object_type: 'contact', + details: { kind: 'skipped' } + ) + previous_run_id = data_import.assign_active_intercom_import_run_id + data_import.save! + service = described_class.new(account: account, data_import: data_import) + + expect(service.perform).to eq(:enqueue) + expect(service.data_import).to be_pending + expect(service.data_import.abandoned_at).to be_nil + expect(service.data_import.started_at).to be_nil + expect(service.data_import.active_intercom_import_run_id).not_to eq(previous_run_id) + expect(service.data_import.import_errors).to contain_exactly(retained_skip_log) + expect(service.data_import.stats).to eq( + 'contacts' => { 'imported' => 1, 'skipped' => 1, 'total' => 10 }, + 'conversations' => { 'imported' => 2, 'skipped' => 0, 'total' => 10 }, + 'messages' => { 'imported' => 3, 'skipped' => 0, 'total' => 10 }, + 'errors' => { 'count' => 0 } + ) + end + + it 'returns the active import instead of restarting another import', :aggregate_failures do + active_import = create(:data_import, :intercom, account: account, status: :processing) + service = described_class.new(account: account, data_import: data_import) + + expect(service.perform).to eq(:render_show) + expect(service.data_import).to eq(active_import) + expect(data_import.reload).to be_abandoned + end + + it 'does not restart when the stored access token is missing' do + data_import.update!(access_token: nil) + + result = described_class.new(account: account, data_import: data_import).perform + + expect(result).to eq(:access_token_missing) + expect(data_import.reload).to be_abandoned + end +end diff --git a/spec/services/data_imports/intercom/source_bucket_spec.rb b/spec/services/data_imports/intercom/source_bucket_spec.rb new file mode 100644 index 000000000..b3db4294c --- /dev/null +++ b/spec/services/data_imports/intercom/source_bucket_spec.rb @@ -0,0 +1,17 @@ +require 'rails_helper' + +RSpec.describe DataImports::Intercom::SourceBucket do + describe '.for' do + it 'maps Intercom source types to Chatwoot inbox buckets' do + expect(described_class.for('email')).to eq({ key: 'email', name: 'Email' }) + expect(described_class.for('phone_switch')).to eq({ key: 'phone', name: 'Phone' }) + expect(described_class.for('inapp')).to eq({ key: 'messenger', name: 'Messenger' }) + expect(described_class.for('messenger')).to eq({ key: 'messenger', name: 'Messenger' }) + expect(described_class.for('push')).to eq({ key: 'messenger', name: 'Messenger' }) + end + + it 'uses an unknown bucket for unsupported source types' do + expect(described_class.for('unsupported_source')).to eq({ key: 'unknown', name: 'Unknown' }) + end + end +end