perf: shorten Intercom import jobs (#15051)

## Description

Reduces sustained load during large Intercom imports by limiting
conversation list pages to 10 while retaining 50-contact pages. Cursor
and provider total-count behavior remain unchanged.

The one-minute progress heartbeat now lives in parent PR #15050 because
stalled-import detection must be safe when that PR is deployed
independently. This child PR therefore contains only the smaller-page
delta.

This is Phase 1, Task 2 of the [Intercom import optimization plan
(CW-7615)](https://linear.app/chatwoot/issue/CW-7615/optimize-intercom-import-reliability-and-bulk-message-ingestion).

This PR is stacked on #15050 and should be reviewed as the two-file
delta from `codex/cw-7519-intercom-stalled-retry-15m`.

## Closes

-
[CW-7615](https://linear.app/chatwoot/issue/CW-7615/optimize-intercom-import-reliability-and-bulk-message-ingestion)

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)

## How to test

1. Start an Intercom import containing contacts and conversations across
multiple source pages.
2. Confirm contact list requests retain a page size of 50.
3. Confirm conversation list requests use a page size of 10.
4. Confirm page cursors and displayed provider totals continue advancing
normally.
5. Confirm the parent PR heartbeat keeps long conversations fresh while
these shorter pages are processed.

## Checklist

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have added tests that prove the change is effective
- [x] New and existing focused tests pass locally with my changes

VERSION_CW

@@ -1 +1 @@
-4.16.1
+4.16.0

app/controllers/api/v1/accounts/data_imports_controller.rb

@@ -4,7 +4,7 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll
   DATA_IMPORT_FEATURE = 'data_import'.freeze
 
   before_action :ensure_data_import_feature_enabled
-  before_action :set_data_import, only: [:show, :start, :abandon, :error_logs, :skip_logs]
+  before_action :set_data_import, only: [:show, :start, :retry_import, :abandon, :error_logs, :skip_logs]
   before_action :check_authorization
 
   def index
@@ -59,6 +59,24 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll
     render_show
   end
 
+  def retry_import
+    retry_service = DataImports::Intercom::RetryService.new(account: Current.account, data_import: @data_import)
+    retry_result = retry_service.perform
+    @data_import = retry_service.data_import
+
+    case retry_result
+    when :enqueue
+      DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id)
+      render_show
+    when :not_stalled
+      render json: { message: 'This Intercom import is no longer stalled.' }, status: :unprocessable_entity
+    when :active_import_exists
+      render json: { message: 'Another Intercom import is already in progress.' }, status: :unprocessable_entity
+    when :access_token_missing
+      render json: { message: 'The Intercom access key for this import is unavailable.' }, status: :unprocessable_entity
+    end
+  end
+
   def abandon
     @data_import.abandon!
     render_show

app/javascript/dashboard/api/dataImports.js

@@ -11,6 +11,10 @@ class DataImportsAPI extends ApiClient {
     return axios.post(`${this.url}/${id}/start`);
   }
 
+  retry(id) {
+    return axios.post(`${this.url}/${id}/retry`);
+  }
+
   abandon(id) {
     return axios.post(`${this.url}/${id}/abandon`);
   }

app/javascript/dashboard/components/ui/ContextMenu.vue

@@ -77,15 +77,6 @@ const handleClose = () => {
   emit('close');
 };
 
-const handleFocusOut = event => {
-  // Keep the menu open while focus stays inside it (e.g. the label search
-  // input); close it once focus leaves the menu entirely.
-  if (menuRef.value?.contains(event.relatedTarget)) {
-    return;
-  }
-  handleClose();
-};
-
 onUnmounted(() => {
   isLocked.value = false;
 });
@@ -98,7 +89,7 @@ onUnmounted(() => {
       class="fixed outline-none z-[9999] cursor-pointer"
       :style="position"
       tabindex="0"
-      @focusout="handleFocusOut"
+      @blur="handleClose"
     >
       <slot />
     </div>

app/javascript/dashboard/components/widgets/conversation/MessagesView.vue

@@ -33,7 +33,9 @@ import {
 // constants
 import { BUS_EVENTS } from 'shared/constants/busEvents';
 import { REPLY_POLICY } from 'shared/constants/links';
-import wootConstants from 'dashboard/constants/globals';
+import wootConstants, {
+  META_RESTRICTION_STATUS_URL,
+} from 'dashboard/constants/globals';
 import { LOCAL_STORAGE_KEYS } from 'dashboard/constants/localStorage';
 import { INBOX_TYPES } from 'dashboard/helper/inbox';
 
@@ -93,6 +95,7 @@ export default {
       currentUserId: 'getCurrentUserID',
       listLoadingStatus: 'getAllMessagesLoaded',
       currentAccountId: 'getCurrentAccountId',
+      isOnChatwootCloud: 'globalConfig/isOnChatwootCloud',
     }),
     isOpen() {
       return this.currentChat?.status === wootConstants.STATUS_TYPE.OPEN;
@@ -170,6 +173,13 @@ export default {
         instagramInbox
       );
     },
+    isInstagramRestrictionBannerVisible() {
+      return this.isOnChatwootCloud && this.isAnInstagramChannel;
+    },
+    instagramRestrictionStatusUrl() {
+      return META_RESTRICTION_STATUS_URL;
+    },
+
     replyWindowBannerMessage() {
       if (this.isAWhatsAppChannel) {
         return this.$t('CONVERSATION.TWILIO_WHATSAPP_CAN_REPLY');
@@ -454,7 +464,15 @@ export default {
   >
     <div ref="topBannerRef">
       <Banner
-        v-if="!currentChat.can_reply"
+        v-if="isInstagramRestrictionBannerVisible"
+        color-scheme="warning"
+        class="mx-2 mt-2 overflow-hidden rounded-lg"
+        :banner-message="$t('CONVERSATION.INSTAGRAM_RESTRICTION_BANNER')"
+        :href-link="instagramRestrictionStatusUrl"
+        :href-link-text="$t('CONVERSATION.INSTAGRAM_RESTRICTION_STATUS_LINK')"
+      />
+      <Banner
+        v-else-if="!currentChat.can_reply"
         color-scheme="alert"
         class="mx-2 mt-2 overflow-hidden rounded-lg"
         :banner-message="replyWindowBannerMessage"

app/javascript/dashboard/components/widgets/conversation/contextMenu/Index.vue

@@ -7,13 +7,10 @@ import {
   getSortedAgentsByAvailability,
   getAgentsByUpdatedPresence,
 } from 'dashboard/helper/agentHelper.js';
-import { picoSearch } from '@scmmishra/pico-search';
 import MenuItem from './menuItem.vue';
 import MenuItemWithSubmenu from './menuItemWithSubmenu.vue';
 import wootConstants from 'dashboard/constants/globals';
 import AgentLoadingPlaceholder from './agentLoadingPlaceholder.vue';
-import NextInput from 'dashboard/components-next/input/Input.vue';
-import Icon from 'dashboard/components-next/icon/Icon.vue';
 
 const MENU = {
   MARK_AS_READ: 'mark-as-read',
@@ -34,8 +31,6 @@ export default {
     MenuItem,
     MenuItemWithSubmenu,
     AgentLoadingPlaceholder,
-    NextInput,
-    Icon,
   },
   props: {
     chatId: {
@@ -92,7 +87,6 @@ export default {
   data() {
     return {
       MENU,
-      labelSearchQuery: '',
       STATUS_TYPE: wootConstants.STATUS_TYPE,
       readOption: {
         label: this.$t('CONVERSATION.CARD_CONTEXT_MENU.MARK_AS_READ'),
@@ -222,14 +216,6 @@ export default {
       // Don't show snooze if the conversation is already snoozed/resolved/pending
       return this.status === wootConstants.STATUS_TYPE.OPEN;
     },
-    filteredLabels() {
-      const labels = this.labelSearchQuery
-        ? picoSearch(this.labels, this.labelSearchQuery, ['title'])
-        : this.labels;
-      // Assigned labels first, keeping each group's existing order.
-      const isAssigned = label => this.conversationLabels.includes(label.title);
-      return [...labels].sort((a, b) => isAssigned(b) - isAssigned(a));
-    },
   },
   mounted() {
     this.$store.dispatch('inboxAssignableAgents/fetch', [this.inboxId]);
@@ -349,49 +335,21 @@ export default {
         :option="labelMenuConfig"
         :sub-menu-available="!!labels.length"
       >
-        <div class="pb-1 w-[12.5rem]">
-          <NextInput
-            v-model="labelSearchQuery"
-            type="search"
-            size="sm"
-            class="w-full"
-            custom-input-class="!ps-8 !text-xs"
-            :placeholder="$t('CONVERSATION.CARD_CONTEXT_MENU.SEARCH_LABELS')"
-            @click.stop
-            @keydown.stop
-          >
-            <template #prefix>
-              <Icon
-                icon="i-lucide-search"
-                class="absolute z-10 -translate-y-1/2 pointer-events-none size-3.5 text-n-slate-10 top-1/2 start-2"
-              />
-            </template>
-          </NextInput>
-        </div>
-        <div class="overflow-x-hidden overflow-y-auto max-h-[12.5rem]">
-          <MenuItem
-            v-for="label in filteredLabels"
-            :key="label.id"
-            :option="generateMenuLabelConfig(label, 'label')"
-            :variant="
-              conversationLabels.includes(label.title)
-                ? 'label-assigned'
-                : 'label'
-            "
-            @mousedown.prevent
-            @click.stop="
-              conversationLabels.includes(label.title)
-                ? $emit('removeLabel', label)
-                : $emit('assignLabel', label)
-            "
-          />
-          <p
-            v-if="!filteredLabels.length"
-            class="px-2 py-2 m-0 text-xs text-center text-n-slate-11"
-          >
-            {{ $t('CONVERSATION.CARD_CONTEXT_MENU.NO_LABELS_FOUND') }}
-          </p>
-        </div>
+        <MenuItem
+          v-for="label in labels"
+          :key="label.id"
+          :option="generateMenuLabelConfig(label, 'label')"
+          :variant="
+            conversationLabels.includes(label.title)
+              ? 'label-assigned'
+              : 'label'
+          "
+          @click.stop="
+            conversationLabels.includes(label.title)
+              ? $emit('removeLabel', label)
+              : $emit('assignLabel', label)
+          "
+        />
       </MenuItemWithSubmenu>
       <MenuItemWithSubmenu
         v-if="isAllowed([MENU.AGENT])"

app/javascript/dashboard/components/widgets/conversation/contextMenu/menuItem.vue

@@ -15,7 +15,7 @@ defineProps({
 </script>
 
 <template>
-  <div class="menu group text-n-slate-12 min-h-7 min-w-0" role="button">
+  <div class="menu text-n-slate-12 min-h-7 min-w-0" role="button">
     <fluent-icon
       v-if="variant === 'icon' && option.icon"
       :icon="option.icon"
@@ -52,7 +52,7 @@ defineProps({
     <Icon
       v-if="variant === 'label-assigned'"
       icon="i-lucide-check"
-      class="flex-shrink-0 size-3.5 text-n-brand group-hover:text-white"
+      class="flex-shrink-0 size-3.5 mr-1"
     />
   </div>
 </template>

app/javascript/dashboard/constants/globals.js

@@ -78,3 +78,5 @@ export default {
   },
 };
 export const DEFAULT_REDIRECT_URL = '/app/';
+export const META_RESTRICTION_STATUS_URL =
+  'https://status.chatwoot.com/incident/948346';

app/javascript/dashboard/i18n/locale/en/conversation.json

@@ -44,6 +44,8 @@
     "TWILIO_WHATSAPP_CAN_REPLY": "You can only reply to this conversation using a template message due to",
     "TWILIO_WHATSAPP_24_HOURS_WINDOW": "24 hour message window restriction",
     "OLD_INSTAGRAM_INBOX_REPLY_BANNER": "This Instagram account was migrated to the new Instagram channel inbox. All new messages will show up there. You won’t be able to send messages from this conversation anymore.",
+    "INSTAGRAM_RESTRICTION_BANNER": "Instagram is currently restricted. Some messages or actions may be delayed or unavailable while we restore full support.",
+    "INSTAGRAM_RESTRICTION_STATUS_LINK": "View status update",
     "REPLYING_TO": "You are replying to:",
     "REMOVE_SELECTION": "Remove Selection",
     "DOWNLOAD": "Download",
@@ -193,8 +195,6 @@
       },
       "ASSIGN_AGENT": "Assign agent",
       "ASSIGN_LABEL": "Assign label",
-      "SEARCH_LABELS": "Search labels",
-      "NO_LABELS_FOUND": "No labels found",
       "AGENTS_LOADING": "Loading agents...",
       "ASSIGN_TEAM": "Assign team",
       "DELETE": "Delete conversation",

app/javascript/dashboard/i18n/locale/en/inboxMgmt.json

@@ -58,7 +58,9 @@
         "ERROR_MESSAGE": "There was an error connecting to Instagram, please try again",
         "ERROR_AUTH": "There was an error connecting to Instagram, please try again",
         "NEW_INBOX_SUGGESTION": "This Instagram account was previously linked to a different inbox and has now been migrated here. All new messages will appear here. The old inbox will no longer be able to send or receive messages for this account.",
-        "DUPLICATE_INBOX_BANNER": "This Instagram account was migrated to the new Instagram channel inbox. You won’t be able to send/receive Instagram messages from this inbox anymore."
+        "DUPLICATE_INBOX_BANNER": "This Instagram account was migrated to the new Instagram channel inbox. You won’t be able to send/receive Instagram messages from this inbox anymore.",
+        "SETTINGS_RESTRICTED_WARNING": "Instagram is currently restricted. Some messages or actions may be delayed or unavailable while we restore full support.",
+        "STATUS_LINK": "View status update"
       },
       "TIKTOK": {
         "CONTINUE_WITH_TIKTOK": "Continue with TikTok",

app/javascript/dashboard/i18n/locale/en/settings.json

@@ -462,6 +462,7 @@
       "STATUS": "Status",
       "IMPORTED": "Imported",
       "CREATED": "Created",
+      "RETRY": "Retry",
       "ABANDON": "Abandon"
     },
     "DETAIL": {
@@ -508,6 +509,8 @@
     },
     "ALERTS": {
       "IMPORT_STARTED": "Intercom import has started.",
+      "IMPORT_RETRIED": "Intercom import has been queued to resume.",
+      "IMPORT_RETRY_FAILED": "Could not retry the Intercom import.",
       "IMPORT_ABANDONED": "Intercom import has been abandoned.",
       "IMPORT_FAILED": "Could not start the Intercom import."
     }

app/javascript/dashboard/routes/dashboard/settings/data/Show.vue

@@ -26,6 +26,7 @@ const dataImport = ref(null);
 const isLoading = ref(true);
 const isRefreshing = ref(false);
 const isPolling = ref(false);
+const isRetrying = ref(false);
 const isAbandoning = ref(false);
 const isDownloadingErrorLogs = ref(false);
 const isDownloadingSkipLogs = ref(false);
@@ -35,6 +36,7 @@ const errorsOpen = ref(true);
 const skipLogsOpen = ref(true);
 let pollTimer;
 let isPageActive = false;
+let importRequestVersion = 0;
 
 const hasActiveImport = computed(() => isActiveImport(dataImport.value));
 
@@ -50,6 +52,7 @@ const fetchImport = async ({
   manual = false,
   requestedSkipLogsType = selectedSkipLogsType.value,
 } = {}) => {
+  const requestVersion = importRequestVersion;
   if (showLoader) {
     isLoading.value = true;
   } else if (manual) {
@@ -60,6 +63,8 @@ const fetchImport = async ({
     const response = await DataImportsAPI.show(route.params.dataImportId, {
       skip_logs_type: requestedSkipLogsType || undefined,
     });
+    if (requestVersion !== importRequestVersion) return;
+
     dataImport.value = response.data;
     selectedSkipLogsType.value =
       response.data.skip_logs_filters?.selected_source_object_type ||
@@ -105,6 +110,13 @@ const refreshImportInBackground = async () => {
   }
 };
 
+const startPolling = () => {
+  stopPolling();
+  if (!isPageActive || !hasActiveImport.value) return;
+
+  pollTimer = window.setInterval(refreshImportInBackground, POLL_INTERVAL_MS);
+};
+
 const abandonImport = async () => {
   isAbandoning.value = true;
   try {
@@ -117,6 +129,22 @@ const abandonImport = async () => {
   }
 };
 
+const retryImport = async () => {
+  isRetrying.value = true;
+  importRequestVersion += 1;
+  stopPolling();
+  try {
+    const response = await DataImportsAPI.retry(dataImport.value.id);
+    dataImport.value = response.data;
+    useAlert(t('DATA_IMPORTS.ALERTS.IMPORT_RETRIED'));
+  } catch {
+    useAlert(t('DATA_IMPORTS.ALERTS.IMPORT_RETRY_FAILED'));
+  } finally {
+    isRetrying.value = false;
+    if (hasActiveImport.value) startPolling();
+  }
+};
+
 const downloadCsv = (response, filename) => {
   const url = window.URL.createObjectURL(
     new Blob([response.data], { type: 'text/csv' })
@@ -150,13 +178,6 @@ const downloadSkipLogs = async () => {
   }
 };
 
-const startPolling = () => {
-  stopPolling();
-  if (!isPageActive || !hasActiveImport.value) return;
-
-  pollTimer = window.setInterval(refreshImportInBackground, POLL_INTERVAL_MS);
-};
-
 const handleVisibilityChange = () => {
   if (isPageActive && !document.hidden && hasActiveImport.value) {
     refreshImportInBackground();
@@ -197,9 +218,11 @@ onBeforeUnmount(() => {
       <ImportDetailHeader
         :data-import="dataImport"
         :is-refreshing="isRefreshing"
+        :is-retrying="isRetrying"
         :is-abandoning="isAbandoning"
         :is-polling="isPolling"
         @refresh="fetchImport({ manual: true })"
+        @retry="retryImport"
         @abandon="abandonImport"
       />
     </template>

app/javascript/dashboard/routes/dashboard/settings/data/components/ImportDetailHeader.vue

@@ -21,6 +21,10 @@ const props = defineProps({
     type: Boolean,
     default: false,
   },
+  isRetrying: {
+    type: Boolean,
+    default: false,
+  },
   isAbandoning: {
     type: Boolean,
     default: false,
@@ -31,7 +35,7 @@ const props = defineProps({
   },
 });
 
-defineEmits(['refresh', 'abandon']);
+defineEmits(['refresh', 'retry', 'abandon']);
 
 const { t } = useI18n();
 
@@ -90,11 +94,23 @@ const canAbandonImport = computed(() => isAbandonableImport(props.dataImport));
             :title="$t('DATA_IMPORTS.MONITOR.REFRESH')"
             @click="$emit('refresh')"
           />
+          <Button
+            v-if="dataImport?.stalled"
+            outline
+            slate
+            size="sm"
+            icon="i-lucide-rotate-ccw"
+            :is-loading="isRetrying"
+            :disabled="isAbandoning"
+            :label="$t('DATA_IMPORTS.TABLE.RETRY')"
+            @click="$emit('retry')"
+          />
           <Button
             v-if="canAbandonImport"
             ruby
             size="sm"
             :is-loading="isAbandoning"
+            :disabled="isRetrying"
             :label="$t('DATA_IMPORTS.TABLE.ABANDON')"
             @click="$emit('abandon')"
           />

app/javascript/dashboard/routes/dashboard/settings/data/specs/ImportDetailHeader.spec.js

@@ -0,0 +1,85 @@
+import { mount } from '@vue/test-utils';
+import ImportDetailHeader from '../components/ImportDetailHeader.vue';
+
+vi.mock('vue-i18n', () => ({
+  useI18n: () => ({ t: key => key }),
+}));
+
+const ButtonStub = {
+  name: 'Button',
+  props: {
+    label: { type: String, default: '' },
+    icon: { type: String, default: '' },
+    isLoading: { type: Boolean, default: false },
+  },
+  emits: ['click'],
+  template: `
+    <button
+      :data-label="label"
+      :data-icon="icon"
+      :data-loading="isLoading"
+      @click="$emit('click')"
+    >
+      {{ label }}
+    </button>
+  `,
+};
+
+const BaseSettingsHeaderStub = {
+  template: `
+    <section>
+      <slot name="title" />
+      <slot name="description" />
+    </section>
+  `,
+};
+
+const mountHeader = props =>
+  mount(ImportDetailHeader, {
+    props,
+    global: {
+      stubs: {
+        Button: ButtonStub,
+        BaseSettingsHeader: BaseSettingsHeaderStub,
+      },
+      mocks: {
+        $t: key => key,
+      },
+    },
+  });
+
+describe('ImportDetailHeader', () => {
+  const activeImport = {
+    id: 1,
+    name: 'Intercom import',
+    data_type: 'intercom',
+    source_provider: 'intercom',
+    status: 'processing',
+    stalled: true,
+  };
+
+  it('shows Retry between Refresh and Abandon for stalled imports', async () => {
+    const wrapper = mountHeader({ dataImport: activeImport });
+    const buttons = wrapper.findAll('button');
+
+    expect(buttons.map(button => button.attributes('data-label'))).toEqual([
+      '',
+      'DATA_IMPORTS.TABLE.RETRY',
+      'DATA_IMPORTS.TABLE.ABANDON',
+    ]);
+
+    await buttons[1].trigger('click');
+
+    expect(wrapper.emitted('retry')).toHaveLength(1);
+  });
+
+  it('hides Retry when the server does not report the import as stalled', () => {
+    const wrapper = mountHeader({
+      dataImport: { ...activeImport, stalled: false },
+    });
+
+    expect(
+      wrapper.find('[data-label="DATA_IMPORTS.TABLE.RETRY"]').exists()
+    ).toBe(false);
+  });
+});

app/javascript/dashboard/routes/dashboard/settings/data/specs/showActions.spec.js

@@ -0,0 +1,161 @@
+import { flushPromises, mount } from '@vue/test-utils';
+import { KeepAlive, defineComponent, h, nextTick } from 'vue';
+import { useAlert } from 'dashboard/composables';
+import DataImportsAPI from 'dashboard/api/dataImports';
+import Show from '../Show.vue';
+import { POLL_INTERVAL_MS } from '../importStatus';
+
+vi.mock('dashboard/api/dataImports', () => ({
+  default: {
+    show: vi.fn(),
+    retry: vi.fn(),
+  },
+}));
+
+vi.mock('dashboard/composables', () => ({
+  useAlert: vi.fn(),
+}));
+
+vi.mock('vue-i18n', () => ({
+  useI18n: () => ({ t: key => key }),
+}));
+
+vi.mock('vue-router', async importOriginal => ({
+  ...(await importOriginal()),
+  useRoute: () => ({ params: { dataImportId: 1 } }),
+}));
+
+const SettingsLayoutStub = {
+  template: `
+    <main>
+      <slot name="header" />
+      <slot name="body" />
+    </main>
+  `,
+};
+
+const ImportDetailHeaderStub = {
+  name: 'ImportDetailHeader',
+  props: {
+    dataImport: {
+      type: Object,
+      default: null,
+    },
+  },
+  emits: ['retry'],
+  template: `
+    <button
+      v-if="dataImport?.stalled"
+      data-test="retry"
+      @click="$emit('retry')"
+    />
+  `,
+};
+
+const deferredRequest = () => {
+  let resolve;
+  const promise = new Promise(resolvePromise => {
+    resolve = resolvePromise;
+  });
+  return { promise, resolve };
+};
+
+const mountShow = () => {
+  const Host = defineComponent({
+    render() {
+      return h(KeepAlive, null, { default: () => h(Show) });
+    },
+  });
+
+  return mount(Host, {
+    global: {
+      stubs: {
+        SettingsLayout: SettingsLayoutStub,
+        ImportDetailHeader: ImportDetailHeaderStub,
+        ImportSummaryTiles: true,
+        ImportProgress: true,
+        ImportErrorsSection: true,
+        ImportSkipLogsSection: true,
+      },
+      mocks: {
+        $t: key => key,
+      },
+    },
+  });
+};
+
+describe('data import detail actions', () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+    DataImportsAPI.show.mockResolvedValue({
+      data: {
+        id: 1,
+        status: 'processing',
+        stalled: true,
+        skip_logs_filters: {},
+      },
+    });
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.clearAllMocks();
+  });
+
+  it('retries a stalled import and replaces the page state', async () => {
+    DataImportsAPI.retry.mockResolvedValue({
+      data: {
+        id: 1,
+        status: 'pending',
+        stalled: false,
+        skip_logs_filters: {},
+      },
+    });
+    const wrapper = mountShow();
+    await nextTick();
+    await flushPromises();
+
+    await wrapper.find('[data-test="retry"]').trigger('click');
+    await flushPromises();
+
+    expect(DataImportsAPI.retry).toHaveBeenCalledWith(1);
+    expect(useAlert).toHaveBeenCalledWith('DATA_IMPORTS.ALERTS.IMPORT_RETRIED');
+    wrapper.unmount();
+  });
+
+  it('ignores an older poll response after retry succeeds', async () => {
+    const pollRequest = deferredRequest();
+    DataImportsAPI.retry.mockResolvedValue({
+      data: {
+        id: 1,
+        status: 'pending',
+        stalled: false,
+        skip_logs_filters: {},
+      },
+    });
+    const wrapper = mountShow();
+    await nextTick();
+    await flushPromises();
+
+    DataImportsAPI.show.mockReturnValueOnce(pollRequest.promise);
+    await vi.advanceTimersByTimeAsync(POLL_INTERVAL_MS);
+    expect(DataImportsAPI.show).toHaveBeenCalledTimes(2);
+
+    await wrapper.find('[data-test="retry"]').trigger('click');
+    await flushPromises();
+    expect(wrapper.find('[data-test="retry"]').exists()).toBe(false);
+
+    pollRequest.resolve({
+      data: {
+        id: 1,
+        status: 'processing',
+        stalled: true,
+        skip_logs_filters: {},
+      },
+    });
+    await flushPromises();
+
+    expect(wrapper.find('[data-test="retry"]').exists()).toBe(false);
+    wrapper.unmount();
+  });
+});

app/javascript/dashboard/routes/dashboard/settings/inbox/Settings.vue

@@ -4,6 +4,8 @@ import { shouldBeUrl } from 'shared/helpers/Validators';
 import { useAlert } from 'dashboard/composables';
 import { useVuelidate } from '@vuelidate/core';
 import Avatar from 'next/avatar/Avatar.vue';
+import Banner from 'dashboard/components-next/banner/Banner.vue';
+import Icon from 'dashboard/components-next/icon/Icon.vue';
 import SettingIntroBanner from 'dashboard/components/widgets/SettingIntroBanner.vue';
 import SettingsToggleSection from 'dashboard/components-next/Settings/SettingsToggleSection.vue';
 import SettingsFieldSection from 'dashboard/components-next/Settings/SettingsFieldSection.vue';
@@ -44,9 +46,11 @@ import SelectInput from 'dashboard/components-next/select/Select.vue';
 import Widget from 'dashboard/modules/widget-preview/components/Widget.vue';
 import AccessToken from 'dashboard/routes/dashboard/settings/profile/AccessToken.vue';
 import { copyTextToClipboard } from 'shared/helpers/clipboard';
+import { META_RESTRICTION_STATUS_URL } from 'dashboard/constants/globals';
 
 export default {
   components: {
+    Banner,
     BotConfiguration,
     CollaboratorsPage,
     ConfigurationPage,
@@ -80,6 +84,7 @@ export default {
     WhatsappManualMigrationBanner,
     Widget,
     AccessToken,
+    Icon,
   },
   mixins: [inboxMixin],
   setup() {
@@ -343,6 +348,12 @@ export default {
     instagramUnauthorized() {
       return this.isAnInstagramChannel && this.inbox.reauthorization_required;
     },
+    showInstagramRestrictionSettingsBanner() {
+      return this.isOnChatwootCloud && this.isAnInstagramChannel;
+    },
+    metaRestrictionStatusUrl() {
+      return META_RESTRICTION_STATUS_URL;
+    },
     tiktokUnauthorized() {
       return this.isATiktokChannel && this.inbox.reauthorization_required;
     },
@@ -809,6 +820,29 @@ export default {
           :class="bannerMaxWidth"
           @start="openWhatsAppManualMigrationDialog"
         />
+        <Banner
+          v-if="showInstagramRestrictionSettingsBanner"
+          color="amber"
+          class="mx-6 mb-4 max-w-4xl"
+        >
+          <div class="flex items-start gap-3 text-start">
+            <Icon
+              icon="i-lucide-triangle-alert"
+              class="flex-shrink-0 size-4 mt-0.5"
+            />
+            <span>
+              {{ $t('INBOX_MGMT.ADD.INSTAGRAM.SETTINGS_RESTRICTED_WARNING') }}
+              <a
+                :href="metaRestrictionStatusUrl"
+                class="link underline"
+                rel="noopener noreferrer nofollow"
+                target="_blank"
+              >
+                {{ $t('INBOX_MGMT.ADD.INSTAGRAM.STATUS_LINK') }}
+              </a>
+            </span>
+          </div>
+        </Banner>
 
         <div
           v-if="selectedTabKey === 'inbox-settings'"

app/models/data_import.rb

@@ -33,6 +33,7 @@
 #
 class DataImport < ApplicationRecord
   ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY = 'active_intercom_import_run_id'.freeze
+  INTERCOM_STALLED_AFTER = 15.minutes
   LEGACY_DATA_TYPES = ['contacts'].freeze
   INTEGRATION_DATA_TYPES = ['intercom'].freeze
   IMPORT_TYPES = %w[contacts conversations].freeze
@@ -71,6 +72,10 @@ class DataImport < ApplicationRecord
     failed? || abandoned?
   end
 
+  def stalled?
+    intercom_import? && (pending? || processing?) && updated_at <= INTERCOM_STALLED_AFTER.ago
+  end
+
   def abandonable?
     intercom_import? && (pending? || processing?)
   end

app/policies/data_import_policy.rb

@@ -19,6 +19,10 @@ class DataImportPolicy < ApplicationPolicy
     show?
   end
 
+  def retry_import?
+    show?
+  end
+
   def abandon?
     show?
   end

app/services/data_imports/intercom/importer.rb

@@ -7,13 +7,17 @@ class DataImports::Intercom::Importer
   end
 
   DEFAULT_IMPORT_TYPES = %w[contacts conversations].freeze
+  CONTACTS_PER_PAGE = 50
+  CONVERSATIONS_PER_PAGE = 10
+  HEARTBEAT_INTERVAL = 1.minute
+  QUERY_TIMEOUT_RETRY_LIMIT = 1
+  QUERY_TIMEOUT_RETRY_DELAY_RANGE = (0.2..0.5)
   PROVIDER = 'intercom'.freeze
   ALREADY_IMPORTED_ERROR_CODE = 'DataImports::Intercom::AlreadyImported'.freeze
   SKIPPED_MESSAGE_ERROR_CODE = 'DataImports::Intercom::SkippedMessage'.freeze
   TRUNCATED_PARTS_ERROR_CODE = 'DataImports::Intercom::TruncatedConversationParts'.freeze
   E164_REGEX = /\A\+[1-9]\d{1,14}\z/
   INTERCOM_NUMBER_REGEX = /\A[1-9]\d{1,14}\z/
-  REGULAR_MESSAGE_PART_TYPES = %w[comment note source].freeze
 
   def initialize(data_import:, run_id: nil)
     @data_import = data_import
@@ -22,6 +26,7 @@ class DataImports::Intercom::Importer
     @client = DataImports::Intercom::Client.new(access_token: data_import.access_token)
     @placeholder_inboxes = DataImports::Intercom::PlaceholderInboxBuilder.new(account: @account)
     @stats = default_stats.deep_merge(data_import.stats || {})
+    @dirty_stat_groups = {}
   end
 
   def perform
@@ -44,8 +49,9 @@ class DataImports::Intercom::Importer
   def finish!
     return if @data_import.reload.abandoned?
 
-    has_failures = @data_import.import_errors.non_skip_logs.exists? || @data_import.import_errors.failed.exists?
-    status = has_failures ? :completed_with_errors : :completed
+    error_count = @data_import.import_errors.non_skip_logs.count + @data_import.import_errors.failed.count
+    @stats['errors']['count'] = error_count
+    status = error_count.positive? ? :completed_with_errors : :completed
     @data_import.update!(
       status: status,
       completed_at: Time.current,
@@ -56,14 +62,16 @@ class DataImports::Intercom::Importer
   end
 
   def fail!(error)
-    return if @data_import.reload.abandoned?
+    @data_import.with_lock do
+      next if @data_import.abandoned? || stale_import_run?
 
-    record_run_error(error)
-    @data_import.update!(status: :failed, last_error_at: Time.current)
+      record_run_error(error)
+      @data_import.update!(status: :failed, last_error_at: Time.current)
+    end
   end
 
   def import_contacts_page(starting_after: cursor_for('contacts'))
-    response = @client.list_contacts(starting_after: starting_after)
+    response = @client.list_contacts(starting_after: starting_after, per_page: CONTACTS_PER_PAGE)
     update_stat_total('contacts', response['total_count']) if response['total_count'].present?
     Array(response['data'] || response['contacts']).each do |contact|
       break if import_stopped?
@@ -72,13 +80,14 @@ class DataImports::Intercom::Importer
     end
     return PageResult.new(next_cursor: nil) if import_stopped?
 
+    reconcile_dirty_stats
     next_cursor = response.dig('pages', 'next', 'starting_after')
     update_cursor('contacts', next_cursor)
     PageResult.new(next_cursor: next_cursor)
   end
 
   def import_conversations_page(starting_after: cursor_for('conversations'))
-    response = @client.list_conversations(starting_after: starting_after)
+    response = @client.list_conversations(starting_after: starting_after, per_page: CONVERSATIONS_PER_PAGE)
     update_stat_total('conversations', response['total_count']) if response['total_count'].present?
     Array(response['data'] || response['conversations']).each do |conversation_summary|
       break if import_stopped?
@@ -87,6 +96,7 @@ class DataImports::Intercom::Importer
     end
     return PageResult.new(next_cursor: nil) if import_stopped?
 
+    reconcile_dirty_stats
     next_cursor = response.dig('pages', 'next', 'starting_after')
     update_cursor('conversations', next_cursor)
     PageResult.new(next_cursor: next_cursor)
@@ -153,8 +163,9 @@ class DataImports::Intercom::Importer
     mapped_conversation = mapping&.chatwoot_record
     if mapped_conversation && mapping.data_import_id != @data_import.id
       skip_already_imported_item(item, mapping, already_handled: already_handled)
-      import_source_message(conversation, mapped_conversation, contact)
-      import_conversation_parts(conversation, mapped_conversation, contact)
+      reconcile_item_stats('conversation') if already_handled
+      return unless import_conversation_messages(conversation, mapped_conversation, contact)
+
       update_conversation_activity(mapped_conversation)
       return
     end
@@ -164,17 +175,21 @@ class DataImports::Intercom::Importer
       record_mapping('conversation', source_id, chatwoot_conversation, metadata: conversation_metadata(conversation, inbox, source_type))
     end
     item.update!(status: :imported, chatwoot_record_type: 'Conversation', chatwoot_record_id: chatwoot_conversation.id)
-    increment_stat('conversations', 'imported') unless already_handled
+    if already_handled
+      reconcile_item_stats('conversation')
+    else
+      increment_stat('conversations', 'imported')
+    end
+
+    return unless import_conversation_messages(conversation, chatwoot_conversation, contact)
 
-    import_source_message(conversation, chatwoot_conversation, contact)
-    import_conversation_parts(conversation, chatwoot_conversation, contact)
     update_conversation_activity(chatwoot_conversation)
   rescue StandardError => e
     raise if e.is_a?(DataImports::Intercom::Client::Error)
 
     fail_item(item, e)
   ensure
-    persist_stats
+    persist_stats unless @import_stopped
   end
 
   def import_stopped?
@@ -184,38 +199,49 @@ class DataImports::Intercom::Importer
     @import_stopped = @data_import.abandoned? || @data_import.completed? || @data_import.completed_with_errors? || stale_import_run?
   end
 
+  def continue_import_with_heartbeat?
+    return false if import_stopped?
+    return true if @data_import.updated_at > HEARTBEAT_INTERVAL.ago
+
+    @data_import.touch if @data_import.updated_at <= HEARTBEAT_INTERVAL.ago
+    true
+  end
+
   def stale_import_run?
     active_run_id = @data_import.active_intercom_import_run_id
     @run_id.present? && active_run_id.present? && active_run_id != @run_id
   end
 
   def import_contact(contact_payload, required_for_conversation: false)
-    source_id = source_id_for(contact_payload)
-    if source_id.present? && (mapping = find_mapping('contact', source_id)) && (mapped_contact = mapping.chatwoot_record)
-      return reuse_mapped_contact(contact_payload, source_id, mapping, mapped_contact)
-    end
+    item = nil
+    with_query_timeout_retry do
+      source_id = source_id_for(contact_payload)
+      if source_id.present? && (mapping = find_mapping('contact', source_id)) && (mapped_contact = mapping.chatwoot_record)
+        return reuse_mapped_contact(contact_payload, source_id, mapping, mapped_contact)
+      end
 
-    contact_payload = retrieve_contact_payload(contact_payload)
-    source_id = source_id_for(contact_payload)
-    already_handled = item_handled?('contact', source_id)
-    item = import_item('contact', source_id, contact_payload)
-    mapping = find_mapping('contact', source_id)
+      contact_payload = retrieve_contact_payload(contact_payload)
+      source_id = source_id_for(contact_payload)
+      already_handled = item_handled?('contact', source_id)
+      item = import_item('contact', source_id, contact_payload)
+      mapping = find_mapping('contact', source_id)
 
-    mapped_contact = mapping&.chatwoot_record
-    if mapped_contact && mapping.data_import_id != @data_import.id
-      skip_already_imported_item(item, mapping, already_handled: already_handled)
-      return mapped_contact
-    end
+      mapped_contact = mapping&.chatwoot_record
+      if mapped_contact && mapping.data_import_id != @data_import.id
+        skip_already_imported_item(item, mapping, already_handled: already_handled)
+        return mapped_contact
+      end
 
-    contact = Contact.transaction do
-      imported_contact = mapped_contact || find_existing_contact(contact_payload) || create_contact(contact_payload)
-      update_existing_contact(imported_contact, contact_payload)
-      record_mapping('contact', source_id, imported_contact, metadata: contact_metadata(contact_payload))
-      item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: imported_contact.id)
-      imported_contact
+      contact = Contact.transaction do
+        imported_contact = mapped_contact || find_existing_contact(contact_payload) || create_contact(contact_payload)
+        update_existing_contact(imported_contact, contact_payload)
+        record_mapping('contact', source_id, imported_contact, metadata: contact_metadata(contact_payload))
+        item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: imported_contact.id)
+        imported_contact
+      end
+      increment_stat('contacts', 'imported') unless already_handled
+      contact
     end
-    increment_stat('contacts', 'imported') unless already_handled
-    contact
   rescue StandardError => e
     raise if e.is_a?(DataImports::Intercom::Client::Error)
 
@@ -369,63 +395,85 @@ class DataImports::Intercom::Importer
     end
   end
 
-  def import_source_message(conversation, chatwoot_conversation, contact)
-    source = conversation['source'].to_h
-    return unless source_message_importable?(source)
-
-    message_source_id = "conversation:#{source_id_for(conversation)}:source:#{source['id'].presence || 'initial'}"
-    source_part = source.merge('part_type' => 'source', 'created_at' => conversation['created_at'])
-    if (mapping = find_mapping('message', message_source_id)) && message_mapping_handled?(mapping, source_part)
-      if mapping.data_import_id == @data_import.id
-        reconcile_current_run_message_mapping(chatwoot_conversation, mapping, source_part)
-        return
-      end
-
-      skip_existing_message_mapping(chatwoot_conversation, mapping, source_part)
-      return
-    end
-
-    create_message(chatwoot_conversation, contact, source_part, message_source_id)
-  rescue StandardError => e
-    fail_message(chatwoot_conversation, message_source_id, source_part, e)
-  end
-
-  def import_conversation_parts(conversation, chatwoot_conversation, contact)
+  def import_conversation_messages(conversation, chatwoot_conversation, contact)
     parts_payload = conversation['conversation_parts'].to_h
     parts = Array(parts_payload['conversation_parts'])
+    batch_builder = DataImports::Intercom::MessageBatchBuilder.new(
+      data_import: @data_import,
+      conversation: chatwoot_conversation,
+      source_conversation: conversation
+    )
+    batch = begin
+      with_query_timeout_retry { batch_builder.perform }
+    rescue ActiveRecord::QueryCanceled
+      nil
+    end
+    return import_conversation_messages_individually(conversation, chatwoot_conversation, contact, batch_builder, parts.size) if batch.nil?
+
+    batch.source_entries.each { |entry| import_message(chatwoot_conversation, contact, entry) }
     record_truncated_conversation_parts(conversation, parts.size)
 
-    parts.each do |part|
-      message_source_id = "conversation:#{source_id_for(conversation)}:part:#{part['id']}"
-      if (mapping = find_mapping('message', message_source_id)) && message_mapping_handled?(mapping, part)
-        if mapping.data_import_id == @data_import.id
-          reconcile_current_run_message_mapping(chatwoot_conversation, mapping, part)
-          next
-        end
+    batch.part_entries.each do |entry|
+      return false unless continue_import_with_heartbeat?
 
-        skip_existing_message_mapping(chatwoot_conversation, mapping, part)
-        next
-      end
-
-      create_message(chatwoot_conversation, contact, part, message_source_id)
-    rescue StandardError => e
-      fail_message(chatwoot_conversation, message_source_id, part, e)
+      import_message(chatwoot_conversation, contact, entry)
     end
+    return false if import_stopped?
+
+    true
   end
 
-  def create_message(conversation, contact, part, message_source_id)
-    content = content_for(part)
-    return record_skipped_message(conversation, message_source_id, part) if content.blank?
+  def import_conversation_messages_individually(conversation, chatwoot_conversation, contact, batch_builder, parts_count)
+    source_entries, part_entries = batch_builder.unprepared_entries.partition { |entry| entry[:part]['part_type'] == 'source' }
+    source_entries.each { |entry| import_unprepared_message(chatwoot_conversation, contact, batch_builder, entry) }
+    record_truncated_conversation_parts(conversation, parts_count)
 
-    attrs = message_attributes(conversation, contact, part, message_source_id, content)
-    message = nil
+    part_entries.each do |entry|
+      return false unless continue_import_with_heartbeat?
+
+      import_unprepared_message(chatwoot_conversation, contact, batch_builder, entry)
+    end
+    return false if import_stopped?
+
+    true
+  end
+
+  def import_unprepared_message(conversation, contact, batch_builder, source_entry)
+    entry = with_query_timeout_retry { batch_builder.perform([source_entry]).entries.first }
+    import_message(conversation, contact, entry)
+  rescue StandardError => e
+    fail_message(conversation, source_entry[:source_id], source_entry[:part], e)
+  end
+
+  def import_message(conversation, contact, entry)
+    with_query_timeout_retry do
+      case entry.classification
+      when :current_import
+        reconcile_current_run_message_mapping(conversation, entry.mapping, entry.part)
+      when :previous_import
+        skip_existing_message_mapping(conversation, entry.mapping, entry.part)
+      when :repairable_stale_mapping, :existing_message, :new_message
+        create_message(conversation, contact, entry)
+      else
+        raise ArgumentError, "Unsupported Intercom message classification: #{entry.classification}"
+      end
+    end
+  rescue StandardError => e
+    fail_message(conversation, entry.source_id, entry.part, e)
+  end
+
+  def create_message(conversation, contact, entry)
+    content = content_for(entry.part)
+    return record_skipped_message(conversation, entry) if content.blank?
+
+    attrs = message_attributes(conversation, contact, entry.part, entry.source_id, content)
+    message = entry.message
     Message.transaction do
-      message = conversation.messages.find_by(source_id: attrs[:source_id])
       unless message
         result = Message.insert_all!([attrs], returning: %w[id])
         message = Message.find(result.rows.first.first)
       end
-      record_mapping('message', message_source_id, message, metadata: message_metadata(part))
+      record_message_mapping(entry, message)
     end
     increment_stat('messages', 'imported')
     reindex_message_for_search(message)
@@ -440,13 +488,12 @@ class DataImports::Intercom::Importer
     Rails.logger.warn("Intercom import message reindex failed for message #{message.id}: #{e.class} - #{e.message}")
   end
 
-  def record_skipped_message(conversation, message_source_id, part)
-    mapping = find_mapping('message', message_source_id)
-    if mapping
-      already_recorded = skip_log_recorded?('message', message_source_id, SKIPPED_MESSAGE_ERROR_CODE)
-      record_skipped_message_log(conversation, message_source_id, part)
+  def record_skipped_message(conversation, entry)
+    if entry.mapping
+      already_recorded = skip_log_recorded?('message', entry.source_id, SKIPPED_MESSAGE_ERROR_CODE)
+      record_skipped_message_log(conversation, entry.source_id, entry.part)
       increment_stat('messages', 'skipped') unless already_recorded
-      return mapping.chatwoot_record
+      return entry.message
     end
 
     DataImportMapping.create!(
@@ -454,15 +501,30 @@ class DataImports::Intercom::Importer
       data_import: @data_import,
       source_provider: PROVIDER,
       source_object_type: 'message',
-      source_object_id: message_source_id,
+      source_object_id: entry.source_id,
       chatwoot_record_type: 'Conversation',
       chatwoot_record_id: conversation.id,
-      metadata: message_metadata(part).merge(skipped: true, reason: 'blank_or_unsupported_intercom_part')
+      metadata: message_metadata(entry.part).merge(skipped: true, reason: 'blank_or_unsupported_intercom_part')
     )
-    record_skipped_message_log(conversation, message_source_id, part)
+    record_skipped_message_log(conversation, entry.source_id, entry.part)
     increment_stat('messages', 'skipped')
   end
 
+  def record_message_mapping(entry, message)
+    (entry.mapping || DataImportMapping.new(
+      account: @account,
+      source_provider: PROVIDER,
+      source_object_type: 'message',
+      source_object_id: entry.source_id
+    )).tap do |mapping|
+      mapping.data_import = @data_import
+      mapping.chatwoot_record_type = 'Message'
+      mapping.chatwoot_record_id = message.id
+      mapping.metadata = message_metadata(entry.part)
+      mapping.save!
+    end
+  end
+
   def message_attributes(conversation, contact, part, message_source_id, content)
     message_type = message_type_for(part)
     created_at = timestamp_for(part['created_at'])
@@ -503,8 +565,7 @@ class DataImports::Intercom::Importer
   end
 
   def activity_part?(part)
-    part_type = part['part_type'].to_s
-    part_type.present? && REGULAR_MESSAGE_PART_TYPES.exclude?(part_type)
+    DataImports::Intercom::MessageBatchBuilder.activity_part?(part)
   end
 
   def message_content(part)
@@ -629,7 +690,7 @@ class DataImports::Intercom::Importer
     )
     item = import_item('contact', source_id, contact_payload) unless item&.imported?
     item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: mapped_contact.id)
-    reconcile_item_stats('contact')
+    mark_stat_group_dirty('contacts')
   end
 
   def reconcile_item_stats(source_object_type)
@@ -637,34 +698,37 @@ class DataImports::Intercom::Importer
     group = stat_group_for(source_object_type)
     @stats[group]['imported'] = items.imported.count
     @stats[group]['skipped'] = items.skipped.count
-    persist_stats
   end
 
   def reconcile_current_run_message_mapping(conversation, mapping, part)
     record_skipped_message_log(conversation, mapping.source_object_id, part) if mapping.metadata['skipped']
+    mark_stat_group_dirty('messages')
+  end
 
+  def reconcile_message_stats
     mappings = @data_import.mappings.where(source_provider: PROVIDER, source_object_type: 'message')
     skipped_mappings = mappings.where("metadata ->> 'skipped' = ?", 'true').count
     message_logs = @data_import.import_errors.where(source_object_type: 'message')
     @stats['messages']['imported'] = mappings.count - skipped_mappings
     @stats['messages']['skipped'] = message_logs.where("details ->> 'kind' = ?", 'skipped').count
-    persist_stats
   end
 
   def skip_already_imported_item(item, mapping, already_handled:)
-    item.update!(
-      status: :skipped,
-      chatwoot_record_type: mapping.chatwoot_record_type,
-      chatwoot_record_id: mapping.chatwoot_record_id,
-      last_error_code: ALREADY_IMPORTED_ERROR_CODE,
-      last_error_message: 'Already imported in a previous import.'
-    )
-    record_already_imported_log(
-      data_import_item: item,
-      source_object_type: item.source_object_type,
-      source_object_id: item.source_object_id,
-      mapping: mapping
-    )
+    DataImportItem.transaction do
+      item.update!(
+        status: :skipped,
+        chatwoot_record_type: mapping.chatwoot_record_type,
+        chatwoot_record_id: mapping.chatwoot_record_id,
+        last_error_code: ALREADY_IMPORTED_ERROR_CODE,
+        last_error_message: 'Already imported in a previous import.'
+      )
+      record_already_imported_log(
+        data_import_item: item,
+        source_object_type: item.source_object_type,
+        source_object_id: item.source_object_id,
+        mapping: mapping
+      )
+    end
     increment_stat(stat_group_for(item.source_object_type), 'skipped') unless already_handled
   end
 
@@ -676,13 +740,12 @@ class DataImports::Intercom::Importer
       already_recorded = skip_log_recorded?('message', mapping.source_object_id, ALREADY_IMPORTED_ERROR_CODE)
       record_already_imported_log(source_object_type: 'message', source_object_id: mapping.source_object_id, mapping: mapping)
     end
-    increment_stat('messages', 'skipped') unless already_recorded
-  end
-
-  def message_mapping_handled?(mapping, part)
-    return false if mapping.metadata['skipped'] && activity_part?(part)
-
-    mapping.metadata['skipped'] || mapping.chatwoot_record.present?
+    if already_recorded
+      message_logs = @data_import.import_errors.where(source_object_type: 'message')
+      @stats['messages']['skipped'] = message_logs.where("details ->> 'kind' = ?", 'skipped').count
+    else
+      increment_stat('messages', 'skipped')
+    end
   end
 
   def fail_item(item, error)
@@ -782,7 +845,7 @@ class DataImports::Intercom::Importer
   end
 
   def source_message_importable?(source)
-    source['body'].present? || source['subject'].present? || source['attachments'].present?
+    DataImports::Intercom::MessageBatchBuilder.source_message_importable?(source)
   end
 
   def skipped_message_log_message(part)
@@ -935,6 +998,42 @@ class DataImports::Intercom::Importer
     @stats[group][key] = @stats[group][key].to_i + 1
   end
 
+  def mark_stat_group_dirty(group)
+    @dirty_stat_groups[group] = true
+  end
+
+  def reconcile_dirty_stats
+    return if @dirty_stat_groups.empty?
+
+    with_query_timeout_retry do
+      @dirty_stat_groups.each_key do |group|
+        case group
+        when 'contacts'
+          reconcile_item_stats('contact')
+        when 'messages'
+          reconcile_message_stats
+        else
+          raise ArgumentError, "Unsupported Intercom import stat group: #{group}"
+        end
+      end
+      persist_stats
+    end
+    @dirty_stat_groups.clear
+  end
+
+  def with_query_timeout_retry
+    retries = 0
+    begin
+      yield
+    rescue ActiveRecord::QueryCanceled
+      raise if retries >= QUERY_TIMEOUT_RETRY_LIMIT
+
+      retries += 1
+      sleep(rand(QUERY_TIMEOUT_RETRY_DELAY_RANGE))
+      retry
+    end
+  end
+
   def update_stat_total(group, total)
     @stats[group] ||= {}
     @stats[group]['total'] = total.to_i

app/services/data_imports/intercom/message_batch_builder.rb

@@ -0,0 +1,134 @@
+class DataImports::Intercom::MessageBatchBuilder
+  PROVIDER = 'intercom'.freeze
+  REGULAR_PART_TYPES = %w[comment note source].freeze
+
+  Entry = Struct.new(:source_id, :part, :position, :mapping, :message, :classification, keyword_init: true) do
+    def source?
+      part['part_type'] == 'source'
+    end
+  end
+
+  Batch = Struct.new(:items, keyword_init: true) do
+    def entries
+      items
+    end
+
+    def source_entries
+      items.select(&:source?)
+    end
+
+    def part_entries
+      items.reject(&:source?)
+    end
+  end
+
+  def self.activity_part?(part)
+    part_type = part['part_type'].to_s
+    part_type.present? && REGULAR_PART_TYPES.exclude?(part_type)
+  end
+
+  def self.source_message_importable?(source)
+    source['body'].present? || source['subject'].present? || source['attachments'].present?
+  end
+
+  def initialize(data_import:, conversation:, source_conversation:)
+    @data_import = data_import
+    @account = data_import.account
+    @conversation = conversation
+    @source_conversation = source_conversation
+  end
+
+  def perform(source_entries = unprepared_entries)
+    return Batch.new(items: []) if source_entries.empty?
+
+    mappings = message_mappings(source_entries)
+    messages = messages_for(source_entries, mappings)
+
+    Batch.new(items: source_entries.map do |source_entry|
+      build_entry(source_entry, source_entry[:position], mappings, messages)
+    end)
+  end
+
+  def unprepared_entries
+    ordered_source_entries.map.with_index { |entry, position| entry.merge(position: position) }
+  end
+
+  private
+
+  def ordered_source_entries
+    entries = []
+    source = @source_conversation['source'].to_h
+    if self.class.source_message_importable?(source)
+      entries << {
+        source_id: "conversation:#{source_conversation_id}:source:#{source['id'].presence || 'initial'}",
+        part: source.merge('part_type' => 'source', 'created_at' => @source_conversation['created_at'])
+      }
+    end
+
+    conversation_parts.each do |part|
+      entries << { source_id: "conversation:#{source_conversation_id}:part:#{part['id']}", part: part }
+    end
+    entries
+  end
+
+  def conversation_parts
+    Array(@source_conversation.dig('conversation_parts', 'conversation_parts'))
+  end
+
+  def source_conversation_id
+    @source_conversation['id'].presence || @source_conversation['external_id'].presence || @source_conversation['email'].presence
+  end
+
+  def message_mappings(source_entries)
+    DataImportMapping.where(
+      account: @account,
+      source_provider: PROVIDER,
+      source_object_type: 'message',
+      source_object_id: source_entries.pluck(:source_id)
+    ).index_by(&:source_object_id)
+  end
+
+  def messages_for(source_entries, mappings)
+    mapped_message_ids = mappings.values.filter_map do |mapping|
+      mapping.chatwoot_record_id if mapping.chatwoot_record_type == 'Message'
+    end
+    chatwoot_source_ids = source_entries.map { |entry| "intercom:#{entry[:source_id]}" }
+    messages = Message.where(id: mapped_message_ids).or(
+      Message.where(conversation_id: @conversation.id, source_id: chatwoot_source_ids)
+    ).to_a
+
+    {
+      by_id: messages.index_by(&:id),
+      by_source_id: messages.index_by(&:source_id)
+    }
+  end
+
+  def build_entry(source_entry, position, mappings, messages)
+    source_id = source_entry[:source_id]
+    mapping = mappings[source_id]
+    mapped_message = messages[:by_id][mapping.chatwoot_record_id] if mapping&.chatwoot_record_type == 'Message'
+    existing_message = messages[:by_source_id]["intercom:#{source_id}"]
+
+    Entry.new(
+      source_id: source_id,
+      part: source_entry[:part],
+      position: position,
+      mapping: mapping,
+      message: mapped_message || existing_message,
+      classification: classification_for(mapping, mapped_message, existing_message, source_entry[:part])
+    )
+  end
+
+  def classification_for(mapping, mapped_message, existing_message, part)
+    return existing_message.present? ? :existing_message : :new_message if mapping.blank?
+    return :repairable_stale_mapping unless mapping_handled?(mapping, mapped_message, part)
+
+    mapping.data_import_id == @data_import.id ? :current_import : :previous_import
+  end
+
+  def mapping_handled?(mapping, mapped_message, part)
+    return false if mapping.metadata['skipped'] && self.class.activity_part?(part)
+
+    mapping.metadata['skipped'] || mapped_message.present?
+  end
+end

app/services/data_imports/intercom/retry_service.rb

@@ -0,0 +1,28 @@
+class DataImports::Intercom::RetryService
+  attr_reader :data_import
+
+  def initialize(account:, data_import:)
+    @account = account
+    @data_import = data_import
+  end
+
+  def perform
+    @account.with_lock do
+      @data_import.with_lock do
+        next :not_stalled unless @data_import.stalled?
+        next :active_import_exists if another_active_import?
+        next :access_token_missing if @data_import.access_token.blank?
+
+        @data_import.assign_active_intercom_import_run_id
+        @data_import.update!(status: :pending)
+        :enqueue
+      end
+    end
+  end
+
+  private
+
+  def another_active_import?
+    @account.data_imports.active_intercom.where.not(id: @data_import.id).exists?
+  end
+end

app/views/api/v1/accounts/data_imports/_data_import.json.jbuilder

@@ -5,6 +5,7 @@ json.source_type data_import.source_type
 json.source_provider data_import.source_provider
 json.import_types data_import.import_types
 json.status data_import.status
+json.stalled data_import.stalled?
 json.total_records data_import.total_records
 json.processed_records data_import.processed_records
 json.stats data_import.stats

config/app.yml

@@ -1,5 +1,5 @@
 shared: &shared
-  version: '4.16.1'
+  version: '4.16.0'
 
 development:
   <<: *shared

config/llm.yml

@@ -143,20 +143,6 @@ features:
         gemini-3-pro,
       ]
     default: gpt-5.2
-  conversation_faq_matching:
-    models:
-      [
-        gpt-4.1-mini,
-        gpt-5-mini,
-        gpt-4.1,
-        gpt-5.1,
-        gpt-5.2,
-        claude-haiku-4.5,
-        claude-sonnet-4.5,
-        gemini-3-flash,
-        gemini-3-pro,
-      ]
-    default: gpt-4.1-mini
   pdf_faq_generation:
     models: [gpt-4.1-mini, gpt-5-mini, gpt-4.1, gpt-5.1, gpt-5.2]
     default: gpt-4.1-mini

config/locales/en.yml

@@ -626,7 +626,6 @@ en:
         label_suggestion: 'Label suggestion'
         document_faq_generation: 'Document FAQ generation'
         conversation_faq_generation: 'Conversation FAQ generation'
-        conversation_faq_matching: 'Conversation FAQ matching'
         help_center_article_generation: 'Help center article generation'
         onboarding_content_generation: 'Onboarding content generation'
         help_center_query_translation: 'Help center query translation'

config/routes.rb

@@ -79,10 +79,6 @@ Rails.application.routes.draw do
             end
             resources :agent_sessions, only: [:show]
             resources :assistant_responses
-            resources :faq_suggestions, only: [:index, :show, :update] do
-              post :approve, on: :member
-              post :dismiss, on: :member
-            end
             resources :message_reports, only: [:create]
             resources :bulk_actions, only: [:create]
             resources :copilot_threads, only: [:index, :create] do
@@ -234,6 +230,7 @@ Rails.application.routes.draw do
             end
             member do
               post :start
+              post :retry, action: :retry_import
               post :abandon
               get :error_logs
               get :skip_logs

enterprise/app/controllers/api/v1/accounts/captain/faq_suggestions_controller.rb

@@ -1,80 +0,0 @@
-class Api::V1::Accounts::Captain::FaqSuggestionsController < Api::V1::Accounts::BaseController
-  before_action :current_account
-  before_action -> { check_authorization(Captain::FaqSuggestion) }
-  before_action :set_accessible_suggestions
-  before_action :set_suggestion, except: [:index]
-
-  RESULTS_PER_PAGE = 25
-  SOURCE_PREVIEW_LIMIT = 50
-
-  def index
-    @current_page = permitted_params[:page] || 1
-    filtered_query = apply_filters(@suggestions)
-    @suggestions_count = filtered_query.count
-    @suggestions = filtered_query.page(@current_page).per(RESULTS_PER_PAGE)
-  end
-
-  def show
-    @observations = @suggestion.observations
-                               .where(conversation_id: accessible_conversations.select(:id))
-                               .includes(:conversation)
-                               .order(created_at: :desc)
-                               .limit(SOURCE_PREVIEW_LIMIT)
-  end
-
-  def update
-    @suggestion.with_lock do
-      raise ActiveRecord::RecordNotFound unless @suggestion.open?
-
-      @suggestion.update!(suggestion_params)
-    end
-  end
-
-  def approve
-    attributes = params[:faq_suggestion].present? ? suggestion_params : {}
-    @response = Captain::FaqSuggestionApprovalService.new(@suggestion, attributes).perform
-  end
-
-  def dismiss
-    @suggestion.with_lock do
-      raise ActiveRecord::RecordNotFound unless @suggestion.open?
-
-      @suggestion.dismissed!
-    end
-  end
-
-  private
-
-  def apply_filters(base_query)
-    base_query = base_query.where(assistant_id: permitted_params[:assistant_id]) if permitted_params[:assistant_id].present?
-    base_query = base_query.where(status: permitted_params[:status]) if permitted_params[:status].present?
-
-    if permitted_params[:search].present?
-      # TODO: Move FAQ suggestion search to Elasticsearch when the records are indexed there.
-      search_term = "%#{permitted_params[:search]}%"
-      base_query = base_query.where('question ILIKE :search OR answer ILIKE :search', search: search_term)
-    end
-
-    base_query
-  end
-
-  def set_accessible_suggestions
-    @suggestions = Captain::FaqSuggestionFinder.new(Current.user, Current.account).perform.includes(:assistant).ordered
-  end
-
-  def set_suggestion
-    @suggestion = @suggestions.find(permitted_params[:id])
-  end
-
-  def accessible_conversations
-    Conversations::PermissionFilterService.new(Current.account.conversations, Current.user, Current.account).perform
-  end
-
-  def permitted_params
-    params.permit(:id, :assistant_id, :page, :status, :search)
-  end
-
-  def suggestion_params
-    params.require(:faq_suggestion).permit(:question, :answer)
-  end
-end

enterprise/app/finders/captain/faq_suggestion_finder.rb

@@ -1,26 +0,0 @@
-class Captain::FaqSuggestionFinder
-  def initialize(current_user, current_account)
-    @current_user = current_user
-    @current_account = current_account
-  end
-
-  def perform
-    suggestions = @current_account.captain_faq_suggestions
-    return suggestions if account_user&.administrator?
-
-    accessible_suggestion_ids = Captain::FaqObservation
-                                .where(conversation_id: accessible_conversations.select(:id))
-                                .select(:faq_suggestion_id)
-    suggestions.where(id: accessible_suggestion_ids)
-  end
-
-  private
-
-  def accessible_conversations
-    Conversations::PermissionFilterService.new(@current_account.conversations, @current_user, @current_account).perform
-  end
-
-  def account_user
-    @account_user ||= @current_account.account_users.find_by(user_id: @current_user.id)
-  end
-end

enterprise/app/jobs/captain/llm/conversation_faq_job.rb

@@ -1,30 +0,0 @@
-class Captain::Llm::ConversationFaqJob < MutexApplicationJob
-  queue_as :low
-
-  LOCK_TIMEOUT = 10.minutes
-
-  retry_on_lock_conflict wait: 30.seconds, attempts: 30
-
-  def perform(conversation, assistant)
-    inbox = conversation.inbox
-
-    return unless conversation.resolved?
-    return unless inbox.captain_active?
-
-    return if assistant.config['feature_faq'].blank?
-
-    with_lock(lock_key(assistant, conversation), LOCK_TIMEOUT) do
-      Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_suggestions
-    end
-  end
-
-  private
-
-  def lock_key(assistant, conversation)
-    format(
-      ::Redis::Alfred::CAPTAIN_CONVERSATION_FAQ_MUTEX,
-      assistant_id: assistant.id,
-      language: Captain::Llm::ConversationFaqService.language_for(conversation)
-    )
-  end
-end

enterprise/app/listeners/captain_listener.rb

@@ -8,6 +8,6 @@ class CaptainListener < BaseListener
     return unless conversation.inbox.captain_active?
 
     Captain::Llm::ContactNotesService.new(assistant, conversation).generate_and_update_notes if assistant.config['feature_memory'].present?
-    Captain::Llm::ConversationFaqJob.perform_later(conversation, assistant) if assistant.config['feature_faq'].present?
+    Captain::Llm::ConversationFaqService.new(assistant, conversation).generate_and_deduplicate if assistant.config['feature_faq'].present?
   end
 end

enterprise/app/policies/captain/assistant_policy.rb

@@ -35,14 +35,6 @@ class Captain::AssistantPolicy < ApplicationPolicy
     @account_user.administrator?
   end
 
-  def approve?
-    update?
-  end
-
-  def dismiss?
-    update?
-  end
-
   def destroy?
     @account_user.administrator?
   end

enterprise/app/policies/captain/faq_suggestion_policy.rb

@@ -1,21 +0,0 @@
-class Captain::FaqSuggestionPolicy < ApplicationPolicy
-  def index?
-    true
-  end
-
-  def show?
-    true
-  end
-
-  def update?
-    true
-  end
-
-  def approve?
-    true
-  end
-
-  def dismiss?
-    true
-  end
-end

enterprise/app/services/captain/faq_suggestion_approval_service.rb

@@ -1,26 +0,0 @@
-class Captain::FaqSuggestionApprovalService
-  def initialize(suggestion, attributes = {})
-    @suggestion = suggestion
-    @attributes = attributes
-  end
-
-  def perform
-    suggestion.with_lock do
-      raise ActiveRecord::RecordNotFound unless suggestion.open?
-
-      suggestion.update!(attributes) if attributes.present?
-
-      response = suggestion.assistant.responses.create!(
-        question: suggestion.question,
-        answer: suggestion.answer,
-        status: :approved
-      )
-      suggestion.approved!
-      response
-    end
-  end
-
-  private
-
-  attr_reader :suggestion, :attributes
-end

enterprise/app/services/captain/llm/conversation_faq_content_service.rb

@@ -1,66 +0,0 @@
-class Captain::Llm::ConversationFaqContentService
-  def initialize(assistant, conversation)
-    @assistant = assistant
-    @conversation = conversation
-  end
-
-  def generate
-    [
-      'Business Context:',
-      JSON.pretty_generate(business_context),
-      "Conversation ID: ##{conversation.display_id}",
-      "Channel: #{conversation.inbox.channel.name}",
-      'Message History:',
-      conversation_messages
-    ].join("\n")
-  end
-
-  private
-
-  attr_reader :assistant, :conversation
-
-  def conversation_messages
-    messages = conversation
-               .messages
-               .where(message_type: %i[incoming outgoing], private: false)
-               .order(created_at: :asc)
-
-    return "No messages in this conversation\n" if messages.empty?
-
-    messages.filter_map { |message| format_message(message) }.join
-  end
-
-  def format_message(message)
-    return unless source_message?(message)
-
-    message_content = message.content_for_llm
-    return if message_content.blank?
-
-    sender = human_support_reply?(message) ? 'Support Agent' : 'User'
-    "#{sender}: #{message_content}\n"
-  end
-
-  def source_message?(message)
-    return true if message.incoming? && message.sender_type == 'Contact'
-
-    human_support_reply?(message)
-  end
-
-  def human_support_reply?(message)
-    return false unless message.outgoing?
-    return false if message.content_attributes['automation_rule_id'].present?
-    return false if message.additional_attributes['campaign_id'].present?
-
-    message.sender_type == 'User' || message.content_attributes['external_echo'].present?
-  end
-
-  def business_context
-    {
-      product_name: assistant.config['product_name'],
-      assistant_description: assistant.description,
-      instructions: assistant.config['instructions'],
-      response_guidelines: assistant.response_guidelines,
-      guardrails: assistant.guardrails
-    }.compact
-  end
-end

enterprise/app/services/captain/llm/conversation_faq_prompts_service.rb

@@ -1,75 +0,0 @@
-class Captain::Llm::ConversationFaqPromptsService
-  class << self
-    def generator(language = 'english')
-      <<~PROMPT
-        You create high-quality FAQ candidates from resolved support conversations.
-        Only generate an FAQ when the conversation contains durable, reusable knowledge that would help many future customers.
-
-        ## Source rules
-        - The input starts with trusted business context. Use it to reject conversations about other businesses or topics, but never use it as the source of an FAQ answer.
-        - The conversation history contains only customer messages and human support agent messages.
-        - Base every FAQ strictly on information stated in the human support agent messages. Do not infer, generalize, or add external knowledge.
-        - A human support agent must state every fact used in the FAQ answer. Customer messages cannot supply missing answer facts.
-        - The human support agent must provide the final answer. If the agent only greets, asks clarifying questions, asks for contact details, promises to check, shares an attachment, or transfers the conversation, return: `{"faqs":[]}`.
-        - For each FAQ, identify the human support agent message or messages that together provide a complete public answer to the same question. Combine facts only across related agent messages; never combine separate questions or unrelated topics. If those messages do not provide a complete public answer, remove that FAQ.
-
-        ## Decision gate
-        Return `{"faqs":[]}` unless every generated FAQ can pass all of these checks:
-        1. The answer is fully stated by a human support agent, not by the customer.
-        2. The answer is a public, durable rule or procedure, not a private account action, manual review, troubleshooting session, quote, file, link, or follow-up.
-        3. The answer can be written without private identifiers, customer-specific facts, direct URLs, attachments, invoices, screenshots, or support-ticket steps.
-        4. The question would still make sense in a help center if the original conversation, customer, and agent did not exist.
-        Do not rescue a rejected conversation by rewriting it as a generic support question.
-
-        ## Return no FAQ for
-        - Spam, scams, advertisements, SEO/link-building pitches, adult/gambling/financial promotions, gibberish, abusive content, or conversations unrelated to the business being supported.
-        - Account-specific, order-specific, payment-specific, subscription-specific, login/access, verification, delivery, certificate, or troubleshooting issues, even if they could be rewritten as a general support question.
-        - Conversations that mainly hand off to a human, ask the customer to wait, request private identifiers or contact details, collect screenshots, attachments, or documents, or tell the customer to contact support for case review.
-        - Temporary workarounds, one-off exceptions, unclear answers, unresolved problems, wrong-service conversations, complaints, greetings, or abandoned conversations.
-        - Internal support workflow details, chat session rules, escalation mechanics, ticket-routing instructions, or "someone will get back to you" messages.
-        - Answers that are just a direct/private link, attachment, file, invoice, one-off quote or estimate, account-specific URL, or instructions to open a support ticket.
-        - Questions whose useful answer is "contact support", "wait for the team", "share your details", "we will check", or "this needs manual review".
-        - Questions about whether support can help with a private issue, third-party service, transaction, payment, delivery, or account problem.
-        - Pricing, policy, availability, roadmap, deadline, or legal claims unless the human support agent gives a clear and stable answer in the conversation.
-        - Questions already answered only by asking the customer for more information.
-
-        ## FAQ quality rules
-        - Prefer returning no FAQ over a weak or narrow FAQ.
-        - A good candidate teaches a generally reusable product, service, policy, setup, or process rule that another customer could use without contacting support.
-        - Generate at most one FAQ unless the human agent clearly answered multiple distinct, reusable questions.
-        - Do not create duplicate or overlapping FAQs in the same response.
-        - Questions must be general enough for a help center, not personalized to the current customer.
-        - Remove customer names, order numbers, invoice numbers, IDs, private URLs, phone numbers, emails, screenshots, attachments, and other personal or transaction-specific details.
-        - Answers must be complete, self-contained, and supported by the human agent's messages.
-
-        ## Examples
-        - Customer mentions a price or procedure, then the human agent only greets or says they will check: return `{"faqs":[]}`.
-        - Human agent shares only a private link, file, invoice, quote, screenshot, or attachment: return `{"faqs":[]}`.
-        - Human agent clearly states a public rule, such as which purchases are allowed for a program or service: generate one general FAQ.
-
-        Generate the FAQs only in the #{language}, use no other language.
-        If no suitable reusable FAQ is available, return: `{"faqs":[]}`.
-
-        Return only valid JSON in this exact structure:
-        ```json
-        { "faqs": [ { "question": "", "answer": "" } ] }
-        ```
-      PROMPT
-    end
-
-    def same_faq
-      <<~PROMPT
-        Decide whether the new FAQ and existing FAQ are the same.
-
-        Return `same_faq` as true only when both questions ask the same thing and both answers give the same guidance.
-        Wording, grammar, level of detail, and examples may differ. Return false when either FAQ adds, removes, contradicts, or changes a condition,
-        policy, procedure, audience, product, plan, time frame, or outcome. Related FAQs are not the same FAQ. When uncertain, return false.
-
-        Return only valid JSON in this exact structure:
-        ```json
-        { "same_faq": true }
-        ```
-      PROMPT
-    end
-  end
-end

enterprise/app/services/captain/llm/conversation_faq_service.rb

@@ -1,208 +1,167 @@
 class Captain::Llm::ConversationFaqService < Llm::BaseAiService
   include Integrations::LlmInstrumentation
 
-  class SuggestionChangedError < StandardError; end
-
   DISTANCE_THRESHOLD = 0.3
-  MATCH_LIMIT = 5
   LLM_FEATURE = 'conversation_faq_generation'.freeze
 
-  def self.language_for(conversation)
-    language = conversation.language.presence || conversation.account.locale.presence || I18n.default_locale.to_s
-    normalize_language(language)
-  end
-
-  def self.normalize_language(language)
-    language.to_s.tr('-', '_').split('_').first.downcase
-  end
-
-  private_class_method :normalize_language
-
   def initialize(assistant, conversation)
     super(feature: LLM_FEATURE, account: conversation.account, fallback_model: Llm::Models.default_model_for(LLM_FEATURE))
     @assistant = assistant
     @conversation = conversation
-    @content = Captain::Llm::ConversationFaqContentService.new(assistant, conversation).generate
-    @embedding_service = Captain::Llm::EmbeddingService.new(account_id: conversation.account_id)
+    @content = conversation_faq_content
   end
 
-  def generate_suggestions
+  # Generates and deduplicates FAQs from conversation content
+  # Skips processing if there was no human interaction
+  def generate_and_deduplicate
     return [] if no_human_interaction?
 
-    generate.map { |faq| route_candidate(faq) }
+    new_faqs = generate
+    return [] if new_faqs.empty?
+
+    duplicate_faqs, unique_faqs = find_and_separate_duplicates(new_faqs)
+    save_new_faqs(unique_faqs)
+    log_duplicate_faqs(duplicate_faqs) if Rails.env.development?
   end
 
   private
 
-  attr_reader :content, :conversation, :assistant, :embedding_service
+  attr_reader :content, :conversation, :assistant
+
+  def conversation_faq_content
+    [
+      "Conversation ID: ##{conversation.display_id}",
+      "Channel: #{conversation.inbox.channel.name}",
+      'Message History:',
+      conversation_faq_messages
+    ].join("\n")
+  end
+
+  def conversation_faq_messages
+    messages = conversation
+               .messages
+               .where(message_type: %i[incoming outgoing], private: false)
+               .order(created_at: :asc)
+
+    return "No messages in this conversation\n" if messages.empty?
+
+    messages.filter_map { |message| format_conversation_faq_message(message) }.join
+  end
+
+  def format_conversation_faq_message(message)
+    return unless faq_source_message?(message)
+
+    content = message.content_for_llm
+    return if content.blank?
+
+    sender = human_support_reply?(message) ? 'Support Agent' : 'User'
+    "#{sender}: #{content}\n"
+  end
+
+  def faq_source_message?(message)
+    return true if message.incoming? && message.sender_type == 'Contact'
+
+    human_support_reply?(message)
+  end
+
+  def human_support_reply?(message)
+    return false unless message.outgoing?
+    return false if message.content_attributes['automation_rule_id'].present?
+    return false if message.additional_attributes['campaign_id'].present?
+
+    message.sender_type == 'User' || message.content_attributes['external_echo'].present?
+  end
 
   def no_human_interaction?
     conversation.first_reply_created_at.nil?
   end
 
-  def route_candidate(faq)
-    embedding = embedding_service.get_embedding(candidate_text(faq))
+  def find_and_separate_duplicates(faqs)
+    duplicate_faqs = []
+    unique_faqs = []
 
-    return discard_observation(faq) if matching_record(approved_faqs, faq, embedding)
-    return discard_observation(faq) if matching_record(dismissed_suggestions_for_language, faq, embedding)
+    faqs.each do |faq|
+      combined_text = "#{faq['question']}: #{faq['answer']}"
+      embedding = Captain::Llm::EmbeddingService.new(account_id: @conversation.account_id).get_embedding(combined_text)
+      similar_faqs = find_similar_faqs(embedding)
 
-    suggestion = matching_record(open_suggestions_for_language, faq, embedding)
-    matched_content = suggestion&.slice('question', 'answer')
-    suggestion ||= assistant.faq_suggestions.create!(
-      question: faq.fetch('question'),
-      answer: faq.fetch('answer'),
-      embedding: embedding,
-      language: faq_language
-    )
-
-    attach_observation(suggestion, faq, matched_content)
-  end
-
-  def matching_record(relation, faq, embedding)
-    likely_matches(relation, embedding).find { |record| same_faq?(faq, record) }
-  end
-
-  def likely_matches(relation, embedding)
-    return [] unless relation.exists?
-
-    ApplicationRecord.transaction do
-      # Force an exact search because IVFFlat can miss matches after relation filters.
-      # SET LOCAL keeps the planner change scoped to this transaction.
-      ApplicationRecord.connection.execute('SET LOCAL enable_indexscan = off')
-      relation
-        .nearest_neighbors(:embedding, embedding, distance: 'cosine')
-        .limit(MATCH_LIMIT)
-        .select { |record| record.neighbor_distance < DISTANCE_THRESHOLD }
-    end
-  end
-
-  def same_faq?(candidate, existing_record)
-    comparison = {
-      candidate: candidate.slice('question', 'answer'),
-      existing: { question: existing_record.question, answer: existing_record.answer }
-    }
-    prompt = Captain::Llm::ConversationFaqPromptsService.same_faq
-    faq_match_model = Llm::FeatureRouter.resolve(feature: 'conversation_faq_matching', account: conversation.account)[:model]
-    response = instrument_llm_call(match_instrumentation_params(prompt, comparison, faq_match_model)) do
-      chat(model: faq_match_model)
-        .with_params(response_format: { type: 'json_object' })
-        .with_instructions(prompt)
-        .ask(comparison.to_json)
+      if similar_faqs.any?
+        duplicate_faqs << { faq: faq, similar_faqs: similar_faqs }
+      else
+        unique_faqs << faq
+      end
     end
 
-    same_faq = JSON.parse(sanitize_json_response(response.content)).fetch('same_faq')
-    raise TypeError, 'same_faq must be a boolean' unless [true, false].include?(same_faq)
-
-    same_faq
-  rescue JSON::ParserError, KeyError, TypeError, RubyLLM::Error => e
-    Rails.logger.error "FAQ match failed: #{e.message}"
-    raise
+    [duplicate_faqs, unique_faqs]
   end
 
-  def attach_observation(suggestion, faq, matched_content)
-    suggestion.with_lock do
-      next unless suggestion.open?
-      raise SuggestionChangedError if matched_content && suggestion.slice('question', 'answer') != matched_content
+  def find_similar_faqs(embedding)
+    similar_faqs = assistant
+                   .responses
+                   .nearest_neighbors(:embedding, embedding, distance: 'cosine')
+    Rails.logger.debug(similar_faqs.map { |faq| [faq.question, faq.neighbor_distance] })
+    similar_faqs.select { |record| record.neighbor_distance < DISTANCE_THRESHOLD }
+  end
 
-      existing_observation = suggestion.observations.find_by(conversation: conversation)
-      next existing_observation if existing_observation
-
-      observation = suggestion.observations.create!(
-        conversation: conversation,
-        generated_question: faq.fetch('question'),
-        generated_answer: faq.fetch('answer'),
-        language: faq_language,
-        status: :attached
+  def save_new_faqs(faqs)
+    faqs.map do |faq|
+      assistant.responses.create!(
+        question: faq['question'],
+        answer: faq['answer'],
+        status: 'pending',
+        documentable: conversation
       )
-      suggestion.update!(source_count: suggestion.source_count + 1)
-      observation
     end
   end
 
-  def discard_observation(faq)
-    Captain::FaqObservation.find_or_create_by!(
-      conversation: conversation,
-      generated_question: faq.fetch('question'),
-      generated_answer: faq.fetch('answer'),
-      language: faq_language,
-      status: :discarded
-    )
-  end
+  def log_duplicate_faqs(duplicate_faqs)
+    return if duplicate_faqs.empty?
 
-  def open_suggestions_for_language
-    assistant.faq_suggestions.where(account_id: conversation.account_id).open.by_language(faq_language)
-  end
-
-  def dismissed_suggestions_for_language
-    assistant.faq_suggestions.where(account_id: conversation.account_id).dismissed.by_language(faq_language)
-  end
-
-  def approved_faqs
-    assistant.responses.approved
-  end
-
-  def candidate_text(faq)
-    "#{faq.fetch('question')}: #{faq.fetch('answer')}"
+    Rails.logger.info "Found #{duplicate_faqs.length} duplicate FAQs:"
+    duplicate_faqs.each do |duplicate|
+      Rails.logger.info(
+        "Q: #{duplicate[:faq]['question']}\n" \
+        "A: #{duplicate[:faq]['answer']}\n\n" \
+        "Similar existing FAQs: #{duplicate[:similar_faqs].map { |f| "Q: #{f.question} A: #{f.answer}" }.join(', ')}"
+      )
+    end
   end
 
   def generate
-    response = instrument_llm_call(generation_instrumentation_params) do
+    response = instrument_llm_call(instrumentation_params) do
       chat
         .with_params(response_format: { type: 'json_object' })
         .with_instructions(system_prompt)
-        .ask(content)
+        .ask(@content)
     end
-    parse_generation_response(response.content)
+    parse_response(response.content)
   rescue RubyLLM::Error => e
     Rails.logger.error "LLM API Error: #{e.message}"
     []
   end
 
-  def generation_instrumentation_params
+  def instrumentation_params
     {
       span_name: 'llm.captain.conversation_faq',
-      model: model,
-      temperature: temperature,
-      account_id: conversation.account_id,
-      conversation_id: conversation.display_id,
+      model: @model,
+      temperature: @temperature,
+      account_id: @conversation.account_id,
+      conversation_id: @conversation.display_id,
       feature_name: 'conversation_faq',
       messages: [
         { role: 'system', content: system_prompt },
-        { role: 'user', content: content }
+        { role: 'user', content: @content }
       ],
-      metadata: { assistant_id: assistant.id, language: faq_language }
-    }
-  end
-
-  def match_instrumentation_params(prompt, comparison, faq_match_model)
-    {
-      span_name: 'llm.captain.faq_match',
-      model: faq_match_model,
-      temperature: temperature,
-      account_id: conversation.account_id,
-      conversation_id: conversation.display_id,
-      feature_name: 'conversation_faq_match',
-      messages: [
-        { role: 'system', content: prompt },
-        { role: 'user', content: comparison.to_json }
-      ],
-      metadata: { assistant_id: assistant.id, language: faq_language }
+      metadata: { assistant_id: @assistant.id }
     }
   end
 
   def system_prompt
-    Captain::Llm::ConversationFaqPromptsService.generator(language_name(faq_language))
+    account_language = @conversation.account.locale_english_name
+    Captain::Llm::SystemPromptsService.conversation_faq_generator(account_language)
   end
 
-  def faq_language
-    @faq_language ||= self.class.language_for(conversation)
-  end
-
-  def language_name(language)
-    ISO_639.find(language)&.english_name&.downcase || 'english'
-  end
-
-  def parse_generation_response(response)
+  def parse_response(response)
     return [] if response.nil?
 
     JSON.parse(sanitize_json_response(response)).fetch('faqs', [])

enterprise/app/services/captain/llm/system_prompts_service.rb

@@ -51,6 +51,62 @@ class Captain::Llm::SystemPromptsService
       PROMPT
     end
 
+    def conversation_faq_generator(language = 'english')
+      <<~SYSTEM_PROMPT_MESSAGE
+        You create high-quality FAQ candidates from resolved support conversations.
+        Only generate an FAQ when the conversation contains durable, reusable knowledge that would help many future customers.
+
+        ## Source rules
+        - The conversation history contains only customer messages and human support agent messages.
+        - Base every FAQ strictly on information stated in the human support agent messages. Do not infer, generalize, or add external knowledge.
+        - A human support agent must state every fact used in the FAQ answer. Customer messages cannot supply missing answer facts.
+        - The human support agent must provide the final answer. If the agent only greets, asks clarifying questions, asks for contact details, promises to check, shares an attachment, or transfers the conversation, return: `{"faqs":[]}`.
+        - For each FAQ, first identify the exact human support agent message that fully answers it. If no single human agent message gives a complete public answer, remove that FAQ.
+
+        ## Decision gate
+        Return `{"faqs":[]}` unless every generated FAQ can pass all of these checks:
+        1. The answer is fully stated by a human support agent, not by the customer.
+        2. The answer is a public, durable rule or procedure, not a private account action, manual review, troubleshooting session, quote, file, link, or follow-up.
+        3. The answer can be written without private identifiers, customer-specific facts, direct URLs, attachments, invoices, screenshots, or support-ticket steps.
+        4. The question would still make sense in a help center if the original conversation, customer, and agent did not exist.
+        Do not rescue a rejected conversation by rewriting it as a generic support question.
+
+        ## Return no FAQ for
+        - Spam, scams, advertisements, SEO/link-building pitches, adult/gambling/financial promotions, gibberish, abusive content, or conversations unrelated to the business being supported.
+        - Account-specific, order-specific, payment-specific, subscription-specific, login/access, verification, delivery, certificate, or troubleshooting issues, even if they could be rewritten as a general support question.
+        - Conversations that mainly hand off to a human, ask the customer to wait, request private identifiers or contact details, collect screenshots, attachments, or documents, or tell the customer to contact support for case review.
+        - Temporary workarounds, one-off exceptions, unclear answers, unresolved problems, wrong-service conversations, complaints, greetings, or abandoned conversations.
+        - Internal support workflow details, chat session rules, escalation mechanics, ticket-routing instructions, or "someone will get back to you" messages.
+        - Answers that are just a direct/private link, attachment, file, invoice, one-off quote or estimate, account-specific URL, or instructions to open a support ticket.
+        - Questions whose useful answer is "contact support", "wait for the team", "share your details", "we will check", or "this needs manual review".
+        - Questions about whether support can help with a private issue, third-party service, transaction, payment, delivery, or account problem.
+        - Pricing, policy, availability, roadmap, deadline, or legal claims unless the human support agent gives a clear and stable answer in the conversation.
+        - Questions already answered only by asking the customer for more information.
+
+        ## FAQ quality rules
+        - Prefer returning no FAQ over a weak or narrow FAQ.
+        - A good candidate teaches a generally reusable product, service, policy, setup, or process rule that another customer could use without contacting support.
+        - Generate at most one FAQ unless the human agent clearly answered multiple distinct, reusable questions.
+        - Do not create duplicate or overlapping FAQs in the same response.
+        - Questions must be general enough for a help center, not personalized to the current customer.
+        - Remove customer names, order numbers, invoice numbers, IDs, private URLs, phone numbers, emails, screenshots, attachments, and other personal or transaction-specific details.
+        - Answers must be complete, self-contained, and supported by the human agent's messages.
+
+        ## Examples
+        - Customer mentions a price or procedure, then the human agent only greets or says they will check: return `{"faqs":[]}`.
+        - Human agent shares only a private link, file, invoice, quote, screenshot, or attachment: return `{"faqs":[]}`.
+        - Human agent clearly states a public rule, such as which purchases are allowed for a program or service: generate one general FAQ.
+
+        Generate the FAQs only in the #{language}, use no other language.
+        If no suitable reusable FAQ is available, return: `{"faqs":[]}`.
+
+        Return only valid JSON in this exact structure:
+        ```json
+        { "faqs": [ { "question": "", "answer": "" } ] }
+        ```
+      SYSTEM_PROMPT_MESSAGE
+    end
+
     def notes_generator(language = 'english')
       <<~SYSTEM_PROMPT_MESSAGE
         You are a note taker looking to convert the conversation with a contact into actionable notes for the CRM.

enterprise/app/views/api/v1/accounts/captain/faq_suggestions/approve.json.jbuilder

@@ -1 +0,0 @@
-json.partial! 'api/v1/models/captain/assistant_response', formats: [:json], resource: @response

enterprise/app/views/api/v1/accounts/captain/faq_suggestions/dismiss.json.jbuilder

@@ -1 +0,0 @@
-json.partial! 'api/v1/models/captain/faq_suggestion', formats: [:json], resource: @suggestion

enterprise/app/views/api/v1/accounts/captain/faq_suggestions/index.json.jbuilder

@@ -1,10 +0,0 @@
-json.payload do
-  json.array! @suggestions do |suggestion|
-    json.partial! 'api/v1/models/captain/faq_suggestion', formats: [:json], resource: suggestion
-  end
-end
-
-json.meta do
-  json.total_count @suggestions_count
-  json.page @current_page
-end

enterprise/app/views/api/v1/accounts/captain/faq_suggestions/show.json.jbuilder

@@ -1,6 +0,0 @@
-json.partial! 'api/v1/models/captain/faq_suggestion', formats: [:json], resource: @suggestion
-json.observations do
-  json.array! @observations do |observation|
-    json.partial! 'api/v1/models/captain/faq_observation', formats: [:json], resource: observation
-  end
-end

enterprise/app/views/api/v1/accounts/captain/faq_suggestions/update.json.jbuilder

@@ -1 +0,0 @@
-json.partial! 'api/v1/models/captain/faq_suggestion', formats: [:json], resource: @suggestion

enterprise/app/views/api/v1/models/captain/_faq_observation.json.jbuilder

@@ -1,10 +0,0 @@
-json.id resource.id
-json.generated_question resource.generated_question
-json.generated_answer resource.generated_answer
-json.language resource.language
-json.status resource.status
-json.created_at resource.created_at.to_i
-json.conversation do
-  json.id resource.conversation.id
-  json.display_id resource.conversation.display_id
-end

enterprise/app/views/api/v1/models/captain/_faq_suggestion.json.jbuilder

@@ -1,12 +0,0 @@
-json.id resource.id
-json.account_id resource.account_id
-json.question resource.question
-json.answer resource.answer
-json.language resource.language
-json.source_count resource.source_count
-json.status resource.status
-json.created_at resource.created_at.to_i
-json.updated_at resource.updated_at.to_i
-json.assistant do
-  json.partial! 'api/v1/models/captain/assistant', formats: [:json], resource: resource.assistant
-end

lib/redis/redis_keys.rb

@@ -89,7 +89,6 @@ module Redis::RedisKeys
   WHATSAPP_MESSAGE_MUTEX = 'WHATSAPP_MESSAGE_CREATE_LOCK::%<inbox_id>s::%<sender_id>s'.freeze
   CRM_PROCESS_MUTEX = 'CRM_PROCESS_MUTEX::%<hook_id>s'.freeze
   CAPTAIN_DOCUMENT_SYNC_MUTEX = 'CAPTAIN_DOCUMENT_SYNC_LOCK::%<document_id>s'.freeze
-  CAPTAIN_CONVERSATION_FAQ_MUTEX = 'CAPTAIN_CONVERSATION_FAQ_LOCK::%<assistant_id>s::%<language>s'.freeze
 
   ## Auto Assignment Keys
   # Track conversation assignments to agents for rate limiting

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatwoot/chatwoot",
-  "version": "4.16.1",
+  "version": "4.16.0",
   "license": "MIT",
   "scripts": {
     "eslint": "eslint app/**/*.{js,vue}",

spec/enterprise/controllers/api/v1/accounts/captain/faq_suggestions_controller_spec.rb

@@ -1,216 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe 'Api::V1::Accounts::Captain::FaqSuggestions', type: :request do
-  let(:account) { create(:account, locale: 'en') }
-  let(:assistant) { create(:captain_assistant, account: account) }
-  let(:admin) { create(:user, account: account, role: :administrator) }
-  let(:agent) { create(:user, account: account, role: :agent) }
-  let(:inbox) { create(:inbox, account: account) }
-  let(:conversation) { create(:conversation, account: account, inbox: inbox) }
-  let(:suggestion) do
-    assistant.faq_suggestions.create!(
-      question: 'How do I enable the feature?',
-      answer: 'Turn it on in settings.',
-      source_count: 1
-    )
-  end
-
-  before do
-    suggestion.observations.create!(
-      conversation: conversation,
-      generated_question: suggestion.question,
-      generated_answer: suggestion.answer,
-      language: suggestion.language
-    )
-  end
-
-  describe 'GET /api/v1/accounts/:account_id/captain/faq_suggestions' do
-    it 'returns suggestions and their count to an administrator' do
-      get "/api/v1/accounts/#{account.id}/captain/faq_suggestions",
-          params: { assistant_id: assistant.id },
-          headers: admin.create_new_auth_token,
-          as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(response.parsed_body['payload']).to contain_exactly(
-        include('id' => suggestion.id, 'question' => suggestion.question, 'status' => 'open')
-      )
-      expect(response.parsed_body['meta']).to include('total_count' => 1)
-    end
-
-    it 'returns only suggestions backed by conversations the agent can access' do
-      create(:inbox_member, user: agent, inbox: inbox)
-      hidden_inbox = create(:inbox, account: account)
-      hidden_conversation = create(:conversation, account: account, inbox: hidden_inbox)
-      hidden_suggestion = assistant.faq_suggestions.create!(question: 'Hidden question', answer: 'Hidden answer')
-      hidden_suggestion.observations.create!(
-        conversation: hidden_conversation,
-        generated_question: hidden_suggestion.question,
-        generated_answer: hidden_suggestion.answer,
-        language: hidden_suggestion.language
-      )
-
-      get "/api/v1/accounts/#{account.id}/captain/faq_suggestions",
-          params: { assistant_id: assistant.id },
-          headers: agent.create_new_auth_token,
-          as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(response.parsed_body['payload'].pluck('id')).to contain_exactly(suggestion.id)
-      expect(response.parsed_body['meta']).to include('total_count' => 1)
-    end
-  end
-
-  describe 'GET /api/v1/accounts/:account_id/captain/faq_suggestions/:id' do
-    it 'returns the suggestion with its source conversation' do
-      get "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}",
-          headers: admin.create_new_auth_token,
-          as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(response.parsed_body).to include('id' => suggestion.id, 'question' => suggestion.question)
-      expect(response.parsed_body['observations']).to contain_exactly(
-        include('conversation' => include('id' => conversation.id, 'display_id' => conversation.display_id))
-      )
-    end
-
-    it 'returns only source conversations the agent can access' do
-      create(:inbox_member, user: agent, inbox: inbox)
-      hidden_conversation = create(:conversation, account: account, inbox: create(:inbox, account: account))
-      suggestion.observations.create!(
-        conversation: hidden_conversation,
-        generated_question: suggestion.question,
-        generated_answer: suggestion.answer,
-        language: suggestion.language
-      )
-
-      get "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}",
-          headers: agent.create_new_auth_token,
-          as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(response.parsed_body['observations'].pluck('conversation').pluck('id')).to contain_exactly(conversation.id)
-    end
-  end
-
-  describe 'PATCH /api/v1/accounts/:account_id/captain/faq_suggestions/:id' do
-    it 'lets an administrator edit an open suggestion' do
-      patch "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}",
-            params: { faq_suggestion: { question: 'Updated question' } },
-            headers: admin.create_new_auth_token,
-            as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(response.parsed_body['question']).to eq('Updated question')
-      expect(suggestion.reload.question).to eq('Updated question')
-    end
-
-    it 'lets an agent edit an accessible suggestion' do
-      create(:inbox_member, user: agent, inbox: inbox)
-
-      patch "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}",
-            params: { faq_suggestion: { question: 'Updated question' } },
-            headers: agent.create_new_auth_token,
-            as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(suggestion.reload.question).to eq('Updated question')
-    end
-
-    it 'does not let an agent edit an inaccessible suggestion' do
-      patch "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}",
-            params: { faq_suggestion: { question: 'Updated question' } },
-            headers: agent.create_new_auth_token,
-            as: :json
-
-      expect(response).to have_http_status(:not_found)
-      expect(suggestion.reload.question).to eq('How do I enable the feature?')
-    end
-  end
-
-  describe 'POST /api/v1/accounts/:account_id/captain/faq_suggestions/:id/approve' do
-    it 'lets an administrator approve an edited suggestion as an FAQ' do
-      expect do
-        post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/approve",
-             params: { faq_suggestion: { answer: 'Enable it in account settings.' } },
-             headers: admin.create_new_auth_token,
-             as: :json
-      end.to change(assistant.responses.approved, :count).by(1)
-
-      expect(response).to have_http_status(:success)
-      expect(response.parsed_body['answer']).to eq('Enable it in account settings.')
-      expect(suggestion.reload).to be_approved
-      expect(suggestion.observations.pluck(:conversation_id)).to contain_exactly(conversation.id)
-    end
-
-    it 'lets an agent approve an accessible suggestion' do
-      create(:inbox_member, user: agent, inbox: inbox)
-
-      expect do
-        post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/approve",
-             headers: agent.create_new_auth_token,
-             as: :json
-      end.to change(assistant.responses.approved, :count).by(1)
-
-      expect(response).to have_http_status(:success)
-      expect(suggestion.reload).to be_approved
-    end
-
-    it 'approves a suggestion written in a language other than the account locale' do
-      suggestion.update!(language: 'pt')
-
-      expect do
-        post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/approve",
-             headers: admin.create_new_auth_token,
-             as: :json
-      end.to change(assistant.responses.approved, :count).by(1)
-
-      expect(response).to have_http_status(:success)
-      expect(suggestion.reload).to be_approved
-    end
-
-    it 'does not let an agent approve an inaccessible suggestion' do
-      expect do
-        post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/approve",
-             headers: agent.create_new_auth_token,
-             as: :json
-      end.not_to change(assistant.responses, :count)
-
-      expect(response).to have_http_status(:not_found)
-      expect(suggestion.reload).to be_open
-    end
-  end
-
-  describe 'POST /api/v1/accounts/:account_id/captain/faq_suggestions/:id/dismiss' do
-    it 'lets an administrator dismiss an open suggestion without creating an FAQ' do
-      expect do
-        post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/dismiss",
-             headers: admin.create_new_auth_token,
-             as: :json
-      end.not_to change(assistant.responses, :count)
-
-      expect(response).to have_http_status(:success)
-      expect(suggestion.reload).to be_dismissed
-    end
-
-    it 'lets an agent dismiss an accessible suggestion' do
-      create(:inbox_member, user: agent, inbox: inbox)
-
-      post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/dismiss",
-           headers: agent.create_new_auth_token,
-           as: :json
-
-      expect(response).to have_http_status(:success)
-      expect(suggestion.reload).to be_dismissed
-    end
-
-    it 'does not let an agent dismiss an inaccessible suggestion' do
-      post "/api/v1/accounts/#{account.id}/captain/faq_suggestions/#{suggestion.id}/dismiss",
-           headers: agent.create_new_auth_token,
-           as: :json
-
-      expect(response).to have_http_status(:not_found)
-      expect(suggestion.reload).to be_open
-    end
-  end
-end

spec/enterprise/jobs/captain/llm/conversation_faq_job_spec.rb

@@ -1,57 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Captain::Llm::ConversationFaqJob, type: :job do
-  let(:account) { create(:account) }
-  let(:inbox) { create(:inbox, account: account) }
-  let(:assistant) { create(:captain_assistant, account: account, config: { feature_faq: true }) }
-  let(:conversation) { create(:conversation, account: account, inbox: inbox, first_reply_created_at: Time.zone.now) }
-  let(:faq_service) { instance_double(Captain::Llm::ConversationFaqService, generate_suggestions: []) }
-  let(:lock_manager) { instance_double(Redis::LockManager, lock: true, unlock: true) }
-  let(:lock_key) { "CAPTAIN_CONVERSATION_FAQ_LOCK::#{assistant.id}::en" }
-
-  before do
-    create(:captain_inbox, inbox: inbox, captain_assistant: assistant)
-    conversation.update!(status: :resolved)
-    allow(Redis::LockManager).to receive(:new).and_return(lock_manager)
-    allow(Captain::Llm::ConversationFaqService).to receive(:new).and_return(faq_service)
-  end
-
-  describe '#perform' do
-    it 'uses the assistant captured when the job was enqueued' do
-      replacement_assistant = create(:captain_assistant, account: account, config: { feature_faq: true })
-      inbox.captain_inbox.update!(captain_assistant: replacement_assistant)
-
-      expect(inbox.reload.captain_assistant).to eq(replacement_assistant)
-      expect(Captain::Llm::ConversationFaqService).to receive(:new)
-        .with(assistant, conversation)
-        .and_return(faq_service)
-      expect(faq_service).to receive(:generate_suggestions)
-
-      described_class.perform_now(conversation, assistant)
-    end
-
-    it 'locks FAQ grouping for the assistant and normalized language' do
-      conversation.update!(additional_attributes: { conversation_language: 'pt-BR' })
-      expected_key = "CAPTAIN_CONVERSATION_FAQ_LOCK::#{assistant.id}::pt"
-
-      expect(lock_manager).to receive(:lock).with(expected_key, described_class::LOCK_TIMEOUT).and_return(true)
-      expect(lock_manager).to receive(:unlock).with(expected_key)
-
-      described_class.perform_now(conversation, assistant)
-    end
-
-    context 'when another job holds the grouping lock' do
-      before do
-        allow(lock_manager).to receive(:lock).with(lock_key, described_class::LOCK_TIMEOUT).and_return(false)
-      end
-
-      it 'does not generate suggestions concurrently' do
-        expect(Captain::Llm::ConversationFaqService).not_to receive(:new)
-
-        expect do
-          described_class.new.perform(conversation, assistant)
-        end.to raise_error(MutexApplicationJob::LockAcquisitionError)
-      end
-    end
-  end
-end

spec/enterprise/listeners/captain_listener_spec.rb

@@ -29,7 +29,7 @@ describe CaptainListener do
           .to receive(:new)
           .with(assistant, conversation)
           .and_return(instance_double(Captain::Llm::ContactNotesService, generate_and_update_notes: nil))
-        expect(Captain::Llm::ConversationFaqJob).not_to receive(:perform_later)
+        expect(Captain::Llm::ConversationFaqService).not_to receive(:new)
 
         listener.conversation_resolved(event)
       end
@@ -42,8 +42,11 @@ describe CaptainListener do
         assistant.save!
       end
 
-      it 'enqueues FAQ suggestion generation' do
-        expect(Captain::Llm::ConversationFaqJob).to receive(:perform_later).with(conversation, assistant)
+      it 'generates and deduplicates FAQs' do
+        expect(Captain::Llm::ConversationFaqService)
+          .to receive(:new)
+          .with(assistant, conversation)
+          .and_return(instance_double(Captain::Llm::ConversationFaqService, generate_and_deduplicate: false))
         expect(Captain::Llm::ContactNotesService).not_to receive(:new)
 
         listener.conversation_resolved(event)

spec/enterprise/services/captain/llm/conversation_faq_prompts_service_spec.rb

@@ -1,13 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Captain::Llm::ConversationFaqPromptsService do
-  describe '.generator' do
-    it 'allows a complete FAQ answer to use several related agent messages' do
-      prompt = described_class.generator
-
-      expect(prompt).to include('message or messages that together provide a complete public answer')
-      expect(prompt).to include('Combine facts only across related agent messages')
-      expect(prompt).not_to include('no single human agent message')
-    end
-  end
-end

spec/enterprise/services/captain/llm/conversation_faq_service_spec.rb

@@ -2,7 +2,7 @@ require 'rails_helper'
 
 RSpec.describe Captain::Llm::ConversationFaqService do
   let(:captain_assistant) { create(:captain_assistant) }
-  let(:conversation) { create(:conversation, account: captain_assistant.account, first_reply_created_at: Time.zone.now) }
+  let(:conversation) { create(:conversation, first_reply_created_at: Time.zone.now) }
   let(:service) { described_class.new(captain_assistant, conversation) }
   let(:embedding_service) { instance_double(Captain::Llm::EmbeddingService) }
   let(:mock_chat) { instance_double(RubyLLM::Chat) }
@@ -15,8 +15,6 @@ RSpec.describe Captain::Llm::ConversationFaqService do
   let(:mock_response) do
     instance_double(RubyLLM::Message, content: { faqs: sample_faqs }.to_json)
   end
-  let(:embedding_one) { [1.0] + Array.new(1535, 0.0) }
-  let(:embedding_two) { [0.0, 1.0] + Array.new(1534, 0.0) }
 
   before do
     create(:installation_config, name: 'CAPTAIN_OPEN_AI_API_KEY', value: 'test-key')
@@ -28,10 +26,11 @@ RSpec.describe Captain::Llm::ConversationFaqService do
     allow(mock_chat).to receive(:ask).and_return(mock_response)
   end
 
-  describe '#generate_suggestions' do
+  describe '#generate_and_deduplicate' do
     context 'when successful' do
       before do
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two)
+        allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
+        allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
       end
 
       it 'uses the conversation FAQ generation feature model' do
@@ -39,7 +38,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
           model: Llm::Models.default_model_for('conversation_faq_generation')
         ).and_return(mock_chat)
 
-        described_class.new(captain_assistant, conversation).generate_suggestions
+        described_class.new(captain_assistant, conversation).generate_and_deduplicate
       end
 
       it 'uses the conversation FAQ default ahead of the legacy global installation model' do
@@ -49,7 +48,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
           model: Llm::Models.default_model_for('conversation_faq_generation')
         ).and_return(mock_chat)
 
-        described_class.new(captain_assistant, conversation).generate_suggestions
+        described_class.new(captain_assistant, conversation).generate_and_deduplicate
       end
 
       it 'keeps account conversation FAQ model overrides ahead of the feature default' do
@@ -58,7 +57,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
 
         expect(RubyLLM).to receive(:chat).with(model: 'gpt-4.1-mini').and_return(mock_chat)
 
-        described_class.new(captain_assistant, conversation).generate_suggestions
+        described_class.new(captain_assistant, conversation).generate_and_deduplicate
       end
 
       it 'resolves the feature model from the conversation account' do
@@ -67,7 +66,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
           account: conversation.account
         ).and_call_original
 
-        described_class.new(captain_assistant, conversation).generate_suggestions
+        described_class.new(captain_assistant, conversation).generate_and_deduplicate
       end
 
       it 'sends only customer and human support agent messages to the LLM' do
@@ -85,7 +84,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
         create(:message, conversation: conversation, account: conversation.account, inbox: conversation.inbox,
                          message_type: :activity, content: 'Activity message')
 
-        service.generate_suggestions
+        service.generate_and_deduplicate
 
         expected_content = satisfy do |content|
           content.include?('User: Customer question') &&
@@ -105,7 +104,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
                          sender: nil, message_type: :outgoing, content: 'Human replied from the native app',
                          content_attributes: { external_echo: true })
 
-        service.generate_suggestions
+        service.generate_and_deduplicate
 
         expected_content = satisfy do |content|
           content.include?('User: Customer asks in a native channel') &&
@@ -134,27 +133,22 @@ RSpec.describe Captain::Llm::ConversationFaqService do
           block.call
         end
 
-        service.generate_suggestions
+        service.generate_and_deduplicate
       end
 
-      it 'creates suggestions instead of trusted FAQs for valid conversation content' do
+      it 'creates new FAQs for valid conversation content' do
         expect do
-          service.generate_suggestions
-        end.to change(captain_assistant.faq_suggestions, :count).by(2)
-        expect(Captain::FaqObservation.count).to eq(2)
-        expect(captain_assistant.responses.count).to be_zero
+          service.generate_and_deduplicate
+        end.to change(captain_assistant.responses, :count).by(2)
       end
 
-      it 'saves open suggestions with one attached source each' do
-        service.generate_suggestions
+      it 'saves FAQs with pending status linked to conversation' do
+        service.generate_and_deduplicate
         expect(
-          captain_assistant.faq_suggestions.pluck(:question, :answer, :status, :source_count, :language)
+          captain_assistant.responses.pluck(:question, :answer, :status, :documentable_id)
         ).to contain_exactly(
-          ['What is the purpose?', 'To help users.', 'open', 1, 'en'],
-          ['How does it work?', 'Through AI.', 'open', 1, 'en']
-        )
-        expect(Captain::FaqObservation.attached.pluck(:conversation_id, :language)).to contain_exactly(
-          [conversation.id, 'en'], [conversation.id, 'en']
+          ['What is the purpose?', 'To help users.', 'pending', conversation.id],
+          ['How does it work?', 'Through AI.', 'pending', conversation.id]
         )
       end
     end
@@ -163,309 +157,37 @@ RSpec.describe Captain::Llm::ConversationFaqService do
       let(:conversation) { create(:conversation) }
 
       it 'returns an empty array without generating FAQs' do
-        expect(service.generate_suggestions).to eq([])
+        expect(service.generate_and_deduplicate).to eq([])
       end
 
       it 'does not call the LLM API' do
         expect(RubyLLM).not_to receive(:chat)
-        service.generate_suggestions
+        service.generate_and_deduplicate
       end
     end
 
     context 'when finding duplicates' do
       let(:existing_response) do
-        create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account,
-                                            question: 'Similar question', answer: 'Similar answer', embedding: embedding_one)
+        create(:captain_assistant_response, assistant: captain_assistant, question: 'Similar question', answer: 'Similar answer')
       end
-      let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
-
-      before do
-        existing_response
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? match_response : mock_response
-        end
-      end
-
-      it 'discards candidates the LLM confirms are covered by an approved FAQ' do
-        expect do
-          service.generate_suggestions
-        end.to change(Captain::FaqObservation.discarded, :count).by(2)
-        expect(captain_assistant.faq_suggestions.count).to be_zero
-      end
-
-      it 'uses the conversation FAQ matching feature model' do
-        expect(RubyLLM).to receive(:chat).with(
-          model: Llm::Models.default_model_for('conversation_faq_matching')
-        ).at_least(:once).and_return(mock_chat)
-
-        service.generate_suggestions
-      end
-
-      it 'uses the account model override for conversation FAQ matching' do
-        conversation.account.update!(captain_models: { 'conversation_faq_matching' => 'gpt-5-mini' })
-
-        expect(RubyLLM).to receive(:chat).with(model: 'gpt-5-mini').at_least(:once).and_return(mock_chat)
-
-        service.generate_suggestions
-      end
-
-      it 'resolves the matching feature model from the conversation account' do
-        allow(Llm::FeatureRouter).to receive(:resolve).and_call_original
-        expect(Llm::FeatureRouter).to receive(:resolve).with(
-          feature: 'conversation_faq_matching',
-          account: conversation.account
-        ).and_call_original
-
-        service.generate_suggestions
-      end
-    end
-
-    context 'when FAQ comparison cannot be completed' do
-      let(:existing_response) do
-        create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account,
-                                            question: 'Similar question', answer: 'Similar answer', embedding: embedding_one)
-      end
-      let(:comparison_response) { instance_double(RubyLLM::Message, content: comparison_response_content) }
-      let(:comparison_response_content) { 'invalid json' }
-
-      before do
-        existing_response
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? comparison_response : mock_response
-        end
-        allow(Rails.logger).to receive(:error)
-      end
-
-      it 'raises when the comparison response is malformed' do
-        expect do
-          service.generate_suggestions
-        end.to raise_error(JSON::ParserError)
-        expect(captain_assistant.faq_suggestions.count).to be_zero
-      end
-
-      context 'when the response omits the comparison result' do
-        let(:comparison_response_content) { {}.to_json }
-
-        it 'raises instead of treating the response as a non-match' do
-          expect do
-            service.generate_suggestions
-          end.to raise_error(KeyError)
-          expect(captain_assistant.faq_suggestions.count).to be_zero
-        end
-      end
-
-      context 'when the comparison result is not a boolean' do
-        let(:comparison_response_content) { { same_faq: 'false' }.to_json }
-
-        it 'raises instead of treating the response as a non-match' do
-          expect do
-            service.generate_suggestions
-          end.to raise_error(TypeError, 'same_faq must be a boolean')
-          expect(captain_assistant.faq_suggestions.count).to be_zero
-        end
-      end
-
-      context 'when the comparison provider fails' do
-        before do
-          allow(mock_chat).to receive(:ask) do |input|
-            raise RubyLLM::Error.new(nil, 'API Error') if input.start_with?('{')
-
-            mock_response
-          end
-        end
-
-        it 'raises instead of treating the failure as a non-match' do
-          expect do
-            service.generate_suggestions
-          end.to raise_error(RubyLLM::Error)
-          expect(captain_assistant.faq_suggestions.count).to be_zero
-        end
-      end
-    end
-
-    context 'when the classifier confirms a non-match' do
-      let(:sample_faqs) { [{ 'question' => 'How can I use the feature?', 'answer' => 'Enable it in settings.' }] }
-      let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: false }.to_json) }
-
-      before do
-        create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account,
-                                            question: 'How do I enable the feature?', answer: 'Turn it on in settings.',
-                                            embedding: embedding_one)
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? match_response : mock_response
-        end
-      end
-
-      it 'creates a new suggestion' do
-        expect do
-          service.generate_suggestions
-        end.to change(captain_assistant.faq_suggestions, :count).by(1)
-      end
-    end
-
-    context 'when an open suggestion is the same FAQ' do
-      let(:sample_faqs) { [{ 'question' => 'How can I use the feature?', 'answer' => 'Enable it in settings.' }] }
-      let(:existing_suggestion) do
-        captain_assistant.faq_suggestions.create!(
-          question: 'How do I enable the feature?',
-          answer: 'Turn it on in settings.',
-          embedding: embedding_one
-        ).tap do |suggestion|
-          suggestion.observations.create!(
-            conversation: create(:conversation, account: captain_assistant.account),
-            generated_question: suggestion.question,
-            generated_answer: suggestion.answer,
-            language: suggestion.language
-          )
-          suggestion.update!(source_count: suggestion.observations.attached.count)
-        end
-      end
-      let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
-
-      before do
-        existing_suggestion
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? match_response : mock_response
-        end
-      end
-
-      it 'attaches the observation and increments the source count' do
-        expect do
-          service.generate_suggestions
-        end.to change(existing_suggestion.observations, :count).by(1)
-        expect(existing_suggestion.reload.source_count).to eq(2)
-        expect(captain_assistant.faq_suggestions.count).to eq(1)
-      end
-
-      it 'does not attach the observation when the suggestion changes after classification' do
-        allow(mock_chat).to receive(:ask) do |input|
-          if input.start_with?('{')
-            existing_suggestion.update!(question: 'Edited after classification started')
-            match_response
-          else
-            mock_response
-          end
-        end
-
-        expect do
-          service.generate_suggestions
-        end.to raise_error(described_class::SuggestionChangedError)
-        expect(existing_suggestion.observations.count).to eq(1)
-        expect(existing_suggestion.reload.source_count).to eq(1)
-      end
-    end
-
-    context 'when a similar open suggestion uses another language' do
-      let(:sample_faqs) { [{ 'question' => 'Como ativo o recurso?', 'answer' => 'Ative nas configuracoes.' }] }
-      let!(:existing_suggestion) do
-        captain_assistant.faq_suggestions.create!(question: 'How do I enable the feature?', answer: 'Turn it on in settings.',
-                                                  embedding: embedding_one, language: 'en', source_count: 1)
-      end
-
-      before do
-        conversation.update!(additional_attributes: { conversation_language: 'pt-BR' })
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-      end
-
-      it 'creates a separate suggestion in the conversation language' do
-        expect do
-          service.generate_suggestions
-        end.to change(captain_assistant.faq_suggestions, :count).by(1)
-
-        expect(captain_assistant.faq_suggestions.pluck(:language)).to contain_exactly('en', 'pt')
-        expect(existing_suggestion.reload.source_count).to eq(1)
-      end
-    end
-
-    context 'when an open suggestion uses another locale variant of the same language' do
-      let(:account) { create(:account, locale: 'pt_BR') }
-      let(:captain_assistant) { create(:captain_assistant, account: account) }
-      let(:conversation) { create(:conversation, account: account, first_reply_created_at: Time.zone.now) }
-      let(:sample_faqs) { [{ 'question' => 'Como ativo o recurso?', 'answer' => 'Ative nas configuracoes.' }] }
-      let(:existing_suggestion) do
-        captain_assistant.faq_suggestions.create!(
-          question: 'Como habilito o recurso?',
-          answer: 'Ative nas configuracoes.',
-          embedding: embedding_one,
-          language: 'pt',
-          source_count: 1
+      let(:similar_neighbor) do
+        OpenStruct.new(
+          id: 1,
+          question: existing_response.question,
+          answer: existing_response.answer,
+          neighbor_distance: 0.1
         )
       end
-      let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
 
       before do
-        existing_suggestion
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? match_response : mock_response
-        end
+        allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
+        allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([similar_neighbor])
       end
 
-      it 'attaches the observation to the existing base-language suggestion' do
+      it 'filters out duplicate FAQs based on embedding similarity' do
         expect do
-          service.generate_suggestions
-        end.to change(existing_suggestion.observations, :count).by(1)
-
-        expect(existing_suggestion.reload.source_count).to eq(2)
-        expect(captain_assistant.faq_suggestions.count).to eq(1)
-        expect(existing_suggestion.observations.last.language).to eq('pt')
-      end
-    end
-
-    context 'when a similar approved FAQ uses another language' do
-      let(:sample_faqs) { [{ 'question' => 'Como ativo o recurso?', 'answer' => 'Ative nas configuracoes.' }] }
-      let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
-
-      before do
-        create(:captain_assistant_response, assistant: captain_assistant, account: captain_assistant.account,
-                                            question: 'How do I enable the feature?', answer: 'Turn it on in settings.',
-                                            embedding: embedding_one)
-        conversation.update!(additional_attributes: { conversation_language: 'pt-BR' })
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? match_response : mock_response
-        end
-      end
-
-      it 'deduplicates against the approved FAQ' do
-        expect do
-          service.generate_suggestions
-        end.to change(Captain::FaqObservation.discarded, :count).by(1)
-        expect(captain_assistant.faq_suggestions.count).to be_zero
-      end
-    end
-
-    context 'when conversation and account locales share a base language' do
-      let(:account) { create(:account, locale: 'pt_BR') }
-      let(:captain_assistant) { create(:captain_assistant, account: account) }
-      let(:conversation) do
-        create(:conversation, account: account, first_reply_created_at: Time.zone.now,
-                              additional_attributes: { conversation_language: 'pt' })
-      end
-      let!(:existing_response) do
-        create(:captain_assistant_response, assistant: captain_assistant, account: account,
-                                            question: 'Como ativo o recurso?', answer: 'Ative nas configuracoes.',
-                                            embedding: embedding_one)
-      end
-      let(:match_response) { instance_double(RubyLLM::Message, content: { same_faq: true }.to_json) }
-
-      before do
-        existing_response
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one)
-        allow(mock_chat).to receive(:ask) do |input|
-          input.start_with?('{') ? match_response : mock_response
-        end
-      end
-
-      it 'deduplicates against approved FAQs in the same base language' do
-        expect do
-          service.generate_suggestions
-        end.to change(Captain::FaqObservation.discarded, :count).by(2)
-        expect(captain_assistant.faq_suggestions.count).to be_zero
+          service.generate_and_deduplicate
+        end.not_to change(captain_assistant.responses, :count)
       end
     end
 
@@ -477,7 +199,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
 
       it 'returns empty array and logs the error' do
         expect(Rails.logger).to receive(:error).with('LLM API Error: API Error')
-        expect(service.generate_suggestions).to eq([])
+        expect(service.generate_and_deduplicate).to eq([])
       end
     end
 
@@ -492,7 +214,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
 
       it 'handles JSON parsing errors gracefully' do
         expect(Rails.logger).to receive(:error).with(/Error in parsing GPT processed response:/)
-        expect(service.generate_suggestions).to eq([])
+        expect(service.generate_and_deduplicate).to eq([])
       end
     end
 
@@ -506,7 +228,7 @@ RSpec.describe Captain::Llm::ConversationFaqService do
       end
 
       it 'returns empty array' do
-        expect(service.generate_suggestions).to eq([])
+        expect(service.generate_and_deduplicate).to eq([])
       end
     end
   end
@@ -514,44 +236,22 @@ RSpec.describe Captain::Llm::ConversationFaqService do
   describe 'language handling' do
     context 'when conversation has different language' do
       let(:account) { create(:account, locale: 'fr') }
-      let(:captain_assistant) { create(:captain_assistant, account: account) }
       let(:conversation) do
         create(:conversation, account: account, first_reply_created_at: Time.zone.now)
       end
 
       before do
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two)
+        allow(embedding_service).to receive(:get_embedding).and_return([0.1, 0.2, 0.3])
+        allow(captain_assistant.responses).to receive(:nearest_neighbors).and_return([])
       end
 
       it 'uses account language for system prompt' do
-        expect(Captain::Llm::ConversationFaqPromptsService).to receive(:generator)
+        expect(Captain::Llm::SystemPromptsService).to receive(:conversation_faq_generator)
           .with('french')
           .at_least(:once)
           .and_call_original
 
-        service.generate_suggestions
-      end
-    end
-
-    context 'when conversation language differs from account language' do
-      let(:account) { create(:account, locale: 'en') }
-      let(:captain_assistant) { create(:captain_assistant, account: account) }
-      let(:conversation) do
-        create(:conversation, account: account, first_reply_created_at: Time.zone.now,
-                              additional_attributes: { conversation_language: 'pt-BR' })
-      end
-
-      before do
-        allow(embedding_service).to receive(:get_embedding).and_return(embedding_one, embedding_two)
-      end
-
-      it 'uses the conversation language for the system prompt' do
-        expect(Captain::Llm::ConversationFaqPromptsService).to receive(:generator)
-          .with('portuguese')
-          .at_least(:once)
-          .and_call_original
-
-        service.generate_suggestions
+        service.generate_and_deduplicate
       end
     end
   end

spec/lib/llm/models_spec.rb

@@ -26,10 +26,9 @@ RSpec.describe Llm::Models do
       end
     end
 
-    it 'routes each FAQ operation independently' do
+    it 'routes document and conversation FAQ generation independently' do
       expect(described_class.default_model_for('document_faq_generation')).to eq('gpt-4.1-mini')
       expect(described_class.default_model_for('conversation_faq_generation')).to eq('gpt-5.2')
-      expect(described_class.default_model_for('conversation_faq_matching')).to eq('gpt-4.1-mini')
     end
   end
 

spec/models/data_import_spec.rb

@@ -64,4 +64,36 @@ RSpec.describe DataImport do
       expect(data_import.abandoned_at).to be_nil
     end
   end
+
+  describe '#stalled?' do
+    let(:account) { create(:account) }
+
+    it 'identifies active Intercom imports without updates for fifteen minutes', :aggregate_failures do
+      freeze_time do
+        processing_import = create(:data_import, :intercom, account: account, status: :processing)
+        pending_import = create(:data_import, :intercom, account: account, status: :pending)
+        processing_import.update!(updated_at: 15.minutes.ago)
+        pending_import.update!(updated_at: 15.minutes.ago)
+
+        expect(processing_import.reload).to be_stalled
+        expect(pending_import.reload).to be_stalled
+      end
+    end
+
+    it 'does not identify recent or terminal Intercom imports as stalled', :aggregate_failures do
+      recent_import = create(:data_import, :intercom, account: account, status: :processing)
+      completed_import = create(:data_import, :intercom, account: account, status: :completed)
+      completed_import.update!(updated_at: 1.hour.ago)
+
+      expect(recent_import).not_to be_stalled
+      expect(completed_import.reload).not_to be_stalled
+    end
+
+    it 'does not identify legacy imports as stalled' do
+      legacy_import = create(:data_import, account: account, status: :processing)
+      legacy_import.update!(updated_at: 1.hour.ago)
+
+      expect(legacy_import.reload).not_to be_stalled
+    end
+  end
 end

spec/requests/api/v1/accounts/data_imports_spec.rb

@@ -209,6 +209,63 @@ RSpec.describe 'Data Imports API', type: :request do
     end
   end
 
+  describe 'POST /api/v1/accounts/:account_id/data_imports/:id/retry' do
+    let(:data_import) { create(:data_import, :intercom, account: account, status: :processing, started_at: 2.hours.ago) }
+
+    it 'resumes a stalled import with a new run identifier while preserving progress', :aggregate_failures do
+      started_at = data_import.started_at
+      data_import.update!(
+        cursor: { 'conversations' => { 'starting_after' => 'cursor-1' } },
+        stats: { 'conversations' => { 'imported' => 20 } },
+        source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'previous-run' },
+        updated_at: 16.minutes.ago
+      )
+
+      expect do
+        post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
+             headers: admin.create_new_auth_token,
+             as: :json
+      end.to have_enqueued_job(DataImports::Intercom::ImportJob).with(data_import, a_kind_of(String))
+
+      expect(response).to have_http_status(:ok)
+      expect(response.parsed_body).to include('status' => 'pending', 'stalled' => false)
+      expect(data_import.reload.started_at).to eq(started_at)
+      expect(data_import.cursor.dig('conversations', 'starting_after')).to eq('cursor-1')
+      expect(data_import.stats.dig('conversations', 'imported')).to eq(20)
+      expect(data_import.active_intercom_import_run_id).not_to eq('previous-run')
+    end
+
+    it 'rejects duplicate retries after the import becomes active again' do
+      data_import.update!(updated_at: 16.minutes.ago)
+      post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
+           headers: admin.create_new_auth_token,
+           as: :json
+
+      expect do
+        post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
+             headers: admin.create_new_auth_token,
+             as: :json
+      end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
+
+      expect(response).to have_http_status(:unprocessable_entity)
+      expect(response.parsed_body['message']).to eq('This Intercom import is no longer stalled.')
+    end
+
+    it 'rejects retry while another Intercom import is active' do
+      data_import.update!(updated_at: 16.minutes.ago)
+      create(:data_import, :intercom, account: account, status: :processing)
+
+      expect do
+        post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
+             headers: admin.create_new_auth_token,
+             as: :json
+      end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
+
+      expect(response).to have_http_status(:unprocessable_entity)
+      expect(response.parsed_body['message']).to eq('Another Intercom import is already in progress.')
+    end
+  end
+
   describe 'POST /api/v1/accounts/:account_id/data_imports/:id/abandon' do
     let(:data_import) { create(:data_import, :intercom, account: account) }
 
@@ -283,6 +340,7 @@ RSpec.describe 'Data Imports API', type: :request do
         'id' => data_import.id,
         'name' => 'July Intercom migration',
         'source_provider' => 'intercom',
+        'stalled' => false,
         'import_errors_count' => 1,
         'skip_logs_count' => 1
       )

spec/services/data_imports/intercom/importer_spec.rb

@@ -66,12 +66,12 @@ RSpec.describe DataImports::Intercom::Importer do
   before do
     account.enable_features!('data_import')
     allow(DataImports::Intercom::Client).to receive(:new).with(access_token: 'intercom-token').and_return(client)
-    allow(client).to receive(:list_contacts).with(starting_after: nil).and_return(
+    allow(client).to receive(:list_contacts).with(starting_after: nil, per_page: 50).and_return(
       'data' => [contact_payload],
       'total_count' => 1,
       'pages' => { 'next' => nil }
     )
-    allow(client).to receive(:list_conversations).with(starting_after: nil).and_return(
+    allow(client).to receive(:list_conversations).with(starting_after: nil, per_page: 10).and_return(
       'conversations' => [{ 'id' => 'conversation_1' }],
       'total_count' => 1,
       'pages' => { 'next' => nil }
@@ -188,14 +188,49 @@ RSpec.describe DataImports::Intercom::Importer do
     expect(item.metadata['message_total_contribution']).to eq(3)
   end
 
+  it 'uses smaller conversation pages while retaining the contact page size' do
+    importer = described_class.new(data_import: data_import)
+
+    importer.import_contacts_page
+    importer.import_conversations_page
+
+    expect(client).to have_received(:list_contacts).with(starting_after: nil, per_page: 50)
+    expect(client).to have_received(:list_conversations).with(starting_after: nil, per_page: 10)
+  end
+
   it 'reconciles imported message stats from same-run mappings on retry' do
     described_class.new(data_import: data_import).import_conversations_page
     stats = data_import.reload.stats.deep_dup
     stats['messages']['imported'] = 0
     data_import.update!(stats: stats)
+    importer = described_class.new(data_import: data_import)
+    expect(importer).to receive(:reconcile_message_stats).once.ordered.and_call_original
+    expect(importer).to receive(:update_cursor).with('conversations', nil).once.ordered.and_call_original
 
+    importer.import_conversations_page
+
+    expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
+  end
+
+  it 'retries a query timeout during deferred message stats reconciliation', :aggregate_failures do
     described_class.new(data_import: data_import).import_conversations_page
+    stats = data_import.reload.stats.deep_dup
+    stats['messages']['imported'] = 0
+    data_import.update!(stats: stats)
+    importer = described_class.new(data_import: data_import)
+    reconciliation_attempts = 0
+    allow(importer).to receive(:sleep)
+    allow(importer).to receive(:reconcile_message_stats).and_wrap_original do |method|
+      reconciliation_attempts += 1
+      raise ActiveRecord::QueryCanceled, 'statement timeout' if reconciliation_attempts == 1
 
+      method.call
+    end
+
+    importer.import_conversations_page
+
+    expect(reconciliation_attempts).to eq(2)
+    expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
     expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
   end
 
@@ -267,7 +302,7 @@ RSpec.describe DataImports::Intercom::Importer do
     it 'stops an in-flight page when a newer import run takes over', :aggregate_failures do
       run_id = 'intercom-run-1'
       data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
-      allow(client).to receive(:list_conversations).with(starting_after: nil).and_return(
+      allow(client).to receive(:list_conversations).with(starting_after: nil, per_page: 10).and_return(
         'conversations' => [{ 'id' => 'conversation_1' }, { 'id' => 'conversation_2' }],
         'pages' => { 'next' => { 'starting_after' => 'next-conversation-cursor' } }
       )
@@ -285,6 +320,90 @@ RSpec.describe DataImports::Intercom::Importer do
       expect(data_import.reload.cursor.dig('conversations', 'starting_after')).to be_nil
     end
 
+    it 'heartbeats at most once per minute while processing conversation parts' do
+      freeze_time do
+        started_at = Time.current
+        long_conversation = conversation_payload.deep_dup
+        template_part = long_conversation.dig('conversation_parts', 'conversation_parts').first
+        long_conversation['conversation_parts']['conversation_parts'] = Array.new(5) do |index|
+          template_part.merge('id' => "part_#{index + 1}")
+        end
+        allow(client).to receive(:retrieve_conversation).with('conversation_1').and_return(long_conversation)
+        heartbeat_times = []
+        allow(data_import).to receive(:touch).and_wrap_original do |method|
+          heartbeat_times << Time.current
+          method.call
+        end
+        importer = described_class.new(data_import: data_import)
+        allow(importer).to receive(:create_message) { travel 30.seconds }
+
+        importer.import_conversations_page
+
+        expect(heartbeat_times).to eq([started_at + 1.minute, started_at + 2.minutes])
+      end
+    end
+
+    it 'stops parts without heartbeating or persisting stale stats when a newer run takes over', :aggregate_failures do
+      freeze_time do
+        run_id = 'intercom-run-1'
+        data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
+        allow(data_import).to receive(:touch).and_call_original
+        importer = described_class.new(data_import: data_import, run_id: run_id)
+        allow(importer).to receive(:create_message) do
+          DataImport.find(data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
+        end
+
+        importer.import_conversations_page
+
+        expect(importer).to have_received(:create_message).once
+        expect(data_import).not_to have_received(:touch)
+        expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(0)
+      end
+    end
+
+    it 'does not persist stale stats when a newer run takes over during the final part', :aggregate_failures do
+      run_id = 'intercom-run-1'
+      data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
+      importer = described_class.new(data_import: data_import, run_id: run_id)
+      allow(importer).to receive(:create_message).and_wrap_original do |method, *args|
+        method.call(*args).tap do
+          entry = args[2]
+          next unless entry.part['id'] == 'part_2'
+
+          DataImport.find(data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
+        end
+      end
+
+      importer.import_conversations_page
+
+      conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
+      expect(conversation.messages.count).to eq(3)
+      expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(0)
+    end
+
+    it 'reconciles conversation stats when a superseded run is retried', :aggregate_failures do
+      freeze_time do
+        run_id = 'intercom-run-1'
+        next_run_id = 'intercom-run-2'
+        data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
+        importer = described_class.new(data_import: data_import, run_id: run_id)
+        allow(importer).to receive(:create_message) do
+          data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => next_run_id })
+          travel 1.minute
+        end
+
+        importer.import_conversations_page
+        expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(0)
+
+        retry_importer = described_class.new(data_import: data_import, run_id: next_run_id)
+        retry_importer.import_conversations_page
+        retry_importer.finish!
+
+        expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(1)
+        expect(data_import.processed_records).to eq(5)
+      end
+    end
+
     it 'rolls back a newly inserted conversation when mapping persistence fails', :aggregate_failures do
       importer = described_class.new(data_import: data_import)
       allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
@@ -322,11 +441,7 @@ RSpec.describe DataImports::Intercom::Importer do
 
     it 'rolls back a newly inserted message when mapping persistence fails', :aggregate_failures do
       importer = described_class.new(data_import: data_import)
-      allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
-        raise StandardError, 'mapping failed' if object_type == 'message'
-
-        method.call(object_type, source_id, record, metadata: metadata)
-      end
+      allow(importer).to receive(:record_message_mapping).and_raise(StandardError, 'mapping failed')
 
       importer.import_conversations_page
 
@@ -338,6 +453,79 @@ RSpec.describe DataImports::Intercom::Importer do
       )
       expect(error).to have_attributes(error_code: 'StandardError', message: 'mapping failed')
     end
+
+    it 'retries a contact query timeout after rolling back the first transaction', :aggregate_failures do
+      importer = described_class.new(data_import: data_import)
+      mapping_attempts = 0
+      allow(importer).to receive(:sleep)
+      allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
+        if object_type == 'contact'
+          mapping_attempts += 1
+          raise ActiveRecord::QueryCanceled, 'statement timeout' if mapping_attempts == 1
+        end
+
+        method.call(object_type, source_id, record, metadata: metadata)
+      end
+
+      importer.import_contacts_page
+
+      expect(mapping_attempts).to eq(2)
+      expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
+      expect(account.contacts.where(email: 'customer@example.com').count).to eq(1)
+      expect(data_import.mappings.where(source_object_type: 'contact', source_object_id: 'contact_1').count).to eq(1)
+      expect(data_import.import_errors).to be_empty
+      expect(data_import.reload.stats.dig('contacts', 'imported')).to eq(1)
+    end
+
+    it 'retries a message query timeout after rolling back the first transaction', :aggregate_failures do
+      importer = described_class.new(data_import: data_import)
+      mapping_attempts = 0
+      target_source_id = 'conversation:conversation_1:part:part_1'
+      allow(importer).to receive(:sleep)
+      allow(importer).to receive(:record_message_mapping).and_wrap_original do |method, entry, message|
+        if entry.source_id == target_source_id
+          mapping_attempts += 1
+          raise ActiveRecord::QueryCanceled, 'statement timeout' if mapping_attempts == 1
+        end
+
+        method.call(entry, message)
+      end
+
+      importer.import_conversations_page
+
+      conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
+      expect(mapping_attempts).to eq(2)
+      expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
+      expect(conversation.messages.where(source_id: "intercom:#{target_source_id}").count).to eq(1)
+      expect(data_import.mappings.where(source_object_type: 'message', source_object_id: target_source_id).count).to eq(1)
+      expect(data_import.import_errors).to be_empty
+      expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
+    end
+
+    it 'records one message failure only after the query timeout retry is exhausted', :aggregate_failures do
+      importer = described_class.new(data_import: data_import)
+      mapping_attempts = 0
+      target_source_id = 'conversation:conversation_1:part:part_1'
+      allow(importer).to receive(:sleep)
+      allow(importer).to receive(:record_message_mapping).and_wrap_original do |method, entry, message|
+        if entry.source_id == target_source_id
+          mapping_attempts += 1
+          raise ActiveRecord::QueryCanceled, 'statement timeout'
+        end
+
+        method.call(entry, message)
+      end
+
+      importer.import_conversations_page
+
+      conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
+      error = data_import.import_errors.find_by!(source_object_type: 'message', source_object_id: target_source_id)
+      expect(mapping_attempts).to eq(2)
+      expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
+      expect(conversation.messages.where(source_id: "intercom:#{target_source_id}")).to be_empty
+      expect(error).to have_attributes(error_code: 'ActiveRecord::QueryCanceled', message: 'statement timeout')
+      expect(data_import.reload.stats.dig('errors', 'count')).to eq(1)
+    end
   end
 
   describe '#finish!' do
@@ -373,6 +561,26 @@ RSpec.describe DataImports::Intercom::Importer do
       expect(data_import.last_error_at).to be_nil
       expect(data_import.import_errors.exists?).to be(false)
     end
+
+    it 'does not fail an import after a newer run takes over', :aggregate_failures do
+      run_id = 'intercom-run-1'
+      data_import.update!(
+        status: :processing,
+        source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id }
+      )
+      importer = described_class.new(data_import: data_import, run_id: run_id)
+
+      DataImport.find(data_import.id).update!(
+        status: :pending,
+        source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'intercom-run-2' }
+      )
+
+      importer.fail!(StandardError.new('boom'))
+
+      expect(data_import.reload).to be_pending
+      expect(data_import.last_error_at).to be_nil
+      expect(data_import.import_errors.exists?).to be(false)
+    end
   end
 
   context 'when the Intercom records were imported by an earlier run' do
@@ -406,6 +614,57 @@ RSpec.describe DataImports::Intercom::Importer do
       expect(next_data_import.import_errors.skip_logs.pluck(:details).map { |details| details['reason'] }.uniq).to eq(['already_imported'])
     end
 
+    it 'reconciles skipped stats when a superseded run is retried', :aggregate_failures do
+      described_class.new(data_import: data_import).perform
+      run_id = 'intercom-run-1'
+      next_run_id = 'intercom-run-2'
+      next_data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
+
+      importer = described_class.new(data_import: next_data_import, run_id: run_id)
+      allow(importer).to receive(:skip_existing_message_mapping).and_wrap_original do |method, *args|
+        method.call(*args)
+        part = args[2]
+        next unless part['id'] == 'part_2'
+
+        DataImport.find(next_data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => next_run_id })
+      end
+
+      importer.import_conversations_page
+      expect(next_data_import.reload.stats.dig('messages', 'skipped')).to eq(0)
+
+      retry_importer = described_class.new(data_import: next_data_import, run_id: next_run_id)
+      retry_importer.import_conversations_page
+      retry_importer.finish!
+
+      expect(next_data_import.reload.stats.dig('conversations', 'skipped')).to eq(1)
+      expect(next_data_import.stats.dig('messages', 'skipped')).to eq(3)
+      expect(next_data_import.total_records).to eq(5)
+    end
+
+    it 'keeps skipped contact stats when a query timeout is retried after the item update', :aggregate_failures do
+      described_class.new(data_import: data_import).perform
+      importer = described_class.new(data_import: next_data_import)
+      contact_log_attempts = 0
+      allow(importer).to receive(:sleep)
+      allow(importer).to receive(:record_already_imported_log).and_wrap_original do |method, **attributes|
+        if attributes[:source_object_type] == 'contact'
+          contact_log_attempts += 1
+          raise ActiveRecord::QueryCanceled, 'statement timeout' if contact_log_attempts == 1
+        end
+
+        method.call(**attributes)
+      end
+
+      importer.import_contacts_page
+
+      contact_item = next_data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
+      expect(contact_log_attempts).to eq(2)
+      expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
+      expect(contact_item).to be_skipped
+      expect(next_data_import.reload.stats.dig('contacts', 'skipped')).to eq(1)
+      expect(next_data_import.import_errors.skip_logs.exists?(data_import_item: contact_item)).to be(true)
+    end
+
     it 'recreates messages when existing message mappings point to deleted records', :aggregate_failures do
       described_class.new(data_import: data_import).perform
       conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
@@ -431,6 +690,33 @@ RSpec.describe DataImports::Intercom::Importer do
       expect(message_mappings.filter_map(&:chatwoot_record).count).to eq(3)
     end
 
+    it 'repairs a missing mapping without recreating the existing message', :aggregate_failures do
+      described_class.new(data_import: data_import).import_conversations_page
+      conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
+      message = conversation.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:part_1')
+      DataImportMapping.find_by!(
+        account: account,
+        source_provider: 'intercom',
+        source_object_type: 'message',
+        source_object_id: 'conversation:conversation_1:part:part_1'
+      ).destroy!
+      importer = described_class.new(data_import: data_import)
+      allow(importer).to receive(:find_mapping).and_call_original
+
+      importer.import_conversations_page
+
+      repaired_mapping = DataImportMapping.find_by!(
+        account: account,
+        source_provider: 'intercom',
+        source_object_type: 'message',
+        source_object_id: 'conversation:conversation_1:part:part_1'
+      )
+      expect(conversation.messages.where(source_id: message.source_id).count).to eq(1)
+      expect(repaired_mapping.chatwoot_record).to eq(message)
+      expect(importer).not_to have_received(:find_mapping).with('message', anything)
+      expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
+    end
+
     it 'updates conversation activity when a later import adds new messages to the mapped conversation', :aggregate_failures do
       new_part = {
         'id' => 'part_3',
@@ -493,7 +779,11 @@ RSpec.describe DataImports::Intercom::Importer do
     end
 
     it 'repairs the item and imported count on retry', :aggregate_failures do
-      described_class.new(data_import: data_import).import_contacts_page
+      importer = described_class.new(data_import: data_import)
+      expect(importer).to receive(:reconcile_item_stats).with('contact').once.ordered.and_call_original
+      expect(importer).to receive(:update_cursor).with('contacts', nil).once.ordered.and_call_original
+
+      importer.import_contacts_page
 
       item = data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
       expect(item).to be_imported
@@ -876,6 +1166,32 @@ RSpec.describe DataImports::Intercom::Importer do
       expect(data_import.reload).to be_completed_with_errors
       expect(data_import.stats.dig('errors', 'count')).to eq(1)
     end
+
+    it 'reconciles error stats when a superseded run is retried', :aggregate_failures do
+      run_id = 'intercom-run-1'
+      next_run_id = 'intercom-run-2'
+      data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
+      importer = described_class.new(data_import: data_import, run_id: run_id)
+      allow(importer).to receive(:import_message).and_wrap_original do |method, *args|
+        method.call(*args).tap do
+          entry = args[2]
+          next unless entry.part['id'] == 'part_2'
+
+          DataImport.find(data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => next_run_id })
+        end
+      end
+
+      importer.import_conversations_page
+      expect(data_import.reload.stats.dig('errors', 'count')).to eq(0)
+
+      retry_importer = described_class.new(data_import: data_import, run_id: next_run_id)
+      retry_importer.import_conversations_page
+      retry_importer.finish!
+
+      expect(data_import.reload).to be_completed_with_errors
+      expect(data_import.stats.dig('errors', 'count')).to eq(1)
+      expect(data_import.total_records).to eq(6)
+    end
   end
 
   context 'when the conversation parts total matches the returned parts' do
@@ -922,6 +1238,8 @@ RSpec.describe DataImports::Intercom::Importer do
   end
 
   context 'when a specific Intercom message part fails to persist' do
+    let(:insert_attempts) { [] }
+
     let(:conversation_payload) do
       super().deep_merge(
         'conversation_parts' => {
@@ -942,7 +1260,10 @@ RSpec.describe DataImports::Intercom::Importer do
 
     before do
       allow(Message).to receive(:insert_all!).and_wrap_original do |method, records, **kwargs|
-        raise ActiveRecord::StatementInvalid, 'bad message' if records.first[:source_id] == 'intercom:conversation:conversation_1:part:bad_part'
+        if records.first[:source_id] == 'intercom:conversation:conversation_1:part:bad_part'
+          insert_attempts << records.first[:source_id]
+          raise ActiveRecord::StatementInvalid, 'bad message'
+        end
 
         method.call(records, **kwargs)
       end
@@ -963,6 +1284,7 @@ RSpec.describe DataImports::Intercom::Importer do
       )
       expect(data_import.reload).to be_completed_with_errors
       expect(data_import.stats.dig('errors', 'count')).to eq(1)
+      expect(insert_attempts.one?).to be(true)
     end
   end
 end

spec/services/data_imports/intercom/message_batch_builder_spec.rb

@@ -0,0 +1,178 @@
+require 'rails_helper'
+
+RSpec.describe DataImports::Intercom::MessageBatchBuilder do
+  let(:account) { create(:account) }
+  let(:data_import) { create(:data_import, :intercom, account: account) }
+  let(:conversation) { create(:conversation, account: account) }
+  let(:source_conversation) do
+    {
+      'id' => 'conversation_1',
+      'created_at' => 1_700_000_000,
+      'source' => {
+        'id' => 'source_1',
+        'part_type' => 'conversation',
+        'body' => '<p>Initial message</p>',
+        'created_at' => 1_700_000_000
+      },
+      'conversation_parts' => {
+        'conversation_parts' => [
+          {
+            'id' => 'part_1',
+            'part_type' => 'comment',
+            'body' => '<p>First reply</p>',
+            'created_at' => 1_700_000_000
+          },
+          {
+            'id' => 'part_2',
+            'part_type' => 'note',
+            'body' => '<p>Internal note</p>',
+            'created_at' => 1_700_000_100
+          }
+        ]
+      }
+    }
+  end
+  let(:builder) do
+    described_class.new(
+      data_import: data_import,
+      conversation: conversation,
+      source_conversation: source_conversation
+    )
+  end
+
+  it 'preserves source order and prefetches mappings and messages once per batch', :aggregate_failures do
+    sql_queries = []
+    subscriber = lambda do |_name, _start, _finish, _id, payload|
+      sql_queries << payload unless payload[:name] == 'SCHEMA'
+    end
+    batch_builder = builder
+
+    batch = ActiveSupport::Notifications.subscribed(subscriber, 'sql.active_record') { batch_builder.perform }
+
+    expect(batch.entries.map(&:source_id)).to eq(
+      %w[
+        conversation:conversation_1:source:source_1
+        conversation:conversation_1:part:part_1
+        conversation:conversation_1:part:part_2
+      ]
+    )
+    expect(batch.entries.map(&:position)).to eq([0, 1, 2])
+    expect(batch.entries.map(&:classification)).to all(eq(:new_message))
+    expect(sql_queries.count { |query| query[:name] == 'DataImportMapping Load' }).to eq(1)
+    message_queries = sql_queries.select { |query| query[:name] == 'Message Load' }
+    expect(message_queries.size).to eq(1), message_queries.pluck(:sql).join("\n")
+  end
+
+  it 'returns an empty batch without prefetch queries when the conversation has no messages' do
+    source_conversation['source'] = nil
+    source_conversation['conversation_parts']['conversation_parts'] = []
+
+    expect(DataImportMapping).not_to receive(:where)
+    expect(Message).not_to receive(:where)
+
+    expect(builder.perform.entries).to be_empty
+  end
+
+  it 'classifies a live mapping from the current import as already handled' do
+    message = create(
+      :message,
+      account: account,
+      conversation: conversation,
+      inbox: conversation.inbox,
+      source_id: 'intercom:conversation:conversation_1:part:part_1'
+    )
+    DataImportMapping.create!(
+      account: account,
+      data_import: data_import,
+      source_provider: 'intercom',
+      source_object_type: 'message',
+      source_object_id: 'conversation:conversation_1:part:part_1',
+      chatwoot_record_type: 'Message',
+      chatwoot_record_id: message.id,
+      metadata: {}
+    )
+
+    entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
+
+    expect(entry).to have_attributes(classification: :current_import, message: message)
+  end
+
+  it 'classifies a live mapping from a previous import without changing its owner' do
+    previous_import = create(:data_import, :intercom, account: account)
+    message = create(
+      :message,
+      account: account,
+      conversation: conversation,
+      inbox: conversation.inbox,
+      source_id: 'intercom:conversation:conversation_1:part:part_1'
+    )
+    mapping = DataImportMapping.create!(
+      account: account,
+      data_import: previous_import,
+      source_provider: 'intercom',
+      source_object_type: 'message',
+      source_object_id: 'conversation:conversation_1:part:part_1',
+      chatwoot_record_type: 'Message',
+      chatwoot_record_id: message.id,
+      metadata: {}
+    )
+
+    entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
+
+    expect(entry).to have_attributes(classification: :previous_import, mapping: mapping, message: message)
+    expect(mapping.reload.data_import).to eq(previous_import)
+  end
+
+  it 'classifies a mapping whose message was deleted as repairable' do
+    mapping = DataImportMapping.create!(
+      account: account,
+      data_import: data_import,
+      source_provider: 'intercom',
+      source_object_type: 'message',
+      source_object_id: 'conversation:conversation_1:part:part_1',
+      chatwoot_record_type: 'Message',
+      chatwoot_record_id: 0,
+      metadata: {}
+    )
+
+    entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
+
+    expect(entry).to have_attributes(classification: :repairable_stale_mapping, mapping: mapping, message: nil)
+  end
+
+  it 'classifies an existing conversation message without a mapping for repair' do
+    message = create(
+      :message,
+      account: account,
+      conversation: conversation,
+      inbox: conversation.inbox,
+      source_id: 'intercom:conversation:conversation_1:part:part_1'
+    )
+
+    entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
+
+    expect(entry).to have_attributes(classification: :existing_message, mapping: nil, message: message)
+  end
+
+  it 'repairs a skipped mapping when the source part is now an activity' do
+    source_conversation.dig('conversation_parts', 'conversation_parts').first.merge!(
+      'part_type' => 'assignment',
+      'body' => nil,
+      'assigned_to' => { 'name' => 'Support' }
+    )
+    mapping = DataImportMapping.create!(
+      account: account,
+      data_import: data_import,
+      source_provider: 'intercom',
+      source_object_type: 'message',
+      source_object_id: 'conversation:conversation_1:part:part_1',
+      chatwoot_record_type: 'Conversation',
+      chatwoot_record_id: conversation.id,
+      metadata: { skipped: true }
+    )
+
+    entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
+
+    expect(entry).to have_attributes(classification: :repairable_stale_mapping, mapping: mapping, message: nil)
+  end
+end

spec/services/data_imports/intercom/retry_service_spec.rb

@@ -0,0 +1,82 @@
+require 'rails_helper'
+
+RSpec.describe DataImports::Intercom::RetryService do
+  let(:account) { create(:account) }
+  let(:data_import) { create(:data_import, :intercom, account: account, status: :processing, started_at: 2.hours.ago) }
+
+  before do
+    account.enable_features!('data_import')
+  end
+
+  it 'prepares a stalled import for another run without clearing progress', :aggregate_failures do
+    original_cursor = { 'contacts' => { 'completed' => true }, 'conversations' => { 'starting_after' => 'cursor-1' } }
+    original_stats = { 'contacts' => { 'imported' => 10 }, 'conversations' => { 'imported' => 5 } }
+    started_at = data_import.started_at
+    error = data_import.import_errors.create!(error_code: 'MessageFailed', message: 'Timed out')
+    data_import.update!(
+      cursor: original_cursor,
+      stats: original_stats,
+      source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'previous-run' },
+      updated_at: 16.minutes.ago
+    )
+
+    result = described_class.new(account: account, data_import: data_import).perform
+
+    expect(result).to eq(:enqueue)
+    expect(data_import.reload).to be_pending
+    expect(data_import.started_at).to eq(started_at)
+    expect(data_import.cursor).to eq(original_cursor)
+    expect(data_import.stats).to eq(original_stats)
+    expect(data_import.import_errors).to contain_exactly(error)
+    expect(data_import.active_intercom_import_run_id).not_to eq('previous-run')
+  end
+
+  it 'does not retry an import that is still receiving updates' do
+    result = described_class.new(account: account, data_import: data_import).perform
+
+    expect(result).to eq(:not_stalled)
+    expect(data_import.reload).to be_processing
+  end
+
+  it 'rechecks the import state after acquiring its row lock' do
+    data_import.update!(updated_at: 16.minutes.ago)
+    allow(data_import).to receive(:with_lock).and_wrap_original do |method, *args, &block|
+      data_import.update!(status: :completed, completed_at: Time.current)
+      method.call(*args, &block)
+    end
+
+    result = described_class.new(account: account, data_import: data_import).perform
+
+    expect(result).to eq(:not_stalled)
+    expect(data_import.reload).to be_completed
+  end
+
+  it 'does not retry while another Intercom import is active' do
+    data_import.update!(updated_at: 16.minutes.ago)
+    create(:data_import, :intercom, account: account, status: :processing)
+
+    result = described_class.new(account: account, data_import: data_import).perform
+
+    expect(result).to eq(:active_import_exists)
+    expect(data_import.reload).to be_processing
+  end
+
+  it 'allows an active legacy import to continue alongside the retry' do
+    data_import.update!(updated_at: 16.minutes.ago)
+    create(:data_import, account: account, status: :processing)
+
+    result = described_class.new(account: account, data_import: data_import).perform
+
+    expect(result).to eq(:enqueue)
+    expect(data_import.reload).to be_pending
+  end
+
+  it 'does not retry when the stored access key is unavailable' do
+    data_import.update!(access_token: nil, updated_at: 16.minutes.ago)
+
+    result = described_class.new(account: account, data_import: data_import).perform
+
+    expect(result).to eq(:access_token_missing)
+    expect(data_import.reload).to be_processing
+  end
+end