Compare commits

...
Author SHA1 Message Date
Sony MathewandGitHub b6efdae243 perf: shorten Intercom import jobs (#15051)
## Description

Reduces sustained load during large Intercom imports by limiting
conversation list pages to 10 while retaining 50-contact pages. Cursor
and provider total-count behavior remain unchanged.

The one-minute progress heartbeat now lives in parent PR #15050 because
stalled-import detection must be safe when that PR is deployed
independently. This child PR therefore contains only the smaller-page
delta.

This is Phase 1, Task 2 of the [Intercom import optimization plan
(CW-7615)](https://linear.app/chatwoot/issue/CW-7615/optimize-intercom-import-reliability-and-bulk-message-ingestion).

This PR is stacked on #15050 and should be reviewed as the two-file
delta from `codex/cw-7519-intercom-stalled-retry-15m`.

## Closes

-
[CW-7615](https://linear.app/chatwoot/issue/CW-7615/optimize-intercom-import-reliability-and-bulk-message-ingestion)

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)

## How to test

1. Start an Intercom import containing contacts and conversations across
multiple source pages.
2. Confirm contact list requests retain a page size of 50.
3. Confirm conversation list requests use a page size of 10.
4. Confirm page cursors and displayed provider totals continue advancing
normally.
5. Confirm the parent PR heartbeat keeps long conversations fresh while
these shorter pages are processed.

## Checklist

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my code
- [x] I have added tests that prove the change is effective
- [x] New and existing focused tests pass locally with my changes
2026-07-23 23:38:01 +05:30
Sony Mathew ea783a89ea fix(imports): check run ownership before heartbeat 2026-07-23 14:38:23 +05:30
Sony Mathew b8945e8d87 fix(imports): guard stale run failures 2026-07-23 00:21:09 +05:30
Sony Mathew 5b5cdf521a fix(imports): ignore stale detail polls after retry 2026-07-23 00:01:59 +05:30
Sony Mathew 7bdf38732f fix(imports): heartbeat active Intercom runs 2026-07-23 00:00:29 +05:30
Sony Mathew 6a43fd5b5a fix(imports): lock stalled import retries 2026-07-22 23:59:30 +05:30
Sony MathewandGitHub 35167631bb Merge branch 'develop' into codex/cw-7519-intercom-stalled-retry-15m 2026-07-22 23:48:36 +05:30
Sony MathewandGitHub f866adf2c2 Merge branch 'develop' into codex/cw-7519-intercom-stalled-retry-15m 2026-07-21 18:09:06 +05:30
Sony MathewandGitHub 0b5881889a Merge branch 'develop' into codex/cw-7519-intercom-stalled-retry-15m 2026-07-17 15:27:31 +05:30
Sony Mathew 2decd8c544 feat(imports): retry stalled Intercom imports 2026-07-17 14:39:35 +05:30
19 changed files with 1387 additions and 133 deletions
@@ -4,7 +4,7 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll
DATA_IMPORT_FEATURE = 'data_import'.freeze
before_action :ensure_data_import_feature_enabled
before_action :set_data_import, only: [:show, :start, :abandon, :error_logs, :skip_logs]
before_action :set_data_import, only: [:show, :start, :retry_import, :abandon, :error_logs, :skip_logs]
before_action :check_authorization
def index
@@ -59,6 +59,24 @@ class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseControll
render_show
end
def retry_import
retry_service = DataImports::Intercom::RetryService.new(account: Current.account, data_import: @data_import)
retry_result = retry_service.perform
@data_import = retry_service.data_import
case retry_result
when :enqueue
DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id)
render_show
when :not_stalled
render json: { message: 'This Intercom import is no longer stalled.' }, status: :unprocessable_entity
when :active_import_exists
render json: { message: 'Another Intercom import is already in progress.' }, status: :unprocessable_entity
when :access_token_missing
render json: { message: 'The Intercom access key for this import is unavailable.' }, status: :unprocessable_entity
end
end
def abandon
@data_import.abandon!
render_show
@@ -11,6 +11,10 @@ class DataImportsAPI extends ApiClient {
return axios.post(`${this.url}/${id}/start`);
}
retry(id) {
return axios.post(`${this.url}/${id}/retry`);
}
abandon(id) {
return axios.post(`${this.url}/${id}/abandon`);
}
@@ -462,6 +462,7 @@
"STATUS": "Status",
"IMPORTED": "Imported",
"CREATED": "Created",
"RETRY": "Retry",
"ABANDON": "Abandon"
},
"DETAIL": {
@@ -508,6 +509,8 @@
},
"ALERTS": {
"IMPORT_STARTED": "Intercom import has started.",
"IMPORT_RETRIED": "Intercom import has been queued to resume.",
"IMPORT_RETRY_FAILED": "Could not retry the Intercom import.",
"IMPORT_ABANDONED": "Intercom import has been abandoned.",
"IMPORT_FAILED": "Could not start the Intercom import."
}
@@ -26,6 +26,7 @@ const dataImport = ref(null);
const isLoading = ref(true);
const isRefreshing = ref(false);
const isPolling = ref(false);
const isRetrying = ref(false);
const isAbandoning = ref(false);
const isDownloadingErrorLogs = ref(false);
const isDownloadingSkipLogs = ref(false);
@@ -35,6 +36,7 @@ const errorsOpen = ref(true);
const skipLogsOpen = ref(true);
let pollTimer;
let isPageActive = false;
let importRequestVersion = 0;
const hasActiveImport = computed(() => isActiveImport(dataImport.value));
@@ -50,6 +52,7 @@ const fetchImport = async ({
manual = false,
requestedSkipLogsType = selectedSkipLogsType.value,
} = {}) => {
const requestVersion = importRequestVersion;
if (showLoader) {
isLoading.value = true;
} else if (manual) {
@@ -60,6 +63,8 @@ const fetchImport = async ({
const response = await DataImportsAPI.show(route.params.dataImportId, {
skip_logs_type: requestedSkipLogsType || undefined,
});
if (requestVersion !== importRequestVersion) return;
dataImport.value = response.data;
selectedSkipLogsType.value =
response.data.skip_logs_filters?.selected_source_object_type ||
@@ -105,6 +110,13 @@ const refreshImportInBackground = async () => {
}
};
const startPolling = () => {
stopPolling();
if (!isPageActive || !hasActiveImport.value) return;
pollTimer = window.setInterval(refreshImportInBackground, POLL_INTERVAL_MS);
};
const abandonImport = async () => {
isAbandoning.value = true;
try {
@@ -117,6 +129,22 @@ const abandonImport = async () => {
}
};
const retryImport = async () => {
isRetrying.value = true;
importRequestVersion += 1;
stopPolling();
try {
const response = await DataImportsAPI.retry(dataImport.value.id);
dataImport.value = response.data;
useAlert(t('DATA_IMPORTS.ALERTS.IMPORT_RETRIED'));
} catch {
useAlert(t('DATA_IMPORTS.ALERTS.IMPORT_RETRY_FAILED'));
} finally {
isRetrying.value = false;
if (hasActiveImport.value) startPolling();
}
};
const downloadCsv = (response, filename) => {
const url = window.URL.createObjectURL(
new Blob([response.data], { type: 'text/csv' })
@@ -150,13 +178,6 @@ const downloadSkipLogs = async () => {
}
};
const startPolling = () => {
stopPolling();
if (!isPageActive || !hasActiveImport.value) return;
pollTimer = window.setInterval(refreshImportInBackground, POLL_INTERVAL_MS);
};
const handleVisibilityChange = () => {
if (isPageActive && !document.hidden && hasActiveImport.value) {
refreshImportInBackground();
@@ -197,9 +218,11 @@ onBeforeUnmount(() => {
<ImportDetailHeader
:data-import="dataImport"
:is-refreshing="isRefreshing"
:is-retrying="isRetrying"
:is-abandoning="isAbandoning"
:is-polling="isPolling"
@refresh="fetchImport({ manual: true })"
@retry="retryImport"
@abandon="abandonImport"
/>
</template>
@@ -21,6 +21,10 @@ const props = defineProps({
type: Boolean,
default: false,
},
isRetrying: {
type: Boolean,
default: false,
},
isAbandoning: {
type: Boolean,
default: false,
@@ -31,7 +35,7 @@ const props = defineProps({
},
});
defineEmits(['refresh', 'abandon']);
defineEmits(['refresh', 'retry', 'abandon']);
const { t } = useI18n();
@@ -90,11 +94,23 @@ const canAbandonImport = computed(() => isAbandonableImport(props.dataImport));
:title="$t('DATA_IMPORTS.MONITOR.REFRESH')"
@click="$emit('refresh')"
/>
<Button
v-if="dataImport?.stalled"
outline
slate
size="sm"
icon="i-lucide-rotate-ccw"
:is-loading="isRetrying"
:disabled="isAbandoning"
:label="$t('DATA_IMPORTS.TABLE.RETRY')"
@click="$emit('retry')"
/>
<Button
v-if="canAbandonImport"
ruby
size="sm"
:is-loading="isAbandoning"
:disabled="isRetrying"
:label="$t('DATA_IMPORTS.TABLE.ABANDON')"
@click="$emit('abandon')"
/>
@@ -0,0 +1,85 @@
import { mount } from '@vue/test-utils';
import ImportDetailHeader from '../components/ImportDetailHeader.vue';
vi.mock('vue-i18n', () => ({
useI18n: () => ({ t: key => key }),
}));
const ButtonStub = {
name: 'Button',
props: {
label: { type: String, default: '' },
icon: { type: String, default: '' },
isLoading: { type: Boolean, default: false },
},
emits: ['click'],
template: `
<button
:data-label="label"
:data-icon="icon"
:data-loading="isLoading"
@click="$emit('click')"
>
{{ label }}
</button>
`,
};
const BaseSettingsHeaderStub = {
template: `
<section>
<slot name="title" />
<slot name="description" />
</section>
`,
};
const mountHeader = props =>
mount(ImportDetailHeader, {
props,
global: {
stubs: {
Button: ButtonStub,
BaseSettingsHeader: BaseSettingsHeaderStub,
},
mocks: {
$t: key => key,
},
},
});
describe('ImportDetailHeader', () => {
const activeImport = {
id: 1,
name: 'Intercom import',
data_type: 'intercom',
source_provider: 'intercom',
status: 'processing',
stalled: true,
};
it('shows Retry between Refresh and Abandon for stalled imports', async () => {
const wrapper = mountHeader({ dataImport: activeImport });
const buttons = wrapper.findAll('button');
expect(buttons.map(button => button.attributes('data-label'))).toEqual([
'',
'DATA_IMPORTS.TABLE.RETRY',
'DATA_IMPORTS.TABLE.ABANDON',
]);
await buttons[1].trigger('click');
expect(wrapper.emitted('retry')).toHaveLength(1);
});
it('hides Retry when the server does not report the import as stalled', () => {
const wrapper = mountHeader({
dataImport: { ...activeImport, stalled: false },
});
expect(
wrapper.find('[data-label="DATA_IMPORTS.TABLE.RETRY"]').exists()
).toBe(false);
});
});
@@ -0,0 +1,161 @@
import { flushPromises, mount } from '@vue/test-utils';
import { KeepAlive, defineComponent, h, nextTick } from 'vue';
import { useAlert } from 'dashboard/composables';
import DataImportsAPI from 'dashboard/api/dataImports';
import Show from '../Show.vue';
import { POLL_INTERVAL_MS } from '../importStatus';
vi.mock('dashboard/api/dataImports', () => ({
default: {
show: vi.fn(),
retry: vi.fn(),
},
}));
vi.mock('dashboard/composables', () => ({
useAlert: vi.fn(),
}));
vi.mock('vue-i18n', () => ({
useI18n: () => ({ t: key => key }),
}));
vi.mock('vue-router', async importOriginal => ({
...(await importOriginal()),
useRoute: () => ({ params: { dataImportId: 1 } }),
}));
const SettingsLayoutStub = {
template: `
<main>
<slot name="header" />
<slot name="body" />
</main>
`,
};
const ImportDetailHeaderStub = {
name: 'ImportDetailHeader',
props: {
dataImport: {
type: Object,
default: null,
},
},
emits: ['retry'],
template: `
<button
v-if="dataImport?.stalled"
data-test="retry"
@click="$emit('retry')"
/>
`,
};
const deferredRequest = () => {
let resolve;
const promise = new Promise(resolvePromise => {
resolve = resolvePromise;
});
return { promise, resolve };
};
const mountShow = () => {
const Host = defineComponent({
render() {
return h(KeepAlive, null, { default: () => h(Show) });
},
});
return mount(Host, {
global: {
stubs: {
SettingsLayout: SettingsLayoutStub,
ImportDetailHeader: ImportDetailHeaderStub,
ImportSummaryTiles: true,
ImportProgress: true,
ImportErrorsSection: true,
ImportSkipLogsSection: true,
},
mocks: {
$t: key => key,
},
},
});
};
describe('data import detail actions', () => {
beforeEach(() => {
vi.useFakeTimers();
DataImportsAPI.show.mockResolvedValue({
data: {
id: 1,
status: 'processing',
stalled: true,
skip_logs_filters: {},
},
});
});
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
});
it('retries a stalled import and replaces the page state', async () => {
DataImportsAPI.retry.mockResolvedValue({
data: {
id: 1,
status: 'pending',
stalled: false,
skip_logs_filters: {},
},
});
const wrapper = mountShow();
await nextTick();
await flushPromises();
await wrapper.find('[data-test="retry"]').trigger('click');
await flushPromises();
expect(DataImportsAPI.retry).toHaveBeenCalledWith(1);
expect(useAlert).toHaveBeenCalledWith('DATA_IMPORTS.ALERTS.IMPORT_RETRIED');
wrapper.unmount();
});
it('ignores an older poll response after retry succeeds', async () => {
const pollRequest = deferredRequest();
DataImportsAPI.retry.mockResolvedValue({
data: {
id: 1,
status: 'pending',
stalled: false,
skip_logs_filters: {},
},
});
const wrapper = mountShow();
await nextTick();
await flushPromises();
DataImportsAPI.show.mockReturnValueOnce(pollRequest.promise);
await vi.advanceTimersByTimeAsync(POLL_INTERVAL_MS);
expect(DataImportsAPI.show).toHaveBeenCalledTimes(2);
await wrapper.find('[data-test="retry"]').trigger('click');
await flushPromises();
expect(wrapper.find('[data-test="retry"]').exists()).toBe(false);
pollRequest.resolve({
data: {
id: 1,
status: 'processing',
stalled: true,
skip_logs_filters: {},
},
});
await flushPromises();
expect(wrapper.find('[data-test="retry"]').exists()).toBe(false);
wrapper.unmount();
});
});
+5
View File
@@ -33,6 +33,7 @@
#
class DataImport < ApplicationRecord
ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY = 'active_intercom_import_run_id'.freeze
INTERCOM_STALLED_AFTER = 15.minutes
LEGACY_DATA_TYPES = ['contacts'].freeze
INTEGRATION_DATA_TYPES = ['intercom'].freeze
IMPORT_TYPES = %w[contacts conversations].freeze
@@ -71,6 +72,10 @@ class DataImport < ApplicationRecord
failed? || abandoned?
end
def stalled?
intercom_import? && (pending? || processing?) && updated_at <= INTERCOM_STALLED_AFTER.ago
end
def abandonable?
intercom_import? && (pending? || processing?)
end
+4
View File
@@ -19,6 +19,10 @@ class DataImportPolicy < ApplicationPolicy
show?
end
def retry_import?
show?
end
def abandon?
show?
end
+213 -114
View File
@@ -7,13 +7,17 @@ class DataImports::Intercom::Importer
end
DEFAULT_IMPORT_TYPES = %w[contacts conversations].freeze
CONTACTS_PER_PAGE = 50
CONVERSATIONS_PER_PAGE = 10
HEARTBEAT_INTERVAL = 1.minute
QUERY_TIMEOUT_RETRY_LIMIT = 1
QUERY_TIMEOUT_RETRY_DELAY_RANGE = (0.2..0.5)
PROVIDER = 'intercom'.freeze
ALREADY_IMPORTED_ERROR_CODE = 'DataImports::Intercom::AlreadyImported'.freeze
SKIPPED_MESSAGE_ERROR_CODE = 'DataImports::Intercom::SkippedMessage'.freeze
TRUNCATED_PARTS_ERROR_CODE = 'DataImports::Intercom::TruncatedConversationParts'.freeze
E164_REGEX = /\A\+[1-9]\d{1,14}\z/
INTERCOM_NUMBER_REGEX = /\A[1-9]\d{1,14}\z/
REGULAR_MESSAGE_PART_TYPES = %w[comment note source].freeze
def initialize(data_import:, run_id: nil)
@data_import = data_import
@@ -22,6 +26,7 @@ class DataImports::Intercom::Importer
@client = DataImports::Intercom::Client.new(access_token: data_import.access_token)
@placeholder_inboxes = DataImports::Intercom::PlaceholderInboxBuilder.new(account: @account)
@stats = default_stats.deep_merge(data_import.stats || {})
@dirty_stat_groups = {}
end
def perform
@@ -44,8 +49,9 @@ class DataImports::Intercom::Importer
def finish!
return if @data_import.reload.abandoned?
has_failures = @data_import.import_errors.non_skip_logs.exists? || @data_import.import_errors.failed.exists?
status = has_failures ? :completed_with_errors : :completed
error_count = @data_import.import_errors.non_skip_logs.count + @data_import.import_errors.failed.count
@stats['errors']['count'] = error_count
status = error_count.positive? ? :completed_with_errors : :completed
@data_import.update!(
status: status,
completed_at: Time.current,
@@ -56,14 +62,16 @@ class DataImports::Intercom::Importer
end
def fail!(error)
return if @data_import.reload.abandoned?
@data_import.with_lock do
next if @data_import.abandoned? || stale_import_run?
record_run_error(error)
@data_import.update!(status: :failed, last_error_at: Time.current)
record_run_error(error)
@data_import.update!(status: :failed, last_error_at: Time.current)
end
end
def import_contacts_page(starting_after: cursor_for('contacts'))
response = @client.list_contacts(starting_after: starting_after)
response = @client.list_contacts(starting_after: starting_after, per_page: CONTACTS_PER_PAGE)
update_stat_total('contacts', response['total_count']) if response['total_count'].present?
Array(response['data'] || response['contacts']).each do |contact|
break if import_stopped?
@@ -72,13 +80,14 @@ class DataImports::Intercom::Importer
end
return PageResult.new(next_cursor: nil) if import_stopped?
reconcile_dirty_stats
next_cursor = response.dig('pages', 'next', 'starting_after')
update_cursor('contacts', next_cursor)
PageResult.new(next_cursor: next_cursor)
end
def import_conversations_page(starting_after: cursor_for('conversations'))
response = @client.list_conversations(starting_after: starting_after)
response = @client.list_conversations(starting_after: starting_after, per_page: CONVERSATIONS_PER_PAGE)
update_stat_total('conversations', response['total_count']) if response['total_count'].present?
Array(response['data'] || response['conversations']).each do |conversation_summary|
break if import_stopped?
@@ -87,6 +96,7 @@ class DataImports::Intercom::Importer
end
return PageResult.new(next_cursor: nil) if import_stopped?
reconcile_dirty_stats
next_cursor = response.dig('pages', 'next', 'starting_after')
update_cursor('conversations', next_cursor)
PageResult.new(next_cursor: next_cursor)
@@ -153,8 +163,9 @@ class DataImports::Intercom::Importer
mapped_conversation = mapping&.chatwoot_record
if mapped_conversation && mapping.data_import_id != @data_import.id
skip_already_imported_item(item, mapping, already_handled: already_handled)
import_source_message(conversation, mapped_conversation, contact)
import_conversation_parts(conversation, mapped_conversation, contact)
reconcile_item_stats('conversation') if already_handled
return unless import_conversation_messages(conversation, mapped_conversation, contact)
update_conversation_activity(mapped_conversation)
return
end
@@ -164,17 +175,21 @@ class DataImports::Intercom::Importer
record_mapping('conversation', source_id, chatwoot_conversation, metadata: conversation_metadata(conversation, inbox, source_type))
end
item.update!(status: :imported, chatwoot_record_type: 'Conversation', chatwoot_record_id: chatwoot_conversation.id)
increment_stat('conversations', 'imported') unless already_handled
if already_handled
reconcile_item_stats('conversation')
else
increment_stat('conversations', 'imported')
end
return unless import_conversation_messages(conversation, chatwoot_conversation, contact)
import_source_message(conversation, chatwoot_conversation, contact)
import_conversation_parts(conversation, chatwoot_conversation, contact)
update_conversation_activity(chatwoot_conversation)
rescue StandardError => e
raise if e.is_a?(DataImports::Intercom::Client::Error)
fail_item(item, e)
ensure
persist_stats
persist_stats unless @import_stopped
end
def import_stopped?
@@ -184,38 +199,49 @@ class DataImports::Intercom::Importer
@import_stopped = @data_import.abandoned? || @data_import.completed? || @data_import.completed_with_errors? || stale_import_run?
end
def continue_import_with_heartbeat?
return false if import_stopped?
return true if @data_import.updated_at > HEARTBEAT_INTERVAL.ago
@data_import.touch if @data_import.updated_at <= HEARTBEAT_INTERVAL.ago
true
end
def stale_import_run?
active_run_id = @data_import.active_intercom_import_run_id
@run_id.present? && active_run_id.present? && active_run_id != @run_id
end
def import_contact(contact_payload, required_for_conversation: false)
source_id = source_id_for(contact_payload)
if source_id.present? && (mapping = find_mapping('contact', source_id)) && (mapped_contact = mapping.chatwoot_record)
return reuse_mapped_contact(contact_payload, source_id, mapping, mapped_contact)
end
item = nil
with_query_timeout_retry do
source_id = source_id_for(contact_payload)
if source_id.present? && (mapping = find_mapping('contact', source_id)) && (mapped_contact = mapping.chatwoot_record)
return reuse_mapped_contact(contact_payload, source_id, mapping, mapped_contact)
end
contact_payload = retrieve_contact_payload(contact_payload)
source_id = source_id_for(contact_payload)
already_handled = item_handled?('contact', source_id)
item = import_item('contact', source_id, contact_payload)
mapping = find_mapping('contact', source_id)
contact_payload = retrieve_contact_payload(contact_payload)
source_id = source_id_for(contact_payload)
already_handled = item_handled?('contact', source_id)
item = import_item('contact', source_id, contact_payload)
mapping = find_mapping('contact', source_id)
mapped_contact = mapping&.chatwoot_record
if mapped_contact && mapping.data_import_id != @data_import.id
skip_already_imported_item(item, mapping, already_handled: already_handled)
return mapped_contact
end
mapped_contact = mapping&.chatwoot_record
if mapped_contact && mapping.data_import_id != @data_import.id
skip_already_imported_item(item, mapping, already_handled: already_handled)
return mapped_contact
end
contact = Contact.transaction do
imported_contact = mapped_contact || find_existing_contact(contact_payload) || create_contact(contact_payload)
update_existing_contact(imported_contact, contact_payload)
record_mapping('contact', source_id, imported_contact, metadata: contact_metadata(contact_payload))
item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: imported_contact.id)
imported_contact
contact = Contact.transaction do
imported_contact = mapped_contact || find_existing_contact(contact_payload) || create_contact(contact_payload)
update_existing_contact(imported_contact, contact_payload)
record_mapping('contact', source_id, imported_contact, metadata: contact_metadata(contact_payload))
item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: imported_contact.id)
imported_contact
end
increment_stat('contacts', 'imported') unless already_handled
contact
end
increment_stat('contacts', 'imported') unless already_handled
contact
rescue StandardError => e
raise if e.is_a?(DataImports::Intercom::Client::Error)
@@ -369,63 +395,85 @@ class DataImports::Intercom::Importer
end
end
def import_source_message(conversation, chatwoot_conversation, contact)
source = conversation['source'].to_h
return unless source_message_importable?(source)
message_source_id = "conversation:#{source_id_for(conversation)}:source:#{source['id'].presence || 'initial'}"
source_part = source.merge('part_type' => 'source', 'created_at' => conversation['created_at'])
if (mapping = find_mapping('message', message_source_id)) && message_mapping_handled?(mapping, source_part)
if mapping.data_import_id == @data_import.id
reconcile_current_run_message_mapping(chatwoot_conversation, mapping, source_part)
return
end
skip_existing_message_mapping(chatwoot_conversation, mapping, source_part)
return
end
create_message(chatwoot_conversation, contact, source_part, message_source_id)
rescue StandardError => e
fail_message(chatwoot_conversation, message_source_id, source_part, e)
end
def import_conversation_parts(conversation, chatwoot_conversation, contact)
def import_conversation_messages(conversation, chatwoot_conversation, contact)
parts_payload = conversation['conversation_parts'].to_h
parts = Array(parts_payload['conversation_parts'])
batch_builder = DataImports::Intercom::MessageBatchBuilder.new(
data_import: @data_import,
conversation: chatwoot_conversation,
source_conversation: conversation
)
batch = begin
with_query_timeout_retry { batch_builder.perform }
rescue ActiveRecord::QueryCanceled
nil
end
return import_conversation_messages_individually(conversation, chatwoot_conversation, contact, batch_builder, parts.size) if batch.nil?
batch.source_entries.each { |entry| import_message(chatwoot_conversation, contact, entry) }
record_truncated_conversation_parts(conversation, parts.size)
parts.each do |part|
message_source_id = "conversation:#{source_id_for(conversation)}:part:#{part['id']}"
if (mapping = find_mapping('message', message_source_id)) && message_mapping_handled?(mapping, part)
if mapping.data_import_id == @data_import.id
reconcile_current_run_message_mapping(chatwoot_conversation, mapping, part)
next
end
batch.part_entries.each do |entry|
return false unless continue_import_with_heartbeat?
skip_existing_message_mapping(chatwoot_conversation, mapping, part)
next
end
create_message(chatwoot_conversation, contact, part, message_source_id)
rescue StandardError => e
fail_message(chatwoot_conversation, message_source_id, part, e)
import_message(chatwoot_conversation, contact, entry)
end
return false if import_stopped?
true
end
def create_message(conversation, contact, part, message_source_id)
content = content_for(part)
return record_skipped_message(conversation, message_source_id, part) if content.blank?
def import_conversation_messages_individually(conversation, chatwoot_conversation, contact, batch_builder, parts_count)
source_entries, part_entries = batch_builder.unprepared_entries.partition { |entry| entry[:part]['part_type'] == 'source' }
source_entries.each { |entry| import_unprepared_message(chatwoot_conversation, contact, batch_builder, entry) }
record_truncated_conversation_parts(conversation, parts_count)
attrs = message_attributes(conversation, contact, part, message_source_id, content)
message = nil
part_entries.each do |entry|
return false unless continue_import_with_heartbeat?
import_unprepared_message(chatwoot_conversation, contact, batch_builder, entry)
end
return false if import_stopped?
true
end
def import_unprepared_message(conversation, contact, batch_builder, source_entry)
entry = with_query_timeout_retry { batch_builder.perform([source_entry]).entries.first }
import_message(conversation, contact, entry)
rescue StandardError => e
fail_message(conversation, source_entry[:source_id], source_entry[:part], e)
end
def import_message(conversation, contact, entry)
with_query_timeout_retry do
case entry.classification
when :current_import
reconcile_current_run_message_mapping(conversation, entry.mapping, entry.part)
when :previous_import
skip_existing_message_mapping(conversation, entry.mapping, entry.part)
when :repairable_stale_mapping, :existing_message, :new_message
create_message(conversation, contact, entry)
else
raise ArgumentError, "Unsupported Intercom message classification: #{entry.classification}"
end
end
rescue StandardError => e
fail_message(conversation, entry.source_id, entry.part, e)
end
def create_message(conversation, contact, entry)
content = content_for(entry.part)
return record_skipped_message(conversation, entry) if content.blank?
attrs = message_attributes(conversation, contact, entry.part, entry.source_id, content)
message = entry.message
Message.transaction do
message = conversation.messages.find_by(source_id: attrs[:source_id])
unless message
result = Message.insert_all!([attrs], returning: %w[id])
message = Message.find(result.rows.first.first)
end
record_mapping('message', message_source_id, message, metadata: message_metadata(part))
record_message_mapping(entry, message)
end
increment_stat('messages', 'imported')
reindex_message_for_search(message)
@@ -440,13 +488,12 @@ class DataImports::Intercom::Importer
Rails.logger.warn("Intercom import message reindex failed for message #{message.id}: #{e.class} - #{e.message}")
end
def record_skipped_message(conversation, message_source_id, part)
mapping = find_mapping('message', message_source_id)
if mapping
already_recorded = skip_log_recorded?('message', message_source_id, SKIPPED_MESSAGE_ERROR_CODE)
record_skipped_message_log(conversation, message_source_id, part)
def record_skipped_message(conversation, entry)
if entry.mapping
already_recorded = skip_log_recorded?('message', entry.source_id, SKIPPED_MESSAGE_ERROR_CODE)
record_skipped_message_log(conversation, entry.source_id, entry.part)
increment_stat('messages', 'skipped') unless already_recorded
return mapping.chatwoot_record
return entry.message
end
DataImportMapping.create!(
@@ -454,15 +501,30 @@ class DataImports::Intercom::Importer
data_import: @data_import,
source_provider: PROVIDER,
source_object_type: 'message',
source_object_id: message_source_id,
source_object_id: entry.source_id,
chatwoot_record_type: 'Conversation',
chatwoot_record_id: conversation.id,
metadata: message_metadata(part).merge(skipped: true, reason: 'blank_or_unsupported_intercom_part')
metadata: message_metadata(entry.part).merge(skipped: true, reason: 'blank_or_unsupported_intercom_part')
)
record_skipped_message_log(conversation, message_source_id, part)
record_skipped_message_log(conversation, entry.source_id, entry.part)
increment_stat('messages', 'skipped')
end
def record_message_mapping(entry, message)
(entry.mapping || DataImportMapping.new(
account: @account,
source_provider: PROVIDER,
source_object_type: 'message',
source_object_id: entry.source_id
)).tap do |mapping|
mapping.data_import = @data_import
mapping.chatwoot_record_type = 'Message'
mapping.chatwoot_record_id = message.id
mapping.metadata = message_metadata(entry.part)
mapping.save!
end
end
def message_attributes(conversation, contact, part, message_source_id, content)
message_type = message_type_for(part)
created_at = timestamp_for(part['created_at'])
@@ -503,8 +565,7 @@ class DataImports::Intercom::Importer
end
def activity_part?(part)
part_type = part['part_type'].to_s
part_type.present? && REGULAR_MESSAGE_PART_TYPES.exclude?(part_type)
DataImports::Intercom::MessageBatchBuilder.activity_part?(part)
end
def message_content(part)
@@ -629,7 +690,7 @@ class DataImports::Intercom::Importer
)
item = import_item('contact', source_id, contact_payload) unless item&.imported?
item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: mapped_contact.id)
reconcile_item_stats('contact')
mark_stat_group_dirty('contacts')
end
def reconcile_item_stats(source_object_type)
@@ -637,34 +698,37 @@ class DataImports::Intercom::Importer
group = stat_group_for(source_object_type)
@stats[group]['imported'] = items.imported.count
@stats[group]['skipped'] = items.skipped.count
persist_stats
end
def reconcile_current_run_message_mapping(conversation, mapping, part)
record_skipped_message_log(conversation, mapping.source_object_id, part) if mapping.metadata['skipped']
mark_stat_group_dirty('messages')
end
def reconcile_message_stats
mappings = @data_import.mappings.where(source_provider: PROVIDER, source_object_type: 'message')
skipped_mappings = mappings.where("metadata ->> 'skipped' = ?", 'true').count
message_logs = @data_import.import_errors.where(source_object_type: 'message')
@stats['messages']['imported'] = mappings.count - skipped_mappings
@stats['messages']['skipped'] = message_logs.where("details ->> 'kind' = ?", 'skipped').count
persist_stats
end
def skip_already_imported_item(item, mapping, already_handled:)
item.update!(
status: :skipped,
chatwoot_record_type: mapping.chatwoot_record_type,
chatwoot_record_id: mapping.chatwoot_record_id,
last_error_code: ALREADY_IMPORTED_ERROR_CODE,
last_error_message: 'Already imported in a previous import.'
)
record_already_imported_log(
data_import_item: item,
source_object_type: item.source_object_type,
source_object_id: item.source_object_id,
mapping: mapping
)
DataImportItem.transaction do
item.update!(
status: :skipped,
chatwoot_record_type: mapping.chatwoot_record_type,
chatwoot_record_id: mapping.chatwoot_record_id,
last_error_code: ALREADY_IMPORTED_ERROR_CODE,
last_error_message: 'Already imported in a previous import.'
)
record_already_imported_log(
data_import_item: item,
source_object_type: item.source_object_type,
source_object_id: item.source_object_id,
mapping: mapping
)
end
increment_stat(stat_group_for(item.source_object_type), 'skipped') unless already_handled
end
@@ -676,13 +740,12 @@ class DataImports::Intercom::Importer
already_recorded = skip_log_recorded?('message', mapping.source_object_id, ALREADY_IMPORTED_ERROR_CODE)
record_already_imported_log(source_object_type: 'message', source_object_id: mapping.source_object_id, mapping: mapping)
end
increment_stat('messages', 'skipped') unless already_recorded
end
def message_mapping_handled?(mapping, part)
return false if mapping.metadata['skipped'] && activity_part?(part)
mapping.metadata['skipped'] || mapping.chatwoot_record.present?
if already_recorded
message_logs = @data_import.import_errors.where(source_object_type: 'message')
@stats['messages']['skipped'] = message_logs.where("details ->> 'kind' = ?", 'skipped').count
else
increment_stat('messages', 'skipped')
end
end
def fail_item(item, error)
@@ -782,7 +845,7 @@ class DataImports::Intercom::Importer
end
def source_message_importable?(source)
source['body'].present? || source['subject'].present? || source['attachments'].present?
DataImports::Intercom::MessageBatchBuilder.source_message_importable?(source)
end
def skipped_message_log_message(part)
@@ -935,6 +998,42 @@ class DataImports::Intercom::Importer
@stats[group][key] = @stats[group][key].to_i + 1
end
def mark_stat_group_dirty(group)
@dirty_stat_groups[group] = true
end
def reconcile_dirty_stats
return if @dirty_stat_groups.empty?
with_query_timeout_retry do
@dirty_stat_groups.each_key do |group|
case group
when 'contacts'
reconcile_item_stats('contact')
when 'messages'
reconcile_message_stats
else
raise ArgumentError, "Unsupported Intercom import stat group: #{group}"
end
end
persist_stats
end
@dirty_stat_groups.clear
end
def with_query_timeout_retry
retries = 0
begin
yield
rescue ActiveRecord::QueryCanceled
raise if retries >= QUERY_TIMEOUT_RETRY_LIMIT
retries += 1
sleep(rand(QUERY_TIMEOUT_RETRY_DELAY_RANGE))
retry
end
end
def update_stat_total(group, total)
@stats[group] ||= {}
@stats[group]['total'] = total.to_i
@@ -0,0 +1,134 @@
class DataImports::Intercom::MessageBatchBuilder
PROVIDER = 'intercom'.freeze
REGULAR_PART_TYPES = %w[comment note source].freeze
Entry = Struct.new(:source_id, :part, :position, :mapping, :message, :classification, keyword_init: true) do
def source?
part['part_type'] == 'source'
end
end
Batch = Struct.new(:items, keyword_init: true) do
def entries
items
end
def source_entries
items.select(&:source?)
end
def part_entries
items.reject(&:source?)
end
end
def self.activity_part?(part)
part_type = part['part_type'].to_s
part_type.present? && REGULAR_PART_TYPES.exclude?(part_type)
end
def self.source_message_importable?(source)
source['body'].present? || source['subject'].present? || source['attachments'].present?
end
def initialize(data_import:, conversation:, source_conversation:)
@data_import = data_import
@account = data_import.account
@conversation = conversation
@source_conversation = source_conversation
end
def perform(source_entries = unprepared_entries)
return Batch.new(items: []) if source_entries.empty?
mappings = message_mappings(source_entries)
messages = messages_for(source_entries, mappings)
Batch.new(items: source_entries.map do |source_entry|
build_entry(source_entry, source_entry[:position], mappings, messages)
end)
end
def unprepared_entries
ordered_source_entries.map.with_index { |entry, position| entry.merge(position: position) }
end
private
def ordered_source_entries
entries = []
source = @source_conversation['source'].to_h
if self.class.source_message_importable?(source)
entries << {
source_id: "conversation:#{source_conversation_id}:source:#{source['id'].presence || 'initial'}",
part: source.merge('part_type' => 'source', 'created_at' => @source_conversation['created_at'])
}
end
conversation_parts.each do |part|
entries << { source_id: "conversation:#{source_conversation_id}:part:#{part['id']}", part: part }
end
entries
end
def conversation_parts
Array(@source_conversation.dig('conversation_parts', 'conversation_parts'))
end
def source_conversation_id
@source_conversation['id'].presence || @source_conversation['external_id'].presence || @source_conversation['email'].presence
end
def message_mappings(source_entries)
DataImportMapping.where(
account: @account,
source_provider: PROVIDER,
source_object_type: 'message',
source_object_id: source_entries.pluck(:source_id)
).index_by(&:source_object_id)
end
def messages_for(source_entries, mappings)
mapped_message_ids = mappings.values.filter_map do |mapping|
mapping.chatwoot_record_id if mapping.chatwoot_record_type == 'Message'
end
chatwoot_source_ids = source_entries.map { |entry| "intercom:#{entry[:source_id]}" }
messages = Message.where(id: mapped_message_ids).or(
Message.where(conversation_id: @conversation.id, source_id: chatwoot_source_ids)
).to_a
{
by_id: messages.index_by(&:id),
by_source_id: messages.index_by(&:source_id)
}
end
def build_entry(source_entry, position, mappings, messages)
source_id = source_entry[:source_id]
mapping = mappings[source_id]
mapped_message = messages[:by_id][mapping.chatwoot_record_id] if mapping&.chatwoot_record_type == 'Message'
existing_message = messages[:by_source_id]["intercom:#{source_id}"]
Entry.new(
source_id: source_id,
part: source_entry[:part],
position: position,
mapping: mapping,
message: mapped_message || existing_message,
classification: classification_for(mapping, mapped_message, existing_message, source_entry[:part])
)
end
def classification_for(mapping, mapped_message, existing_message, part)
return existing_message.present? ? :existing_message : :new_message if mapping.blank?
return :repairable_stale_mapping unless mapping_handled?(mapping, mapped_message, part)
mapping.data_import_id == @data_import.id ? :current_import : :previous_import
end
def mapping_handled?(mapping, mapped_message, part)
return false if mapping.metadata['skipped'] && self.class.activity_part?(part)
mapping.metadata['skipped'] || mapped_message.present?
end
end
@@ -0,0 +1,28 @@
class DataImports::Intercom::RetryService
attr_reader :data_import
def initialize(account:, data_import:)
@account = account
@data_import = data_import
end
def perform
@account.with_lock do
@data_import.with_lock do
next :not_stalled unless @data_import.stalled?
next :active_import_exists if another_active_import?
next :access_token_missing if @data_import.access_token.blank?
@data_import.assign_active_intercom_import_run_id
@data_import.update!(status: :pending)
:enqueue
end
end
end
private
def another_active_import?
@account.data_imports.active_intercom.where.not(id: @data_import.id).exists?
end
end
@@ -5,6 +5,7 @@ json.source_type data_import.source_type
json.source_provider data_import.source_provider
json.import_types data_import.import_types
json.status data_import.status
json.stalled data_import.stalled?
json.total_records data_import.total_records
json.processed_records data_import.processed_records
json.stats data_import.stats
+1
View File
@@ -230,6 +230,7 @@ Rails.application.routes.draw do
end
member do
post :start
post :retry, action: :retry_import
post :abandon
get :error_logs
get :skip_logs
+32
View File
@@ -64,4 +64,36 @@ RSpec.describe DataImport do
expect(data_import.abandoned_at).to be_nil
end
end
describe '#stalled?' do
let(:account) { create(:account) }
it 'identifies active Intercom imports without updates for fifteen minutes', :aggregate_failures do
freeze_time do
processing_import = create(:data_import, :intercom, account: account, status: :processing)
pending_import = create(:data_import, :intercom, account: account, status: :pending)
processing_import.update!(updated_at: 15.minutes.ago)
pending_import.update!(updated_at: 15.minutes.ago)
expect(processing_import.reload).to be_stalled
expect(pending_import.reload).to be_stalled
end
end
it 'does not identify recent or terminal Intercom imports as stalled', :aggregate_failures do
recent_import = create(:data_import, :intercom, account: account, status: :processing)
completed_import = create(:data_import, :intercom, account: account, status: :completed)
completed_import.update!(updated_at: 1.hour.ago)
expect(recent_import).not_to be_stalled
expect(completed_import.reload).not_to be_stalled
end
it 'does not identify legacy imports as stalled' do
legacy_import = create(:data_import, account: account, status: :processing)
legacy_import.update!(updated_at: 1.hour.ago)
expect(legacy_import.reload).not_to be_stalled
end
end
end
@@ -209,6 +209,63 @@ RSpec.describe 'Data Imports API', type: :request do
end
end
describe 'POST /api/v1/accounts/:account_id/data_imports/:id/retry' do
let(:data_import) { create(:data_import, :intercom, account: account, status: :processing, started_at: 2.hours.ago) }
it 'resumes a stalled import with a new run identifier while preserving progress', :aggregate_failures do
started_at = data_import.started_at
data_import.update!(
cursor: { 'conversations' => { 'starting_after' => 'cursor-1' } },
stats: { 'conversations' => { 'imported' => 20 } },
source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'previous-run' },
updated_at: 16.minutes.ago
)
expect do
post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.to have_enqueued_job(DataImports::Intercom::ImportJob).with(data_import, a_kind_of(String))
expect(response).to have_http_status(:ok)
expect(response.parsed_body).to include('status' => 'pending', 'stalled' => false)
expect(data_import.reload.started_at).to eq(started_at)
expect(data_import.cursor.dig('conversations', 'starting_after')).to eq('cursor-1')
expect(data_import.stats.dig('conversations', 'imported')).to eq(20)
expect(data_import.active_intercom_import_run_id).not_to eq('previous-run')
end
it 'rejects duplicate retries after the import becomes active again' do
data_import.update!(updated_at: 16.minutes.ago)
post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
expect do
post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:unprocessable_entity)
expect(response.parsed_body['message']).to eq('This Intercom import is no longer stalled.')
end
it 'rejects retry while another Intercom import is active' do
data_import.update!(updated_at: 16.minutes.ago)
create(:data_import, :intercom, account: account, status: :processing)
expect do
post retry_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:unprocessable_entity)
expect(response.parsed_body['message']).to eq('Another Intercom import is already in progress.')
end
end
describe 'POST /api/v1/accounts/:account_id/data_imports/:id/abandon' do
let(:data_import) { create(:data_import, :intercom, account: account) }
@@ -283,6 +340,7 @@ RSpec.describe 'Data Imports API', type: :request do
'id' => data_import.id,
'name' => 'July Intercom migration',
'source_provider' => 'intercom',
'stalled' => false,
'import_errors_count' => 1,
'skip_logs_count' => 1
)
@@ -66,12 +66,12 @@ RSpec.describe DataImports::Intercom::Importer do
before do
account.enable_features!('data_import')
allow(DataImports::Intercom::Client).to receive(:new).with(access_token: 'intercom-token').and_return(client)
allow(client).to receive(:list_contacts).with(starting_after: nil).and_return(
allow(client).to receive(:list_contacts).with(starting_after: nil, per_page: 50).and_return(
'data' => [contact_payload],
'total_count' => 1,
'pages' => { 'next' => nil }
)
allow(client).to receive(:list_conversations).with(starting_after: nil).and_return(
allow(client).to receive(:list_conversations).with(starting_after: nil, per_page: 10).and_return(
'conversations' => [{ 'id' => 'conversation_1' }],
'total_count' => 1,
'pages' => { 'next' => nil }
@@ -188,14 +188,49 @@ RSpec.describe DataImports::Intercom::Importer do
expect(item.metadata['message_total_contribution']).to eq(3)
end
it 'uses smaller conversation pages while retaining the contact page size' do
importer = described_class.new(data_import: data_import)
importer.import_contacts_page
importer.import_conversations_page
expect(client).to have_received(:list_contacts).with(starting_after: nil, per_page: 50)
expect(client).to have_received(:list_conversations).with(starting_after: nil, per_page: 10)
end
it 'reconciles imported message stats from same-run mappings on retry' do
described_class.new(data_import: data_import).import_conversations_page
stats = data_import.reload.stats.deep_dup
stats['messages']['imported'] = 0
data_import.update!(stats: stats)
importer = described_class.new(data_import: data_import)
expect(importer).to receive(:reconcile_message_stats).once.ordered.and_call_original
expect(importer).to receive(:update_cursor).with('conversations', nil).once.ordered.and_call_original
importer.import_conversations_page
expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
end
it 'retries a query timeout during deferred message stats reconciliation', :aggregate_failures do
described_class.new(data_import: data_import).import_conversations_page
stats = data_import.reload.stats.deep_dup
stats['messages']['imported'] = 0
data_import.update!(stats: stats)
importer = described_class.new(data_import: data_import)
reconciliation_attempts = 0
allow(importer).to receive(:sleep)
allow(importer).to receive(:reconcile_message_stats).and_wrap_original do |method|
reconciliation_attempts += 1
raise ActiveRecord::QueryCanceled, 'statement timeout' if reconciliation_attempts == 1
method.call
end
importer.import_conversations_page
expect(reconciliation_attempts).to eq(2)
expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
end
@@ -267,7 +302,7 @@ RSpec.describe DataImports::Intercom::Importer do
it 'stops an in-flight page when a newer import run takes over', :aggregate_failures do
run_id = 'intercom-run-1'
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
allow(client).to receive(:list_conversations).with(starting_after: nil).and_return(
allow(client).to receive(:list_conversations).with(starting_after: nil, per_page: 10).and_return(
'conversations' => [{ 'id' => 'conversation_1' }, { 'id' => 'conversation_2' }],
'pages' => { 'next' => { 'starting_after' => 'next-conversation-cursor' } }
)
@@ -285,6 +320,90 @@ RSpec.describe DataImports::Intercom::Importer do
expect(data_import.reload.cursor.dig('conversations', 'starting_after')).to be_nil
end
it 'heartbeats at most once per minute while processing conversation parts' do
freeze_time do
started_at = Time.current
long_conversation = conversation_payload.deep_dup
template_part = long_conversation.dig('conversation_parts', 'conversation_parts').first
long_conversation['conversation_parts']['conversation_parts'] = Array.new(5) do |index|
template_part.merge('id' => "part_#{index + 1}")
end
allow(client).to receive(:retrieve_conversation).with('conversation_1').and_return(long_conversation)
heartbeat_times = []
allow(data_import).to receive(:touch).and_wrap_original do |method|
heartbeat_times << Time.current
method.call
end
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:create_message) { travel 30.seconds }
importer.import_conversations_page
expect(heartbeat_times).to eq([started_at + 1.minute, started_at + 2.minutes])
end
end
it 'stops parts without heartbeating or persisting stale stats when a newer run takes over', :aggregate_failures do
freeze_time do
run_id = 'intercom-run-1'
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
allow(data_import).to receive(:touch).and_call_original
importer = described_class.new(data_import: data_import, run_id: run_id)
allow(importer).to receive(:create_message) do
DataImport.find(data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
end
importer.import_conversations_page
expect(importer).to have_received(:create_message).once
expect(data_import).not_to have_received(:touch)
expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(0)
end
end
it 'does not persist stale stats when a newer run takes over during the final part', :aggregate_failures do
run_id = 'intercom-run-1'
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
importer = described_class.new(data_import: data_import, run_id: run_id)
allow(importer).to receive(:create_message).and_wrap_original do |method, *args|
method.call(*args).tap do
entry = args[2]
next unless entry.part['id'] == 'part_2'
DataImport.find(data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
end
end
importer.import_conversations_page
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
expect(conversation.messages.count).to eq(3)
expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(0)
end
it 'reconciles conversation stats when a superseded run is retried', :aggregate_failures do
freeze_time do
run_id = 'intercom-run-1'
next_run_id = 'intercom-run-2'
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
importer = described_class.new(data_import: data_import, run_id: run_id)
allow(importer).to receive(:create_message) do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => next_run_id })
travel 1.minute
end
importer.import_conversations_page
expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(0)
retry_importer = described_class.new(data_import: data_import, run_id: next_run_id)
retry_importer.import_conversations_page
retry_importer.finish!
expect(data_import.reload.stats.dig('conversations', 'imported')).to eq(1)
expect(data_import.processed_records).to eq(5)
end
end
it 'rolls back a newly inserted conversation when mapping persistence fails', :aggregate_failures do
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
@@ -322,11 +441,7 @@ RSpec.describe DataImports::Intercom::Importer do
it 'rolls back a newly inserted message when mapping persistence fails', :aggregate_failures do
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
raise StandardError, 'mapping failed' if object_type == 'message'
method.call(object_type, source_id, record, metadata: metadata)
end
allow(importer).to receive(:record_message_mapping).and_raise(StandardError, 'mapping failed')
importer.import_conversations_page
@@ -338,6 +453,79 @@ RSpec.describe DataImports::Intercom::Importer do
)
expect(error).to have_attributes(error_code: 'StandardError', message: 'mapping failed')
end
it 'retries a contact query timeout after rolling back the first transaction', :aggregate_failures do
importer = described_class.new(data_import: data_import)
mapping_attempts = 0
allow(importer).to receive(:sleep)
allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
if object_type == 'contact'
mapping_attempts += 1
raise ActiveRecord::QueryCanceled, 'statement timeout' if mapping_attempts == 1
end
method.call(object_type, source_id, record, metadata: metadata)
end
importer.import_contacts_page
expect(mapping_attempts).to eq(2)
expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
expect(account.contacts.where(email: 'customer@example.com').count).to eq(1)
expect(data_import.mappings.where(source_object_type: 'contact', source_object_id: 'contact_1').count).to eq(1)
expect(data_import.import_errors).to be_empty
expect(data_import.reload.stats.dig('contacts', 'imported')).to eq(1)
end
it 'retries a message query timeout after rolling back the first transaction', :aggregate_failures do
importer = described_class.new(data_import: data_import)
mapping_attempts = 0
target_source_id = 'conversation:conversation_1:part:part_1'
allow(importer).to receive(:sleep)
allow(importer).to receive(:record_message_mapping).and_wrap_original do |method, entry, message|
if entry.source_id == target_source_id
mapping_attempts += 1
raise ActiveRecord::QueryCanceled, 'statement timeout' if mapping_attempts == 1
end
method.call(entry, message)
end
importer.import_conversations_page
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
expect(mapping_attempts).to eq(2)
expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
expect(conversation.messages.where(source_id: "intercom:#{target_source_id}").count).to eq(1)
expect(data_import.mappings.where(source_object_type: 'message', source_object_id: target_source_id).count).to eq(1)
expect(data_import.import_errors).to be_empty
expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
end
it 'records one message failure only after the query timeout retry is exhausted', :aggregate_failures do
importer = described_class.new(data_import: data_import)
mapping_attempts = 0
target_source_id = 'conversation:conversation_1:part:part_1'
allow(importer).to receive(:sleep)
allow(importer).to receive(:record_message_mapping).and_wrap_original do |method, entry, message|
if entry.source_id == target_source_id
mapping_attempts += 1
raise ActiveRecord::QueryCanceled, 'statement timeout'
end
method.call(entry, message)
end
importer.import_conversations_page
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
error = data_import.import_errors.find_by!(source_object_type: 'message', source_object_id: target_source_id)
expect(mapping_attempts).to eq(2)
expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
expect(conversation.messages.where(source_id: "intercom:#{target_source_id}")).to be_empty
expect(error).to have_attributes(error_code: 'ActiveRecord::QueryCanceled', message: 'statement timeout')
expect(data_import.reload.stats.dig('errors', 'count')).to eq(1)
end
end
describe '#finish!' do
@@ -373,6 +561,26 @@ RSpec.describe DataImports::Intercom::Importer do
expect(data_import.last_error_at).to be_nil
expect(data_import.import_errors.exists?).to be(false)
end
it 'does not fail an import after a newer run takes over', :aggregate_failures do
run_id = 'intercom-run-1'
data_import.update!(
status: :processing,
source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id }
)
importer = described_class.new(data_import: data_import, run_id: run_id)
DataImport.find(data_import.id).update!(
status: :pending,
source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'intercom-run-2' }
)
importer.fail!(StandardError.new('boom'))
expect(data_import.reload).to be_pending
expect(data_import.last_error_at).to be_nil
expect(data_import.import_errors.exists?).to be(false)
end
end
context 'when the Intercom records were imported by an earlier run' do
@@ -406,6 +614,57 @@ RSpec.describe DataImports::Intercom::Importer do
expect(next_data_import.import_errors.skip_logs.pluck(:details).map { |details| details['reason'] }.uniq).to eq(['already_imported'])
end
it 'reconciles skipped stats when a superseded run is retried', :aggregate_failures do
described_class.new(data_import: data_import).perform
run_id = 'intercom-run-1'
next_run_id = 'intercom-run-2'
next_data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
importer = described_class.new(data_import: next_data_import, run_id: run_id)
allow(importer).to receive(:skip_existing_message_mapping).and_wrap_original do |method, *args|
method.call(*args)
part = args[2]
next unless part['id'] == 'part_2'
DataImport.find(next_data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => next_run_id })
end
importer.import_conversations_page
expect(next_data_import.reload.stats.dig('messages', 'skipped')).to eq(0)
retry_importer = described_class.new(data_import: next_data_import, run_id: next_run_id)
retry_importer.import_conversations_page
retry_importer.finish!
expect(next_data_import.reload.stats.dig('conversations', 'skipped')).to eq(1)
expect(next_data_import.stats.dig('messages', 'skipped')).to eq(3)
expect(next_data_import.total_records).to eq(5)
end
it 'keeps skipped contact stats when a query timeout is retried after the item update', :aggregate_failures do
described_class.new(data_import: data_import).perform
importer = described_class.new(data_import: next_data_import)
contact_log_attempts = 0
allow(importer).to receive(:sleep)
allow(importer).to receive(:record_already_imported_log).and_wrap_original do |method, **attributes|
if attributes[:source_object_type] == 'contact'
contact_log_attempts += 1
raise ActiveRecord::QueryCanceled, 'statement timeout' if contact_log_attempts == 1
end
method.call(**attributes)
end
importer.import_contacts_page
contact_item = next_data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(contact_log_attempts).to eq(2)
expect(importer).to have_received(:sleep).with(be_between(0.2, 0.5)).once
expect(contact_item).to be_skipped
expect(next_data_import.reload.stats.dig('contacts', 'skipped')).to eq(1)
expect(next_data_import.import_errors.skip_logs.exists?(data_import_item: contact_item)).to be(true)
end
it 'recreates messages when existing message mappings point to deleted records', :aggregate_failures do
described_class.new(data_import: data_import).perform
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
@@ -431,6 +690,33 @@ RSpec.describe DataImports::Intercom::Importer do
expect(message_mappings.filter_map(&:chatwoot_record).count).to eq(3)
end
it 'repairs a missing mapping without recreating the existing message', :aggregate_failures do
described_class.new(data_import: data_import).import_conversations_page
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
message = conversation.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:part_1')
DataImportMapping.find_by!(
account: account,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:part_1'
).destroy!
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:find_mapping).and_call_original
importer.import_conversations_page
repaired_mapping = DataImportMapping.find_by!(
account: account,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:part_1'
)
expect(conversation.messages.where(source_id: message.source_id).count).to eq(1)
expect(repaired_mapping.chatwoot_record).to eq(message)
expect(importer).not_to have_received(:find_mapping).with('message', anything)
expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
end
it 'updates conversation activity when a later import adds new messages to the mapped conversation', :aggregate_failures do
new_part = {
'id' => 'part_3',
@@ -493,7 +779,11 @@ RSpec.describe DataImports::Intercom::Importer do
end
it 'repairs the item and imported count on retry', :aggregate_failures do
described_class.new(data_import: data_import).import_contacts_page
importer = described_class.new(data_import: data_import)
expect(importer).to receive(:reconcile_item_stats).with('contact').once.ordered.and_call_original
expect(importer).to receive(:update_cursor).with('contacts', nil).once.ordered.and_call_original
importer.import_contacts_page
item = data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(item).to be_imported
@@ -876,6 +1166,32 @@ RSpec.describe DataImports::Intercom::Importer do
expect(data_import.reload).to be_completed_with_errors
expect(data_import.stats.dig('errors', 'count')).to eq(1)
end
it 'reconciles error stats when a superseded run is retried', :aggregate_failures do
run_id = 'intercom-run-1'
next_run_id = 'intercom-run-2'
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
importer = described_class.new(data_import: data_import, run_id: run_id)
allow(importer).to receive(:import_message).and_wrap_original do |method, *args|
method.call(*args).tap do
entry = args[2]
next unless entry.part['id'] == 'part_2'
DataImport.find(data_import.id).update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => next_run_id })
end
end
importer.import_conversations_page
expect(data_import.reload.stats.dig('errors', 'count')).to eq(0)
retry_importer = described_class.new(data_import: data_import, run_id: next_run_id)
retry_importer.import_conversations_page
retry_importer.finish!
expect(data_import.reload).to be_completed_with_errors
expect(data_import.stats.dig('errors', 'count')).to eq(1)
expect(data_import.total_records).to eq(6)
end
end
context 'when the conversation parts total matches the returned parts' do
@@ -922,6 +1238,8 @@ RSpec.describe DataImports::Intercom::Importer do
end
context 'when a specific Intercom message part fails to persist' do
let(:insert_attempts) { [] }
let(:conversation_payload) do
super().deep_merge(
'conversation_parts' => {
@@ -942,7 +1260,10 @@ RSpec.describe DataImports::Intercom::Importer do
before do
allow(Message).to receive(:insert_all!).and_wrap_original do |method, records, **kwargs|
raise ActiveRecord::StatementInvalid, 'bad message' if records.first[:source_id] == 'intercom:conversation:conversation_1:part:bad_part'
if records.first[:source_id] == 'intercom:conversation:conversation_1:part:bad_part'
insert_attempts << records.first[:source_id]
raise ActiveRecord::StatementInvalid, 'bad message'
end
method.call(records, **kwargs)
end
@@ -963,6 +1284,7 @@ RSpec.describe DataImports::Intercom::Importer do
)
expect(data_import.reload).to be_completed_with_errors
expect(data_import.stats.dig('errors', 'count')).to eq(1)
expect(insert_attempts.one?).to be(true)
end
end
end
@@ -0,0 +1,178 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::MessageBatchBuilder do
let(:account) { create(:account) }
let(:data_import) { create(:data_import, :intercom, account: account) }
let(:conversation) { create(:conversation, account: account) }
let(:source_conversation) do
{
'id' => 'conversation_1',
'created_at' => 1_700_000_000,
'source' => {
'id' => 'source_1',
'part_type' => 'conversation',
'body' => '<p>Initial message</p>',
'created_at' => 1_700_000_000
},
'conversation_parts' => {
'conversation_parts' => [
{
'id' => 'part_1',
'part_type' => 'comment',
'body' => '<p>First reply</p>',
'created_at' => 1_700_000_000
},
{
'id' => 'part_2',
'part_type' => 'note',
'body' => '<p>Internal note</p>',
'created_at' => 1_700_000_100
}
]
}
}
end
let(:builder) do
described_class.new(
data_import: data_import,
conversation: conversation,
source_conversation: source_conversation
)
end
it 'preserves source order and prefetches mappings and messages once per batch', :aggregate_failures do
sql_queries = []
subscriber = lambda do |_name, _start, _finish, _id, payload|
sql_queries << payload unless payload[:name] == 'SCHEMA'
end
batch_builder = builder
batch = ActiveSupport::Notifications.subscribed(subscriber, 'sql.active_record') { batch_builder.perform }
expect(batch.entries.map(&:source_id)).to eq(
%w[
conversation:conversation_1:source:source_1
conversation:conversation_1:part:part_1
conversation:conversation_1:part:part_2
]
)
expect(batch.entries.map(&:position)).to eq([0, 1, 2])
expect(batch.entries.map(&:classification)).to all(eq(:new_message))
expect(sql_queries.count { |query| query[:name] == 'DataImportMapping Load' }).to eq(1)
message_queries = sql_queries.select { |query| query[:name] == 'Message Load' }
expect(message_queries.size).to eq(1), message_queries.pluck(:sql).join("\n")
end
it 'returns an empty batch without prefetch queries when the conversation has no messages' do
source_conversation['source'] = nil
source_conversation['conversation_parts']['conversation_parts'] = []
expect(DataImportMapping).not_to receive(:where)
expect(Message).not_to receive(:where)
expect(builder.perform.entries).to be_empty
end
it 'classifies a live mapping from the current import as already handled' do
message = create(
:message,
account: account,
conversation: conversation,
inbox: conversation.inbox,
source_id: 'intercom:conversation:conversation_1:part:part_1'
)
DataImportMapping.create!(
account: account,
data_import: data_import,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:part_1',
chatwoot_record_type: 'Message',
chatwoot_record_id: message.id,
metadata: {}
)
entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
expect(entry).to have_attributes(classification: :current_import, message: message)
end
it 'classifies a live mapping from a previous import without changing its owner' do
previous_import = create(:data_import, :intercom, account: account)
message = create(
:message,
account: account,
conversation: conversation,
inbox: conversation.inbox,
source_id: 'intercom:conversation:conversation_1:part:part_1'
)
mapping = DataImportMapping.create!(
account: account,
data_import: previous_import,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:part_1',
chatwoot_record_type: 'Message',
chatwoot_record_id: message.id,
metadata: {}
)
entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
expect(entry).to have_attributes(classification: :previous_import, mapping: mapping, message: message)
expect(mapping.reload.data_import).to eq(previous_import)
end
it 'classifies a mapping whose message was deleted as repairable' do
mapping = DataImportMapping.create!(
account: account,
data_import: data_import,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:part_1',
chatwoot_record_type: 'Message',
chatwoot_record_id: 0,
metadata: {}
)
entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
expect(entry).to have_attributes(classification: :repairable_stale_mapping, mapping: mapping, message: nil)
end
it 'classifies an existing conversation message without a mapping for repair' do
message = create(
:message,
account: account,
conversation: conversation,
inbox: conversation.inbox,
source_id: 'intercom:conversation:conversation_1:part:part_1'
)
entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
expect(entry).to have_attributes(classification: :existing_message, mapping: nil, message: message)
end
it 'repairs a skipped mapping when the source part is now an activity' do
source_conversation.dig('conversation_parts', 'conversation_parts').first.merge!(
'part_type' => 'assignment',
'body' => nil,
'assigned_to' => { 'name' => 'Support' }
)
mapping = DataImportMapping.create!(
account: account,
data_import: data_import,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:part_1',
chatwoot_record_type: 'Conversation',
chatwoot_record_id: conversation.id,
metadata: { skipped: true }
)
entry = builder.perform.entries.find { |batch_entry| batch_entry.source_id.end_with?('part:part_1') }
expect(entry).to have_attributes(classification: :repairable_stale_mapping, mapping: mapping, message: nil)
end
end
@@ -0,0 +1,82 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::RetryService do
let(:account) { create(:account) }
let(:data_import) { create(:data_import, :intercom, account: account, status: :processing, started_at: 2.hours.ago) }
before do
account.enable_features!('data_import')
end
it 'prepares a stalled import for another run without clearing progress', :aggregate_failures do
original_cursor = { 'contacts' => { 'completed' => true }, 'conversations' => { 'starting_after' => 'cursor-1' } }
original_stats = { 'contacts' => { 'imported' => 10 }, 'conversations' => { 'imported' => 5 } }
started_at = data_import.started_at
error = data_import.import_errors.create!(error_code: 'MessageFailed', message: 'Timed out')
data_import.update!(
cursor: original_cursor,
stats: original_stats,
source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'previous-run' },
updated_at: 16.minutes.ago
)
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:enqueue)
expect(data_import.reload).to be_pending
expect(data_import.started_at).to eq(started_at)
expect(data_import.cursor).to eq(original_cursor)
expect(data_import.stats).to eq(original_stats)
expect(data_import.import_errors).to contain_exactly(error)
expect(data_import.active_intercom_import_run_id).not_to eq('previous-run')
end
it 'does not retry an import that is still receiving updates' do
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:not_stalled)
expect(data_import.reload).to be_processing
end
it 'rechecks the import state after acquiring its row lock' do
data_import.update!(updated_at: 16.minutes.ago)
allow(data_import).to receive(:with_lock).and_wrap_original do |method, *args, &block|
data_import.update!(status: :completed, completed_at: Time.current)
method.call(*args, &block)
end
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:not_stalled)
expect(data_import.reload).to be_completed
end
it 'does not retry while another Intercom import is active' do
data_import.update!(updated_at: 16.minutes.ago)
create(:data_import, :intercom, account: account, status: :processing)
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:active_import_exists)
expect(data_import.reload).to be_processing
end
it 'allows an active legacy import to continue alongside the retry' do
data_import.update!(updated_at: 16.minutes.ago)
create(:data_import, account: account, status: :processing)
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:enqueue)
expect(data_import.reload).to be_pending
end
it 'does not retry when the stored access key is unavailable' do
data_import.update!(access_token: nil, updated_at: 16.minutes.ago)
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:access_token_missing)
expect(data_import.reload).to be_processing
end
end