Merge branch 'develop' into feat/api-webhook-feature-flag

This commit is contained in:
Shivam Mishra
2026-07-13 15:40:53 +05:30
committed by GitHub
75 changed files with 7208 additions and 11 deletions
@@ -0,0 +1,159 @@
require 'csv'
class Api::V1::Accounts::DataImportsController < Api::V1::Accounts::BaseController
DATA_IMPORT_FEATURE = 'data_import'.freeze
before_action :ensure_data_import_feature_enabled
before_action :set_data_import, only: [:show, :start, :abandon, :error_logs, :skip_logs]
before_action :check_authorization
def index
@data_imports = policy_scope(Current.account.data_imports).includes(:initiated_by).order(created_at: :desc)
data_import_ids = @data_imports.map(&:id)
@import_errors_counts = DataImportError.non_skip_logs.where(data_import_id: data_import_ids).group(:data_import_id).count
@skip_logs_counts = DataImportError.skip_logs.where(data_import_id: data_import_ids).group(:data_import_id).count
end
def show
render_show
end
def validate_source
totals = validate_intercom_source
render json: { valid: true, totals: totals }
rescue DataImports::Intercom::Client::AuthenticationError
render_source_validation_error('We could not validate this Intercom access key. Check the key and its permissions.')
rescue DataImports::Intercom::Client::Error
render_source_validation_error('Intercom could not be reached. Please try again.')
rescue ArgumentError => e
render_source_validation_error(e.message)
end
def create
@data_import = creation_service.perform
unless @data_import
render json: { message: 'Another data import is already in progress.' }, status: :unprocessable_entity
return
end
DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id)
render_show
rescue DataImports::Intercom::Client::AuthenticationError
render_source_validation_error('We could not validate this Intercom access key. Check the key and its permissions.')
rescue DataImports::Intercom::Client::Error
render_source_validation_error('Intercom could not be reached. Please try again.')
rescue ArgumentError => e
render_source_validation_error(e.message)
end
def start
restart_service = DataImports::Intercom::RestartService.new(account: Current.account, data_import: @data_import)
restart_result = restart_service.perform
@data_import = restart_service.data_import
if restart_result == :access_token_missing
render json: { message: 'The Intercom access key for this import is unavailable.' }, status: :unprocessable_entity
return
end
DataImports::Intercom::ImportJob.perform_later(@data_import, @data_import.active_intercom_import_run_id) if restart_result == :enqueue
render_show
end
def abandon
@data_import.abandon!
render_show
end
def skip_logs
send_data(
skip_logs_csv,
filename: "data-import-#{@data_import.id}-skip-logs.csv",
type: 'text/csv'
)
end
def error_logs
send_data(
error_logs_csv,
filename: "data-import-#{@data_import.id}-error-logs.csv",
type: 'text/csv'
)
end
private
def ensure_data_import_feature_enabled
raise Pundit::NotAuthorizedError unless Current.account.feature_enabled?(DATA_IMPORT_FEATURE)
end
def set_data_import
@data_import = Current.account.data_imports.find(params[:id])
end
def check_authorization
authorize(@data_import || DataImport)
end
def permitted_params
params.permit(:name, :source_provider, :access_token, import_types: [])
end
def creation_service
DataImports::Intercom::CreationService.new(
account: Current.account,
initiated_by: Current.user,
source_params: permitted_params.to_h
)
end
def import_types
return DataImports::Intercom::Importer::DEFAULT_IMPORT_TYPES unless permitted_params.key?(:import_types)
Array(permitted_params[:import_types]).compact_blank
end
def validate_intercom_source
raise ArgumentError, 'Unsupported import source.' unless permitted_params[:source_provider] == 'intercom'
DataImports::Intercom::CredentialsValidator.new(
access_token: permitted_params[:access_token],
import_types: import_types
).perform
end
def render_source_validation_error(message)
render json: { valid: false, message: message }, status: :unprocessable_entity
end
def render_show
@import_errors_finder = DataImportErrorFinder.new(@data_import)
@skip_logs_finder = DataImportSkipLogFinder.new(@data_import, params)
render :show
end
def skip_logs_csv
logs_csv(@data_import.import_errors.skip_logs)
end
def error_logs_csv
logs_csv(@data_import.import_errors.non_skip_logs)
end
def logs_csv(logs)
CSV.generate(headers: true) do |csv|
csv << %w[created_at kind source_object_type source_object_id error_code message details]
logs.order(:created_at).find_each do |log|
csv << [
log.created_at.iso8601,
log.details['kind'],
log.source_object_type,
log.source_object_id,
log.error_code,
log.message,
log.details.to_json
]
end
end
end
end
+11
View File
@@ -0,0 +1,11 @@
class DataImportErrorFinder
RESULTS_LIMIT = 5
def initialize(data_import)
@data_import = data_import
end
def import_errors
@data_import.import_errors.non_skip_logs.order(created_at: :desc).limit(RESULTS_LIMIT)
end
end
@@ -0,0 +1,35 @@
class DataImportSkipLogFinder
RESULTS_LIMIT = 5
SOURCE_OBJECT_TYPES = %w[contact conversation message].freeze
attr_reader :selected_source_object_type
def initialize(data_import, params = {})
@data_import = data_import
@selected_source_object_type = valid_source_object_type(params[:skip_logs_type])
end
def skip_logs
filtered_scope.order(created_at: :desc).limit(RESULTS_LIMIT)
end
def counts_by_type
base_scope.group(:source_object_type).count
end
private
def base_scope
@base_scope ||= @data_import.import_errors.skip_logs
end
def filtered_scope
return base_scope if selected_source_object_type.blank?
base_scope.where(source_object_type: selected_source_object_type)
end
def valid_source_object_type(source_object_type)
source_object_type if SOURCE_OBJECT_TYPES.include?(source_object_type)
end
end
@@ -0,0 +1,39 @@
/* global axios */
import ApiClient from './ApiClient';
class DataImportsAPI extends ApiClient {
constructor() {
super('data_imports', { accountScoped: true });
}
start(id) {
return axios.post(`${this.url}/${id}/start`);
}
abandon(id) {
return axios.post(`${this.url}/${id}/abandon`);
}
show(id, params = {}) {
return axios.get(`${this.url}/${id}`, { params });
}
validateSource(payload) {
return axios.post(`${this.url}/validate_source`, payload);
}
downloadSkipLogs(id) {
return axios.get(`${this.url}/${id}/skip_logs.csv`, {
responseType: 'blob',
});
}
downloadErrorLogs(id) {
return axios.get(`${this.url}/${id}/error_logs.csv`, {
responseType: 'blob',
});
}
}
export default new DataImportsAPI();
@@ -85,6 +85,13 @@ const hasFilteredUnreadCounts = computed(() => {
);
});
const hasDataImport = computed(() => {
return isFeatureEnabledonAccount.value(
accountId.value,
FEATURE_FLAGS.DATA_IMPORT
);
});
const fetchConversationUnreadCounts = ([currentAccountId, isEnabled]) => {
if (!currentAccountId) return;
@@ -856,6 +863,16 @@ const menuItems = computed(() => {
icon: 'i-lucide-blocks',
to: accountScopedRoute('settings_applications'),
},
...(hasDataImport.value
? [
{
name: 'Settings Data',
label: t('SIDEBAR.DATA'),
icon: 'i-lucide-database',
to: accountScopedRoute('settings_data_imports'),
},
]
: []),
{
name: 'Settings Audit Logs',
label: t('SIDEBAR.AUDIT_LOGS'),
@@ -7,7 +7,7 @@ import Button from 'dashboard/components-next/button/Button.vue';
const { modalType, closeOnBackdropClick, onClose } = defineProps({
closeOnBackdropClick: { type: Boolean, default: true },
showCloseButton: { type: Boolean, default: true },
onClose: { type: Function, required: true },
onClose: { type: Function, default: null },
fullWidth: { type: Boolean, default: false },
modalType: { type: String, default: 'centered' },
size: { type: String, default: '' },
@@ -35,7 +35,7 @@ const handleMouseDown = () => {
const close = () => {
show.value = false;
emit('close');
onClose();
onClose?.();
};
const onMouseUp = () => {
+1
View File
@@ -11,6 +11,7 @@ export const FEATURE_FLAGS = {
CANNED_RESPONSES: 'canned_responses',
CRM: 'crm',
CUSTOM_ATTRIBUTES: 'custom_attributes',
DATA_IMPORT: 'data_import',
INBOX_MANAGEMENT: 'inbox_management',
INTEGRATIONS: 'integrations',
LABELS: 'labels',
@@ -340,6 +340,7 @@
"NOTIFICATIONS": "Notifications",
"CANNED_RESPONSES": "Canned Responses",
"INTEGRATIONS": "Integrations",
"DATA": "Data",
"PROFILE_SETTINGS": "Profile Settings",
"ACCOUNT_SETTINGS": "Account Settings",
"APPLICATIONS": "Applications",
@@ -411,6 +412,104 @@
"CAPTAIN_AI": "Captain",
"CONVERSATION_WORKFLOW": "Conversation Workflow"
},
"DATA_IMPORTS": {
"HEADER": "Data",
"DESCRIPTION": "Bring your existing contacts and past conversations into this account from another support tool. Each import runs in the background, so you can keep working while it finishes, track its progress, and review anything that was skipped along the way.",
"LOADING": "Fetching imports",
"DEFAULT_IMPORT_NAME": "Intercom import",
"TABS": {
"IMPORT": "Import",
"EXPORT": "Export"
},
"TYPES": {
"CONTACTS": "Contacts",
"CONVERSATIONS": "Conversations",
"MESSAGES": "Messages"
},
"DRAWER": {
"TITLE": "New import",
"SOURCE": "Source",
"NAME": "Import name",
"NAME_PLACEHOLDER": "July Intercom migration",
"ACCESS_KEY": "Intercom access key",
"ACCESS_KEY_PLACEHOLDER": "Paste your Intercom access key",
"DATA_TYPES": "Data to import",
"VALIDATING": "Validating access key...",
"VALID_KEY": "Access key validated.",
"INVALID_KEY": "Could not validate this access key.",
"ACTIVE_IMPORT": "Wait for the active import to finish before starting another one.",
"CANCEL": "Cancel",
"IMPORT": "Import"
},
"EXPORT": {
"TITLE": "Exports are on the way",
"DESCRIPTION": "Export your contacts and conversations out of this account. This workflow is coming soon.",
"COMING_SOON": "Coming soon"
},
"TABLE": {
"TITLE": "Recent imports",
"EMPTY": "No imports yet",
"EMPTY_DESCRIPTION": "Start an import to bring your existing customer history into this account.",
"NEW_IMPORT": "Import",
"COUNT": "{count} imports",
"UNNAMED": "Untitled import",
"IMPORTED_COUNT": "{count} imported",
"VIEW": "View import",
"NAME": "Name",
"TYPE": "Type",
"STATUS": "Status",
"IMPORTED": "Imported",
"CREATED": "Created",
"ABANDON": "Abandon"
},
"DETAIL": {
"BACK": "Back to imports",
"ERRORS": "Errors",
"SKIP_LOGS": "Skip logs",
"SOURCE": "Source",
"IMPORT_TYPES": "Import types",
"CREATED": "Created",
"DURATION": "Duration",
"INITIATED_BY": "Started by",
"PROGRESS": "Import progress",
"PROGRESS_WITH_TOTAL": "{imported} of {total} imported",
"PROGRESS_WITHOUT_TOTAL": "{imported} imported",
"PROGRESS_OF_TOTAL": "of {total} imported",
"PROGRESS_IMPORTED": "imported",
"LAST_UPDATED_TOOLTIP": "Last updated {time}",
"NO_SKIP_LOGS": "No skipped or failed records recorded.",
"DOWNLOAD_SKIP_LOGS": "Download CSV",
"DOWNLOAD_ERROR_LOGS": "Download CSV",
"ALL_SKIP_LOGS": "All",
"KIND": "Kind",
"NO_ERRORS": "No errors recorded.",
"ERROR_CODE": "Code",
"SOURCE_OBJECT": "Source object",
"MESSAGE": "Message"
},
"MONITOR": {
"LIVE": "Live updates every {seconds}s",
"LAST_UPDATED": "Last updated {time}",
"REFRESH": "Refresh",
"REFRESHING": "Refreshing",
"STAGES": {
"unknown": "Waiting for update",
"queued": "Queued",
"contacts": "Importing contacts",
"conversations": "Importing conversations",
"finalizing": "Finalizing import",
"completed": "Completed",
"completed_with_errors": "Completed with errors",
"failed": "Failed",
"abandoned": "Abandoned"
}
},
"ALERTS": {
"IMPORT_STARTED": "Intercom import has started.",
"IMPORT_ABANDONED": "Intercom import has been abandoned.",
"IMPORT_FAILED": "Could not start the Intercom import."
}
},
"CAPTAIN_SETTINGS": {
"TITLE": "Captain Settings",
"DESCRIPTION": "Configure your AI models and features for Captain. Captain follows a credit based billing, you will be charged credits for every action Captain takes based on the model selected.",
@@ -52,9 +52,11 @@ const helpURL = getHelpUrlForFeature(props.featureName);
v-if="title"
class="flex items-center justify-between w-full gap-4 min-h-8 mb-2"
>
<h1 class="text-heading-1 text-n-slate-12">
{{ title }}
</h1>
<slot name="title">
<h1 class="text-heading-1 text-n-slate-12">
{{ title }}
</h1>
</slot>
</div>
<div
v-if="description || $slots.description || linkText || helpURL"
@@ -0,0 +1,378 @@
<script setup>
import {
computed,
onActivated,
onBeforeUnmount,
onDeactivated,
ref,
} from 'vue';
import { useI18n } from 'vue-i18n';
import { useRouter } from 'vue-router';
import { useStoreGetters } from 'dashboard/composables/store';
import Button from 'dashboard/components-next/button/Button.vue';
import Icon from 'dashboard/components-next/icon/Icon.vue';
import TabBar from 'dashboard/components-next/tabbar/TabBar.vue';
import SettingsLayout from '../SettingsLayout.vue';
import BaseSettingsHeader from '../components/BaseSettingsHeader.vue';
import DataImportsAPI from 'dashboard/api/dataImports';
import NewImportDialog from './NewImportDialog.vue';
import { importSourceFor } from './importSources';
import {
POLL_INTERVAL_MS,
formatDate,
formatStatus,
importedCount,
isActiveImport,
isActiveIntercomImport,
statusDotClass,
} from './importStatus';
const { t } = useI18n();
const getters = useStoreGetters();
const router = useRouter();
const dataImports = ref([]);
const isLoading = ref(true);
const isRefreshing = ref(false);
const isPolling = ref(false);
const showImportDrawer = ref(false);
const activeTab = ref('import');
let pollTimer;
let isPageActive = false;
const accountId = getters.getCurrentAccountId;
const tabs = computed(() => [
{ key: 'import', label: t('DATA_IMPORTS.TABS.IMPORT') },
{ key: 'export', label: t('DATA_IMPORTS.TABS.EXPORT') },
]);
const activeTabIndex = computed(() =>
tabs.value.findIndex(tab => tab.key === activeTab.value)
);
const hasActiveImport = computed(() => dataImports.value.some(isActiveImport));
const hasActiveIntercomImport = computed(() =>
dataImports.value.some(isActiveIntercomImport)
);
const dataImportRoute = dataImport => ({
name: 'settings_data_import_show',
params: { accountId: accountId.value, dataImportId: dataImport.id },
});
const importTypesFor = dataImport =>
dataImport.import_types?.length
? dataImport.import_types
: [dataImport.data_type];
const importTypeLabel = dataImport =>
importTypesFor(dataImport)
.map(type => {
if (type === 'contacts') return t('DATA_IMPORTS.TYPES.CONTACTS');
if (type === 'conversations') {
return t('DATA_IMPORTS.TYPES.CONVERSATIONS');
}
return type;
})
.join(', ');
const fetchImports = async () => {
const response = await DataImportsAPI.get();
dataImports.value = response.data.payload || [];
};
const stopPolling = () => {
if (!pollTimer) return;
window.clearInterval(pollTimer);
pollTimer = null;
};
const refreshImportsInBackground = async () => {
if (
!isPageActive ||
isPolling.value ||
!hasActiveImport.value ||
document.hidden
) {
return;
}
isPolling.value = true;
try {
await fetchImports();
} finally {
isPolling.value = false;
if (!hasActiveImport.value) stopPolling();
}
};
const startPolling = () => {
stopPolling();
if (!isPageActive || !hasActiveImport.value) return;
pollTimer = window.setInterval(refreshImportsInBackground, POLL_INTERVAL_MS);
};
const refresh = async ({ showLoader = true } = {}) => {
if (showLoader) isLoading.value = true;
else isRefreshing.value = true;
try {
await fetchImports();
} finally {
isLoading.value = false;
isRefreshing.value = false;
if (isPageActive) {
if (hasActiveImport.value && !pollTimer) startPolling();
if (!hasActiveImport.value) stopPolling();
}
}
};
const openImport = dataImport => {
router.push(dataImportRoute(dataImport));
};
const openImportDrawer = () => {
if (!hasActiveIntercomImport.value) showImportDrawer.value = true;
};
const onImportCreated = dataImportId => {
showImportDrawer.value = false;
router.push({
name: 'settings_data_import_show',
params: { accountId: accountId.value, dataImportId },
});
};
const onTabChanged = tab => {
activeTab.value = tab.key;
};
const handleVisibilityChange = () => {
if (isPageActive && !document.hidden && hasActiveImport.value) {
refreshImportsInBackground();
}
};
onActivated(async () => {
isPageActive = true;
await refresh();
if (!isPageActive) return;
startPolling();
document.addEventListener('visibilitychange', handleVisibilityChange);
});
onDeactivated(() => {
isPageActive = false;
stopPolling();
document.removeEventListener('visibilitychange', handleVisibilityChange);
});
onBeforeUnmount(() => {
isPageActive = false;
stopPolling();
document.removeEventListener('visibilitychange', handleVisibilityChange);
});
</script>
<template>
<SettingsLayout
:is-loading="isLoading"
:loading-message="$t('DATA_IMPORTS.LOADING')"
>
<template #header>
<BaseSettingsHeader
:title="$t('DATA_IMPORTS.HEADER')"
:description="$t('DATA_IMPORTS.DESCRIPTION')"
>
<template #tabs>
<TabBar
:tabs="tabs"
:initial-active-tab="activeTabIndex"
@tab-changed="onTabChanged"
/>
</template>
<template v-if="activeTab === 'import' && dataImports.length" #count>
<span class="text-body-main text-n-slate-11">
{{ $t('DATA_IMPORTS.TABLE.COUNT', { count: dataImports.length }) }}
</span>
</template>
<template v-if="activeTab === 'import'" #actions>
<span
v-if="hasActiveImport"
class="hidden items-center gap-1.5 text-body-main text-n-slate-11 sm:inline-flex"
>
<span class="size-2 rounded-full bg-n-teal-9 animate-pulse" />
{{
$t('DATA_IMPORTS.MONITOR.LIVE', {
seconds: POLL_INTERVAL_MS / 1000,
})
}}
</span>
<Button
ghost
slate
size="sm"
icon="i-lucide-refresh-cw"
:is-loading="isRefreshing"
:aria-label="$t('DATA_IMPORTS.MONITOR.REFRESH')"
:title="$t('DATA_IMPORTS.MONITOR.REFRESH')"
@click="refresh({ showLoader: false })"
/>
<Button
size="sm"
:label="$t('DATA_IMPORTS.TABLE.NEW_IMPORT')"
:disabled="hasActiveIntercomImport"
:title="
hasActiveIntercomImport
? $t('DATA_IMPORTS.DRAWER.ACTIVE_IMPORT')
: undefined
"
@click="openImportDrawer"
/>
</template>
</BaseSettingsHeader>
</template>
<template #body>
<div
v-if="activeTab === 'export'"
class="flex min-h-80 flex-col items-center justify-center gap-4 rounded-xl border border-n-weak bg-n-solid-1 px-6 py-16 text-center"
>
<span
class="flex size-12 items-center justify-center rounded-full bg-n-alpha-2"
>
<Icon icon="i-lucide-upload" class="size-5 text-n-slate-11" />
</span>
<div class="flex flex-col gap-1">
<h3 class="text-heading-2 text-n-slate-12">
{{ $t('DATA_IMPORTS.EXPORT.TITLE') }}
</h3>
<p class="max-w-sm text-body-main text-n-slate-11">
{{ $t('DATA_IMPORTS.EXPORT.DESCRIPTION') }}
</p>
</div>
<span
class="inline-flex items-center gap-1.5 rounded-md bg-n-alpha-2 px-2 py-1 text-label-small text-n-slate-11"
>
<Icon icon="i-lucide-clock" class="size-3.5" />
{{ $t('DATA_IMPORTS.EXPORT.COMING_SOON') }}
</span>
</div>
<div
v-else-if="!dataImports.length"
class="flex min-h-80 flex-col items-center justify-center gap-4 rounded-xl border border-n-weak bg-n-solid-1 px-6 py-16 text-center"
>
<span
class="flex size-12 items-center justify-center rounded-full bg-n-alpha-2"
>
<Icon icon="i-lucide-database" class="size-5 text-n-slate-11" />
</span>
<div class="flex flex-col gap-1">
<h3 class="text-heading-2 text-n-slate-12">
{{ $t('DATA_IMPORTS.TABLE.EMPTY') }}
</h3>
<p class="max-w-sm text-body-main text-n-slate-11">
{{ $t('DATA_IMPORTS.TABLE.EMPTY_DESCRIPTION') }}
</p>
</div>
<Button
size="sm"
icon="i-lucide-download"
:label="$t('DATA_IMPORTS.TABLE.NEW_IMPORT')"
@click="openImportDrawer"
/>
</div>
<div v-else class="divide-y divide-n-weak border-t border-n-weak">
<div
v-for="dataImport in dataImports"
:key="dataImport.id"
class="group flex cursor-pointer items-center justify-between gap-4 py-4"
role="button"
tabindex="0"
@click="openImport(dataImport)"
@keydown.enter="openImport(dataImport)"
@keydown.space.prevent="openImport(dataImport)"
>
<div class="flex min-w-0 items-center gap-3">
<img
v-if="importSourceFor(dataImport).icon"
v-tooltip.top="importSourceFor(dataImport).label"
:src="importSourceFor(dataImport).icon"
alt=""
class="size-10 justify-center bg-n-alpha-3 rounded-xl shrink-0 object-contain border border-n-strong"
/>
<span
v-else
v-tooltip.top="importSourceFor(dataImport).label"
class="size-10 justify-center bg-n-alpha-3 rounded-xl ring ring-n-solid-1 border border-n-strong shadow-sm grid place-items-center"
>
<Icon
:icon="importSourceFor(dataImport).iconClass"
class="size-4"
/>
</span>
<div class="flex min-w-0 flex-col gap-1">
<div class="flex items-center gap-2">
<span class="truncate text-heading-3 text-n-slate-12">
{{ dataImport.name || $t('DATA_IMPORTS.TABLE.UNNAMED') }}
</span>
<span class="flex shrink-0 items-center gap-1.5">
<span
class="size-2 rounded-full"
:class="[
statusDotClass(dataImport.status),
{ 'animate-pulse': isActiveImport(dataImport) },
]"
/>
<span
class="whitespace-nowrap capitalize text-body-main text-n-slate-11"
>
{{ formatStatus(dataImport.status) }}
</span>
</span>
</div>
<div
class="flex flex-wrap items-center gap-2 text-body-main text-n-slate-11"
>
<span>{{ importTypeLabel(dataImport) }}</span>
<div class="h-3 w-px rounded-lg bg-n-strong" />
<span class="tabular-nums">
{{
$t('DATA_IMPORTS.TABLE.IMPORTED_COUNT', {
count: importedCount(dataImport),
})
}}
</span>
<div class="h-3 w-px rounded-lg bg-n-strong" />
<span>{{ formatDate(dataImport.created_at) }}</span>
</div>
</div>
</div>
<Button
v-tooltip.top="$t('DATA_IMPORTS.TABLE.VIEW')"
icon="i-lucide-eye"
slate
sm
class="shrink-0"
@click.stop="openImport(dataImport)"
/>
</div>
</div>
</template>
</SettingsLayout>
<NewImportDialog
:show="showImportDrawer"
:has-active-import="hasActiveIntercomImport"
@close="showImportDrawer = false"
@created="onImportCreated"
/>
</template>
@@ -0,0 +1,210 @@
<script setup>
import { computed, ref, watch } from 'vue';
import { useI18n } from 'vue-i18n';
import { useAlert } from 'dashboard/composables';
import Dialog from 'dashboard/components-next/dialog/Dialog.vue';
import Checkbox from 'dashboard/components-next/checkbox/Checkbox.vue';
import Input from 'dashboard/components-next/input/Input.vue';
import Select from 'dashboard/components-next/select/Select.vue';
import DataImportsAPI from 'dashboard/api/dataImports';
import { IMPORT_SOURCES } from './importSources';
const props = defineProps({
show: { type: Boolean, default: false },
hasActiveImport: { type: Boolean, default: false },
});
const emit = defineEmits(['close', 'created']);
const { t } = useI18n();
const dialogRef = ref(null);
const sourceProvider = ref('intercom');
const importName = ref(t('DATA_IMPORTS.DEFAULT_IMPORT_NAME'));
const accessToken = ref('');
const selectedImportTypes = ref(['contacts', 'conversations']);
const validationState = ref('idle');
const validationMessage = ref('');
const isCreating = ref(false);
let validationRequestId = 0;
const closeDrawer = () => emit('close');
const sourceOptions = computed(() =>
IMPORT_SOURCES.map(({ value, label }) => ({ value, label }))
);
const tokenMessageType = computed(() => {
if (validationState.value === 'valid') return 'success';
if (validationState.value === 'invalid') return 'error';
return 'info';
});
const canCreate = computed(
() =>
validationState.value === 'valid' &&
selectedImportTypes.value.length > 0 &&
!props.hasActiveImport &&
!isCreating.value
);
const validationPayload = () => ({
source_provider: sourceProvider.value,
access_token: accessToken.value.trim(),
import_types: selectedImportTypes.value,
});
const invalidateValidation = () => {
validationRequestId += 1;
validationState.value = 'idle';
validationMessage.value = '';
};
const validateSource = async () => {
if (!accessToken.value.trim() || !selectedImportTypes.value.length) {
invalidateValidation();
return;
}
validationRequestId += 1;
const requestId = validationRequestId;
validationState.value = 'validating';
validationMessage.value = t('DATA_IMPORTS.DRAWER.VALIDATING');
try {
await DataImportsAPI.validateSource(validationPayload());
if (requestId !== validationRequestId) return;
validationState.value = 'valid';
validationMessage.value = t('DATA_IMPORTS.DRAWER.VALID_KEY');
} catch (error) {
if (requestId !== validationRequestId) return;
validationState.value = 'invalid';
validationMessage.value =
error?.response?.data?.message || t('DATA_IMPORTS.DRAWER.INVALID_KEY');
}
};
const toggleImportType = type => {
selectedImportTypes.value = selectedImportTypes.value.includes(type)
? selectedImportTypes.value.filter(item => item !== type)
: [...selectedImportTypes.value, type];
};
const createImport = async () => {
if (!canCreate.value) return;
isCreating.value = true;
try {
const response = await DataImportsAPI.create({
...validationPayload(),
name: importName.value.trim() || t('DATA_IMPORTS.DEFAULT_IMPORT_NAME'),
});
useAlert(t('DATA_IMPORTS.ALERTS.IMPORT_STARTED'));
emit('created', response.data.id);
} catch (error) {
useAlert(
error?.response?.data?.message || t('DATA_IMPORTS.ALERTS.IMPORT_FAILED')
);
} finally {
isCreating.value = false;
}
};
watch(accessToken, invalidateValidation);
watch(selectedImportTypes, () => {
invalidateValidation();
if (accessToken.value.trim() && selectedImportTypes.value.length) {
validateSource();
}
});
watch(
() => props.show,
show => {
if (show) {
dialogRef.value?.open();
return;
}
dialogRef.value?.close();
accessToken.value = '';
validationState.value = 'idle';
validationMessage.value = '';
}
);
</script>
<template>
<Dialog
ref="dialogRef"
:title="$t('DATA_IMPORTS.DRAWER.TITLE')"
:confirm-button-label="$t('DATA_IMPORTS.DRAWER.IMPORT')"
:cancel-button-label="$t('DATA_IMPORTS.DRAWER.CANCEL')"
:disable-confirm-button="!canCreate"
:is-loading="isCreating || validationState === 'validating'"
width="md"
@confirm="createImport"
@close="closeDrawer"
>
<div class="flex flex-col gap-4">
<label class="flex flex-col gap-1.5 text-heading-3 text-n-slate-12">
{{ $t('DATA_IMPORTS.DRAWER.SOURCE') }}
<Select
v-model="sourceProvider"
class="!w-full [&>select]:w-full"
:options="sourceOptions"
/>
</label>
<Input
v-model="importName"
:label="$t('DATA_IMPORTS.DRAWER.NAME')"
:placeholder="$t('DATA_IMPORTS.DRAWER.NAME_PLACEHOLDER')"
/>
<Input
v-model="accessToken"
type="password"
autocomplete="off"
:label="$t('DATA_IMPORTS.DRAWER.ACCESS_KEY')"
:placeholder="$t('DATA_IMPORTS.DRAWER.ACCESS_KEY_PLACEHOLDER')"
:message="validationMessage"
:message-type="tokenMessageType"
@blur="validateSource"
/>
<fieldset class="flex flex-col gap-2.5">
<legend class="mb-1.5 text-heading-3 text-n-slate-12">
{{ $t('DATA_IMPORTS.DRAWER.DATA_TYPES') }}
</legend>
<label
class="inline-flex cursor-pointer items-center gap-2 text-body-main text-n-slate-12"
>
<Checkbox
:model-value="selectedImportTypes.includes('contacts')"
@change="toggleImportType('contacts')"
/>
{{ $t('DATA_IMPORTS.TYPES.CONTACTS') }}
</label>
<label
class="inline-flex cursor-pointer items-center gap-2 text-body-main text-n-slate-12"
>
<Checkbox
:model-value="selectedImportTypes.includes('conversations')"
@change="toggleImportType('conversations')"
/>
{{ $t('DATA_IMPORTS.TYPES.CONVERSATIONS') }}
</label>
</fieldset>
<p
v-if="hasActiveImport"
class="rounded-lg bg-n-amber-2 px-3 py-2 text-body-main text-n-amber-11"
>
{{ $t('DATA_IMPORTS.DRAWER.ACTIVE_IMPORT') }}
</p>
</div>
</Dialog>
</template>
@@ -0,0 +1,238 @@
<script setup>
import {
computed,
onActivated,
onBeforeUnmount,
onDeactivated,
ref,
} from 'vue';
import { useI18n } from 'vue-i18n';
import { useRoute } from 'vue-router';
import { useAlert } from 'dashboard/composables';
import DataImportsAPI from 'dashboard/api/dataImports';
import { POLL_INTERVAL_MS, isActiveImport } from './importStatus';
import SettingsLayout from '../SettingsLayout.vue';
import ImportDetailHeader from './components/ImportDetailHeader.vue';
import ImportSummaryTiles from './components/ImportSummaryTiles.vue';
import ImportProgress from './components/ImportProgress.vue';
import ImportErrorsSection from './components/ImportErrorsSection.vue';
import ImportSkipLogsSection from './components/ImportSkipLogsSection.vue';
const { t } = useI18n();
const route = useRoute();
const dataImport = ref(null);
const isLoading = ref(true);
const isRefreshing = ref(false);
const isPolling = ref(false);
const isAbandoning = ref(false);
const isDownloadingErrorLogs = ref(false);
const isDownloadingSkipLogs = ref(false);
const isChangingSkipLogsType = ref(false);
const selectedSkipLogsType = ref('');
const errorsOpen = ref(true);
const skipLogsOpen = ref(true);
let pollTimer;
let isPageActive = false;
const hasActiveImport = computed(() => isActiveImport(dataImport.value));
const stopPolling = () => {
if (!pollTimer) return;
window.clearInterval(pollTimer);
pollTimer = null;
};
const fetchImport = async ({
showLoader = false,
manual = false,
requestedSkipLogsType = selectedSkipLogsType.value,
} = {}) => {
if (showLoader) {
isLoading.value = true;
} else if (manual) {
isRefreshing.value = true;
}
try {
const response = await DataImportsAPI.show(route.params.dataImportId, {
skip_logs_type: requestedSkipLogsType || undefined,
});
dataImport.value = response.data;
selectedSkipLogsType.value =
response.data.skip_logs_filters?.selected_source_object_type ||
requestedSkipLogsType ||
'';
} finally {
if (showLoader) isLoading.value = false;
if (manual) isRefreshing.value = false;
if (!hasActiveImport.value) stopPolling();
}
};
const changeSkipLogsType = async type => {
if (type === selectedSkipLogsType.value || isChangingSkipLogsType.value) {
return;
}
selectedSkipLogsType.value = type;
isChangingSkipLogsType.value = true;
try {
await fetchImport({ requestedSkipLogsType: type });
} finally {
isChangingSkipLogsType.value = false;
}
};
const refreshImportInBackground = async () => {
if (
!isPageActive ||
isPolling.value ||
!hasActiveImport.value ||
document.hidden
) {
return;
}
isPolling.value = true;
try {
await fetchImport();
} finally {
isPolling.value = false;
if (!hasActiveImport.value) stopPolling();
}
};
const abandonImport = async () => {
isAbandoning.value = true;
try {
const response = await DataImportsAPI.abandon(dataImport.value.id);
dataImport.value = response.data;
stopPolling();
useAlert(t('DATA_IMPORTS.ALERTS.IMPORT_ABANDONED'));
} finally {
isAbandoning.value = false;
}
};
const downloadCsv = (response, filename) => {
const url = window.URL.createObjectURL(
new Blob([response.data], { type: 'text/csv' })
);
const link = document.createElement('a');
link.href = url;
link.download = filename;
link.click();
window.URL.revokeObjectURL(url);
};
const downloadErrorLogs = async () => {
isDownloadingErrorLogs.value = true;
try {
const response = await DataImportsAPI.downloadErrorLogs(
dataImport.value.id
);
downloadCsv(response, `data-import-${dataImport.value.id}-error-logs.csv`);
} finally {
isDownloadingErrorLogs.value = false;
}
};
const downloadSkipLogs = async () => {
isDownloadingSkipLogs.value = true;
try {
const response = await DataImportsAPI.downloadSkipLogs(dataImport.value.id);
downloadCsv(response, `data-import-${dataImport.value.id}-skip-logs.csv`);
} finally {
isDownloadingSkipLogs.value = false;
}
};
const startPolling = () => {
stopPolling();
if (!isPageActive || !hasActiveImport.value) return;
pollTimer = window.setInterval(refreshImportInBackground, POLL_INTERVAL_MS);
};
const handleVisibilityChange = () => {
if (isPageActive && !document.hidden && hasActiveImport.value) {
refreshImportInBackground();
}
};
onActivated(async () => {
isPageActive = true;
await fetchImport({ showLoader: true });
if (!isPageActive) return;
// Collapse empty sections by default; expand the ones with records.
errorsOpen.value = Boolean(dataImport.value?.import_errors_count);
skipLogsOpen.value = Boolean(dataImport.value?.skip_logs_count);
startPolling();
document.addEventListener('visibilitychange', handleVisibilityChange);
});
onDeactivated(() => {
isPageActive = false;
stopPolling();
document.removeEventListener('visibilitychange', handleVisibilityChange);
});
onBeforeUnmount(() => {
isPageActive = false;
stopPolling();
document.removeEventListener('visibilitychange', handleVisibilityChange);
});
</script>
<template>
<SettingsLayout
:is-loading="isLoading"
:loading-message="$t('DATA_IMPORTS.LOADING')"
>
<template #header>
<ImportDetailHeader
:data-import="dataImport"
:is-refreshing="isRefreshing"
:is-abandoning="isAbandoning"
:is-polling="isPolling"
@refresh="fetchImport({ manual: true })"
@abandon="abandonImport"
/>
</template>
<template #body>
<div v-if="dataImport" class="flex flex-col gap-3">
<ImportSummaryTiles :data-import="dataImport" />
<ImportProgress
v-if="dataImport.import_types?.length"
:data-import="dataImport"
:title="$t('DATA_IMPORTS.DETAIL.PROGRESS')"
/>
<ImportErrorsSection
:data-import="dataImport"
:is-open="errorsOpen"
:is-downloading="isDownloadingErrorLogs"
@toggle="errorsOpen = !errorsOpen"
@download="downloadErrorLogs"
/>
<ImportSkipLogsSection
:data-import="dataImport"
:is-open="skipLogsOpen"
:is-downloading="isDownloadingSkipLogs"
:selected-type="selectedSkipLogsType"
:is-changing-type="isChangingSkipLogsType"
@toggle="skipLogsOpen = !skipLogsOpen"
@download="downloadSkipLogs"
@change-type="changeSkipLogsType"
/>
</div>
</template>
</SettingsLayout>
</template>
@@ -0,0 +1,128 @@
<script setup>
import { computed } from 'vue';
import { useI18n } from 'vue-i18n';
import Button from 'dashboard/components-next/button/Button.vue';
import BaseSettingsHeader from '../../components/BaseSettingsHeader.vue';
import {
POLL_INTERVAL_MS,
importStageKey,
isAbandonableImport,
isActiveImport,
statusDotClass as getStatusDotClass,
} from '../importStatus';
const props = defineProps({
dataImport: {
type: Object,
default: null,
},
isRefreshing: {
type: Boolean,
default: false,
},
isAbandoning: {
type: Boolean,
default: false,
},
isPolling: {
type: Boolean,
default: false,
},
});
defineEmits(['refresh', 'abandon']);
const { t } = useI18n();
const pollIntervalSeconds = POLL_INTERVAL_MS / 1000;
const title = computed(
() => props.dataImport?.name || t('DATA_IMPORTS.TABLE.UNNAMED')
);
const stageLabels = computed(() => ({
unknown: t('DATA_IMPORTS.MONITOR.STAGES.unknown'),
queued: t('DATA_IMPORTS.MONITOR.STAGES.queued'),
contacts: t('DATA_IMPORTS.MONITOR.STAGES.contacts'),
conversations: t('DATA_IMPORTS.MONITOR.STAGES.conversations'),
finalizing: t('DATA_IMPORTS.MONITOR.STAGES.finalizing'),
completed: t('DATA_IMPORTS.MONITOR.STAGES.completed'),
completed_with_errors: t('DATA_IMPORTS.MONITOR.STAGES.completed_with_errors'),
failed: t('DATA_IMPORTS.MONITOR.STAGES.failed'),
abandoned: t('DATA_IMPORTS.MONITOR.STAGES.abandoned'),
}));
const monitorTitle = computed(
() =>
stageLabels.value[importStageKey(props.dataImport)] ||
stageLabels.value.unknown
);
const statusDotClass = computed(() =>
getStatusDotClass(props.dataImport?.status)
);
const hasActiveImport = computed(() => isActiveImport(props.dataImport));
const canAbandonImport = computed(() => isAbandonableImport(props.dataImport));
</script>
<template>
<BaseSettingsHeader
:title="title"
:back-button-label="$t('DATA_IMPORTS.DETAIL.BACK')"
>
<template #title>
<div class="flex w-full items-center justify-between gap-4">
<h1 class="min-w-0 truncate text-heading-1 text-n-slate-12">
{{ title }}
</h1>
<div class="flex shrink-0 items-center gap-2">
<Button
v-if="hasActiveImport"
outline
slate
size="sm"
icon="i-lucide-refresh-cw"
:is-loading="isRefreshing"
:aria-label="$t('DATA_IMPORTS.MONITOR.REFRESH')"
:title="$t('DATA_IMPORTS.MONITOR.REFRESH')"
@click="$emit('refresh')"
/>
<Button
v-if="canAbandonImport"
ruby
size="sm"
:is-loading="isAbandoning"
:label="$t('DATA_IMPORTS.TABLE.ABANDON')"
@click="$emit('abandon')"
/>
</div>
</div>
</template>
<template #description>
<span class="inline-flex items-center gap-1.5 align-middle">
<span
class="size-2 rounded-full"
:class="[statusDotClass, { 'animate-pulse': hasActiveImport }]"
/>
{{ monitorTitle }}
</span>
<template v-if="hasActiveImport">
<span
class="mx-2 inline-block h-3 w-px rounded-lg bg-n-strong align-middle"
/>
<span class="text-n-teal-11">
{{
isPolling
? $t('DATA_IMPORTS.MONITOR.REFRESHING')
: $t('DATA_IMPORTS.MONITOR.LIVE', {
seconds: pollIntervalSeconds,
})
}}
</span>
</template>
</template>
</BaseSettingsHeader>
</template>
@@ -0,0 +1,78 @@
<script setup>
import { computed } from 'vue';
import { useI18n } from 'vue-i18n';
import { BaseTableRow, BaseTableCell } from 'dashboard/components-next/table';
import ImportLogSection from './ImportLogSection.vue';
import { formatDate, sourceObjectLabel } from '../importStatus';
const props = defineProps({
dataImport: {
type: Object,
required: true,
},
isOpen: {
type: Boolean,
default: false,
},
isDownloading: {
type: Boolean,
default: false,
},
});
defineEmits(['toggle', 'download']);
const { t } = useI18n();
const errors = computed(() => props.dataImport?.import_errors || []);
const headers = computed(() => [
t('DATA_IMPORTS.DETAIL.ERROR_CODE'),
t('DATA_IMPORTS.DETAIL.SOURCE_OBJECT'),
t('DATA_IMPORTS.DETAIL.MESSAGE'),
t('DATA_IMPORTS.DETAIL.CREATED'),
]);
</script>
<template>
<ImportLogSection
:title="$t('DATA_IMPORTS.DETAIL.ERRORS')"
:count="dataImport.import_errors_count"
:is-open="isOpen"
:is-downloading="isDownloading"
:download-label="$t('DATA_IMPORTS.DETAIL.DOWNLOAD_ERROR_LOGS')"
:headers="headers"
:items="errors"
:empty-message="$t('DATA_IMPORTS.DETAIL.NO_ERRORS')"
@toggle="$emit('toggle')"
@download="$emit('download')"
>
<template #row="{ items }">
<BaseTableRow v-for="error in items" :key="error.id" :item="error">
<template #default>
<BaseTableCell>
<span class="text-body-main text-n-slate-12">
{{ error.error_code }}
</span>
</BaseTableCell>
<BaseTableCell>
<span class="text-body-main text-n-slate-12">
{{ sourceObjectLabel(error) }}
</span>
</BaseTableCell>
<BaseTableCell>
<span class="text-body-main text-n-slate-11">
{{ error.message || '-' }}
</span>
</BaseTableCell>
<BaseTableCell>
<span class="whitespace-nowrap text-body-main text-n-slate-11">
{{ formatDate(error.created_at) }}
</span>
</BaseTableCell>
</template>
</BaseTableRow>
</template>
</ImportLogSection>
</template>
@@ -0,0 +1,105 @@
<script setup>
import Button from 'dashboard/components-next/button/Button.vue';
import Icon from 'dashboard/components-next/icon/Icon.vue';
import { BaseTable } from 'dashboard/components-next/table';
defineProps({
title: {
type: String,
required: true,
},
count: {
type: Number,
default: 0,
},
isOpen: {
type: Boolean,
default: false,
},
isDownloading: {
type: Boolean,
default: false,
},
downloadLabel: {
type: String,
default: '',
},
headers: {
type: Array,
default: () => [],
},
items: {
type: Array,
default: () => [],
},
emptyMessage: {
type: String,
default: '',
},
});
defineEmits(['toggle', 'download']);
</script>
<template>
<section class="overflow-hidden rounded-xl border border-n-weak bg-n-solid-1">
<div class="flex items-center justify-between gap-3 px-4 py-3">
<button
type="button"
class="flex min-w-0 items-center gap-2 !p-0"
:aria-expanded="isOpen"
@click="$emit('toggle')"
>
<h2 class="text-heading-3 text-n-slate-12">{{ title }}</h2>
<span
v-if="count"
class="rounded-md bg-n-alpha-2 px-1.5 text-label-small tabular-nums text-n-slate-11"
>
{{ count }}
</span>
<Icon
icon="i-lucide-chevron-down"
class="size-4 shrink-0 text-n-slate-10 transition-transform duration-200"
:class="{ '-rotate-90 rtl:rotate-90': !isOpen }"
/>
</button>
<Button
ghost
slate
xs
icon="i-lucide-download"
:is-loading="isDownloading"
:disabled="!count"
:label="downloadLabel"
@click="$emit('download')"
/>
</div>
<div
class="grid transition-[grid-template-rows] duration-300 ease-in-out"
:class="isOpen ? 'grid-rows-[1fr]' : 'grid-rows-[0fr]'"
>
<div class="min-h-0 overflow-hidden">
<div class="border-t border-n-weak">
<slot name="filters" />
<p
v-if="!items.length"
class="px-4 py-8 text-center text-body-main text-n-slate-11"
>
{{ emptyMessage }}
</p>
<div v-else class="overflow-x-auto">
<BaseTable
class="[&_td:first-child]:ps-4 [&_th:first-child]:ps-4 [&_th]:text-n-slate-11 [&_thead]:border-t-0"
:headers="headers"
:items="items"
>
<template #row="{ items: rows }">
<slot name="row" :items="rows" />
</template>
</BaseTable>
</div>
</div>
</div>
</div>
</section>
</template>
@@ -0,0 +1,100 @@
<script setup>
import { computed } from 'vue';
import { useI18n } from 'vue-i18n';
const props = defineProps({
dataImport: {
type: Object,
required: true,
},
title: {
type: String,
required: true,
},
});
const { t } = useI18n();
const items = computed(() => {
const importTypes = props.dataImport?.import_types || [];
const groups = [];
if (importTypes.includes('contacts')) {
groups.push({ key: 'contacts', label: t('DATA_IMPORTS.TYPES.CONTACTS') });
}
if (importTypes.includes('conversations')) {
groups.push(
{ key: 'conversations', label: t('DATA_IMPORTS.TYPES.CONVERSATIONS') },
{ key: 'messages', label: t('DATA_IMPORTS.TYPES.MESSAGES') }
);
}
return groups.map(({ key, label }) => {
const stats = props.dataImport?.stats?.[key] || {};
const imported = Number(stats.imported || 0);
const hasTotal = Object.prototype.hasOwnProperty.call(stats, 'total');
const total = hasTotal ? Number(stats.total) : null;
const percent =
hasTotal && total > 0
? Math.min(100, Math.round((imported / total) * 100))
: null;
return {
key,
label,
percent,
importedLabel: imported.toLocaleString(),
caption: hasTotal
? t('DATA_IMPORTS.DETAIL.PROGRESS_OF_TOTAL', {
total: total.toLocaleString(),
})
: t('DATA_IMPORTS.DETAIL.PROGRESS_IMPORTED'),
};
});
});
// Fit the grid to the number of groups so no empty cells show.
const columnsClass = computed(() => {
if (items.value.length >= 3) return 'sm:grid-cols-3';
if (items.value.length === 2) return 'sm:grid-cols-2';
return 'sm:grid-cols-1';
});
</script>
<template>
<section class="overflow-hidden rounded-xl border border-n-weak bg-n-solid-1">
<h2 class="border-b border-n-weak px-4 py-3 text-heading-3 text-n-slate-12">
{{ title }}
</h2>
<div class="grid grid-cols-1 gap-px bg-n-weak" :class="columnsClass">
<div
v-for="item in items"
:key="item.key"
class="flex flex-col gap-2 bg-n-solid-1 px-4 py-3"
>
<span class="text-label-small text-n-slate-11">{{ item.label }}</span>
<div class="flex items-end justify-between gap-2">
<span
class="text-xl font-semibold tracking-tight tabular-nums text-n-slate-12"
>
{{ item.importedLabel }}
</span>
<span
v-if="item.percent !== null"
class="text-label-small tabular-nums text-n-slate-11"
>
{{ `${item.percent}%` }}
</span>
</div>
<div
v-if="item.percent !== null"
class="h-1.5 w-full overflow-hidden rounded-full bg-n-alpha-2"
>
<div
class="h-full rounded-full bg-n-brand transition-all duration-500"
:style="{ width: `${item.percent}%` }"
/>
</div>
<span class="text-label-small text-n-slate-10">{{ item.caption }}</span>
</div>
</div>
</section>
</template>
@@ -0,0 +1,127 @@
<script setup>
import { computed } from 'vue';
import { useI18n } from 'vue-i18n';
import Button from 'dashboard/components-next/button/Button.vue';
import { BaseTableRow, BaseTableCell } from 'dashboard/components-next/table';
import ImportLogSection from './ImportLogSection.vue';
import { formatDate, sourceObjectLabel } from '../importStatus';
const props = defineProps({
dataImport: {
type: Object,
required: true,
},
isOpen: {
type: Boolean,
default: false,
},
isDownloading: {
type: Boolean,
default: false,
},
selectedType: {
type: String,
default: '',
},
isChangingType: {
type: Boolean,
default: false,
},
});
defineEmits(['toggle', 'download', 'changeType']);
const { t } = useI18n();
const skipLogs = computed(() => props.dataImport?.skip_logs || []);
const headers = computed(() => [
t('DATA_IMPORTS.DETAIL.KIND'),
t('DATA_IMPORTS.DETAIL.SOURCE_OBJECT'),
t('DATA_IMPORTS.DETAIL.MESSAGE'),
t('DATA_IMPORTS.DETAIL.CREATED'),
]);
const typeOptions = computed(() => {
const counts = props.dataImport?.skip_logs_filters?.counts_by_type || {};
return [
{
value: '',
label: t('DATA_IMPORTS.DETAIL.ALL_SKIP_LOGS'),
count: props.dataImport?.skip_logs_count || 0,
},
{
value: 'contact',
label: t('DATA_IMPORTS.TYPES.CONTACTS'),
count: counts.contact || 0,
},
{
value: 'conversation',
label: t('DATA_IMPORTS.TYPES.CONVERSATIONS'),
count: counts.conversation || 0,
},
{
value: 'message',
label: t('DATA_IMPORTS.TYPES.MESSAGES'),
count: counts.message || 0,
},
];
});
</script>
<template>
<ImportLogSection
:title="$t('DATA_IMPORTS.DETAIL.SKIP_LOGS')"
:count="dataImport.skip_logs_count"
:is-open="isOpen"
:is-downloading="isDownloading"
:download-label="$t('DATA_IMPORTS.DETAIL.DOWNLOAD_SKIP_LOGS')"
:headers="headers"
:items="skipLogs"
:empty-message="$t('DATA_IMPORTS.DETAIL.NO_SKIP_LOGS')"
@toggle="$emit('toggle')"
@download="$emit('download')"
>
<template v-if="dataImport.skip_logs_count" #filters>
<div class="flex flex-wrap gap-2 border-b border-n-weak px-4 py-3">
<Button
v-for="option in typeOptions"
:key="option.value || 'all'"
:variant="option.value === selectedType ? 'solid' : 'faded'"
color="slate"
size="xs"
:disabled="!option.count || isChangingType"
:label="`${option.label} (${option.count})`"
@click="$emit('changeType', option.value)"
/>
</div>
</template>
<template #row="{ items }">
<BaseTableRow v-for="skipLog in items" :key="skipLog.id" :item="skipLog">
<template #default>
<BaseTableCell>
<span class="capitalize text-body-main text-n-slate-12">
{{ skipLog.kind || '-' }}
</span>
</BaseTableCell>
<BaseTableCell>
<span class="text-body-main text-n-slate-11">
{{ sourceObjectLabel(skipLog) }}
</span>
</BaseTableCell>
<BaseTableCell>
<span class="text-body-main text-n-slate-11">
{{ skipLog.message || '-' }}
</span>
</BaseTableCell>
<BaseTableCell>
<span class="whitespace-nowrap text-body-main text-n-slate-11">
{{ formatDate(skipLog.created_at) }}
</span>
</BaseTableCell>
</template>
</BaseTableRow>
</template>
</ImportLogSection>
</template>
@@ -0,0 +1,110 @@
<script setup>
import { computed } from 'vue';
import { useI18n } from 'vue-i18n';
import formatDistanceStrict from 'date-fns/formatDistanceStrict';
import Icon from 'dashboard/components-next/icon/Icon.vue';
import { formatDate, isActiveImport } from '../importStatus';
import { importSourceFor } from '../importSources';
const props = defineProps({
dataImport: {
type: Object,
required: true,
},
});
const { t } = useI18n();
const importTypeLabel = type => {
if (type === 'contacts') return t('DATA_IMPORTS.TYPES.CONTACTS');
if (type === 'conversations') return t('DATA_IMPORTS.TYPES.CONVERSATIONS');
return type;
};
const importTypesLabel = computed(() => {
const importTypes = props.dataImport?.import_types?.length
? props.dataImport.import_types
: [props.dataImport?.data_type].filter(Boolean);
return importTypes.map(importTypeLabel).join(', ');
});
const runDuration = computed(() => {
const startedAt =
props.dataImport?.started_at || props.dataImport?.created_at;
if (!startedAt) return '-';
const finishedAt =
props.dataImport?.completed_at ||
props.dataImport?.abandoned_at ||
(isActiveImport(props.dataImport)
? new Date()
: props.dataImport?.updated_at);
if (!finishedAt) return '-';
return formatDistanceStrict(new Date(startedAt), new Date(finishedAt));
});
const items = computed(() => [
{
key: 'source',
icon: 'i-lucide-plug',
label: t('DATA_IMPORTS.DETAIL.SOURCE'),
value: importSourceFor(props.dataImport).label,
},
{
key: 'import_types',
icon: 'i-lucide-layers',
label: t('DATA_IMPORTS.DETAIL.IMPORT_TYPES'),
value: importTypesLabel.value || '-',
},
{
key: 'created_at',
icon: 'i-lucide-calendar',
label: t('DATA_IMPORTS.DETAIL.CREATED'),
value: formatDate(props.dataImport?.created_at),
},
{
key: 'duration',
icon: 'i-lucide-clock',
label: t('DATA_IMPORTS.DETAIL.DURATION'),
value: runDuration.value,
tooltip: t('DATA_IMPORTS.DETAIL.LAST_UPDATED_TOOLTIP', {
time: formatDate(props.dataImport?.updated_at),
}),
},
{
key: 'initiated_by',
icon: 'i-lucide-user',
label: t('DATA_IMPORTS.DETAIL.INITIATED_BY'),
value:
props.dataImport?.initiated_by?.name ||
props.dataImport?.initiated_by?.email ||
'-',
},
]);
</script>
<template>
<dl
class="grid grid-cols-2 gap-px overflow-hidden rounded-xl border border-n-weak bg-n-weak sm:grid-cols-3 lg:grid-cols-5"
>
<div
v-for="item in items"
:key="item.key"
class="flex min-w-0 flex-col gap-1 bg-n-solid-1 px-4 py-3"
>
<dt class="flex items-center gap-1.5 text-label-small text-n-slate-10">
<Icon :icon="item.icon" class="size-3.5 shrink-0" />
{{ item.label }}
</dt>
<dd
v-tooltip.top="item.tooltip"
class="truncate text-heading-3 text-n-slate-12"
>
{{ item.value }}
</dd>
</div>
</dl>
</template>
@@ -0,0 +1,34 @@
import { FEATURE_FLAGS } from '../../../../featureFlags';
import { frontendURL } from '../../../../helper/URLHelper';
import SettingsWrapper from '../SettingsWrapper.vue';
import Index from './Index.vue';
import Show from './Show.vue';
export default {
routes: [
{
path: frontendURL('accounts/:accountId/settings/data'),
component: SettingsWrapper,
children: [
{
path: '',
name: 'settings_data_imports',
component: Index,
meta: {
featureFlag: FEATURE_FLAGS.DATA_IMPORT,
permissions: ['administrator'],
},
},
{
path: ':dataImportId',
name: 'settings_data_import_show',
component: Show,
meta: {
featureFlag: FEATURE_FLAGS.DATA_IMPORT,
permissions: ['administrator'],
},
},
],
},
],
};
@@ -0,0 +1,17 @@
export const IMPORT_SOURCES = [
{
value: 'intercom',
label: 'Intercom',
icon: '/dashboard/images/integrations/intercom.png',
},
];
const DEFAULT_IMPORT_SOURCE = {
value: 'file',
label: 'File import',
iconClass: 'i-lucide-file-text',
};
export const importSourceFor = dataImport =>
IMPORT_SOURCES.find(source => source.value === dataImport?.source_provider) ||
DEFAULT_IMPORT_SOURCE;
@@ -0,0 +1,84 @@
export const POLL_INTERVAL_MS = 5000;
export const ACTIVE_IMPORT_STATUSES = ['pending', 'processing'];
export const isActiveImport = dataImport =>
ACTIVE_IMPORT_STATUSES.includes(dataImport?.status);
export const isIntercomImport = dataImport =>
dataImport?.data_type === 'intercom' &&
dataImport?.source_provider === 'intercom';
export const isActiveIntercomImport = dataImport =>
isIntercomImport(dataImport) && isActiveImport(dataImport);
export const isAbandonableImport = dataImport =>
isActiveIntercomImport(dataImport);
export const importedCount = dataImport => {
if (!isIntercomImport(dataImport)) {
return Number(dataImport?.processed_records || 0);
}
return ['contacts', 'conversations', 'messages'].reduce(
(total, key) => total + Number(dataImport?.stats?.[key]?.imported || 0),
0
);
};
export const importStageKey = dataImport => {
if (!dataImport) return 'unknown';
if (dataImport.status === 'completed') return 'completed';
if (dataImport.status === 'completed_with_errors') {
return 'completed_with_errors';
}
if (dataImport.status === 'failed') return 'failed';
if (dataImport.status === 'abandoned') return 'abandoned';
if (dataImport.status === 'pending') return 'queued';
const importTypes = dataImport.import_types?.length
? dataImport.import_types
: [dataImport.data_type];
const cursor = dataImport.cursor || {};
if (importTypes.includes('contacts') && !cursor.contacts?.completed) {
return 'contacts';
}
if (
importTypes.includes('conversations') &&
!cursor.conversations?.completed
) {
return 'conversations';
}
return 'finalizing';
};
export const formatStatus = value => value?.replaceAll('_', ' ') || '-';
export const sourceObjectLabel = record =>
[record.source_object_type, record.source_object_id]
.filter(Boolean)
.join(': ') || '-';
export const formatDate = value => {
if (!value) return '-';
return new Intl.DateTimeFormat(undefined, {
dateStyle: 'medium',
timeStyle: 'short',
}).format(new Date(value));
};
const STATUS_DOT_CLASS = {
pending: 'bg-n-amber-9',
processing: 'bg-n-blue-9',
completed: 'bg-n-teal-9',
completed_with_errors: 'bg-n-amber-9',
failed: 'bg-n-ruby-9',
abandoned: 'bg-n-slate-9',
};
export const statusDotClass = status =>
STATUS_DOT_CLASS[status] || 'bg-n-slate-9';
@@ -0,0 +1,92 @@
import {
formatDate,
importedCount,
isActiveIntercomImport,
statusDotClass,
} from '../importStatus';
describe('importStatus', () => {
describe('isActiveIntercomImport', () => {
it('only treats pending or processing Intercom imports as active', () => {
expect(
isActiveIntercomImport({
data_type: 'intercom',
source_provider: 'intercom',
status: 'processing',
})
).toBe(true);
expect(
isActiveIntercomImport({
data_type: 'contacts',
source_provider: null,
status: 'processing',
})
).toBe(false);
expect(
isActiveIntercomImport({
data_type: 'intercom',
source_provider: 'intercom',
status: 'completed',
})
).toBe(false);
});
});
describe('importedCount', () => {
it('sums Intercom imported stats', () => {
expect(
importedCount({
data_type: 'intercom',
source_provider: 'intercom',
processed_records: 20,
stats: {
contacts: { imported: 2 },
conversations: { imported: 3 },
messages: { imported: 10 },
},
})
).toBe(15);
});
it('uses processed records for legacy imports', () => {
expect(
importedCount({
data_type: 'contacts',
source_provider: null,
processed_records: 7,
stats: {},
})
).toBe(7);
});
});
describe('statusDotClass', () => {
it('maps each status to its dot color class', () => {
expect(statusDotClass('pending')).toBe('bg-n-amber-9');
expect(statusDotClass('processing')).toBe('bg-n-blue-9');
expect(statusDotClass('completed')).toBe('bg-n-teal-9');
expect(statusDotClass('completed_with_errors')).toBe('bg-n-amber-9');
expect(statusDotClass('failed')).toBe('bg-n-ruby-9');
expect(statusDotClass('abandoned')).toBe('bg-n-slate-9');
});
it('falls back to slate for unknown or missing status', () => {
expect(statusDotClass('unknown')).toBe('bg-n-slate-9');
expect(statusDotClass(undefined)).toBe('bg-n-slate-9');
});
});
describe('formatDate', () => {
it('returns a dash for empty values', () => {
expect(formatDate(null)).toBe('-');
expect(formatDate('')).toBe('-');
expect(formatDate(undefined)).toBe('-');
});
it('formats a valid date into a readable string', () => {
const formatted = formatDate('2026-07-10T18:09:00Z');
expect(formatted).not.toBe('-');
expect(formatted).toContain('2026');
});
});
});
@@ -0,0 +1,121 @@
import { flushPromises, mount } from '@vue/test-utils';
import { KeepAlive, defineComponent, h, nextTick, ref } from 'vue';
import DataImportsAPI from 'dashboard/api/dataImports';
import Index from '../Index.vue';
import Show from '../Show.vue';
vi.mock('dashboard/api/dataImports', () => ({
default: {
get: vi.fn(),
show: vi.fn(),
},
}));
vi.mock('dashboard/composables/store', () => ({
useStoreGetters: () => ({ getCurrentAccountId: { value: 1 } }),
}));
vi.mock('dashboard/composables', () => ({
useAlert: vi.fn(),
}));
vi.mock('vue-i18n', () => ({
useI18n: () => ({ t: key => key }),
}));
vi.mock('vue-router', async importOriginal => ({
...(await importOriginal()),
useRoute: () => ({ params: { dataImportId: 1 } }),
useRouter: () => ({ push: vi.fn() }),
}));
const deferredRequest = () => {
let resolve;
const promise = new Promise(resolvePromise => {
resolve = resolvePromise;
});
return { promise, resolve };
};
const mountKeptAlive = component => {
const Host = defineComponent({
setup() {
const visible = ref(true);
return { visible };
},
render() {
return h(KeepAlive, null, {
default: () => (this.visible ? h(component) : null),
});
},
});
return mount(Host, {
global: {
stubs: {
SettingsLayout: true,
BaseSettingsHeader: true,
Button: true,
Icon: true,
TabBar: true,
NewImportDialog: true,
ImportDetailHeader: true,
ImportSummaryTiles: true,
ImportProgress: true,
ImportErrorsSection: true,
ImportSkipLogsSection: true,
},
mocks: {
$t: key => key,
},
},
});
};
describe('data import polling lifecycle', () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
});
it('does not start list polling after the page deactivates', async () => {
const request = deferredRequest();
DataImportsAPI.get.mockReturnValue(request.promise);
const wrapper = mountKeptAlive(Index);
await nextTick();
wrapper.vm.visible = false;
await nextTick();
request.resolve({ data: { payload: [{ status: 'processing' }] } });
await flushPromises();
await vi.advanceTimersByTimeAsync(5000);
expect(DataImportsAPI.get).toHaveBeenCalledTimes(1);
wrapper.unmount();
});
it('does not start detail polling after the page deactivates', async () => {
const request = deferredRequest();
DataImportsAPI.show.mockReturnValue(request.promise);
const wrapper = mountKeptAlive(Show);
await nextTick();
wrapper.vm.visible = false;
await nextTick();
request.resolve({
data: {
status: 'processing',
skip_logs_filters: {},
},
});
await flushPromises();
await vi.advanceTimersByTimeAsync(5000);
expect(DataImportsAPI.show).toHaveBeenCalledTimes(1);
wrapper.unmount();
});
});
@@ -1,5 +1,5 @@
<script setup>
import { computed, ref } from 'vue';
import { computed, onActivated, ref } from 'vue';
import { useI18n } from 'vue-i18n';
import { useAlert } from 'dashboard/composables';
import { picoSearch } from '@scmmishra/pico-search';
@@ -27,6 +27,10 @@ const searchQuery = ref('');
const inboxes = useMapGetter('inboxes/getInboxes');
onActivated(() => {
store.dispatch('inboxes/get');
});
const inboxesList = computed(() => {
return inboxes.value?.slice().sort((a, b) => a.name.localeCompare(b.name));
});
@@ -26,6 +26,7 @@ import profile from './profile/profile.routes';
import security from './security/security.routes';
import conversationWorkflow from './conversationWorkflow/conversationWorkflow.routes';
import captain from './captain/captain.routes';
import data from './data/data.routes';
export default {
routes: [
@@ -57,6 +58,7 @@ export default {
...canned.routes,
...inbox.routes,
...integrations.routes,
...data.routes,
...labels.routes,
...macros.routes,
...reports.routes,
@@ -0,0 +1,43 @@
class DataImports::Intercom::BaseJob < ApplicationJob
queue_as :low
retry_on DataImports::Intercom::Client::Error, wait: 1.minute, attempts: 3 do |job, error|
job.fail_import!(error)
end
retry_on DataImports::Intercom::Client::RateLimitError, wait: 1.minute, attempts: 5 do |job, error|
job.fail_import!(error)
end
def fail_import!(error)
data_import = arguments.first
run_id = arguments.length > 1 ? arguments.last : nil
return if data_import.blank? || skip_import?(data_import, run_id)
DataImports::Intercom::Importer.new(data_import: data_import, run_id: run_id).fail!(error)
end
private
def skip_import?(data_import, run_id = nil)
data_import.reload
data_import.abandoned? || data_import.failed? || data_import.completed? ||
data_import.completed_with_errors? || stale_import_run?(data_import, run_id)
end
def stale_import_run?(data_import, run_id)
active_run_id = data_import.active_intercom_import_run_id
active_run_id.present? && active_run_id != run_id
end
def importer_for(data_import, run_id = nil)
DataImports::Intercom::Importer.new(data_import: data_import, run_id: run_id)
end
def fail_unexpected_error(importer, error)
raise error if error.is_a?(DataImports::Intercom::Client::Error)
importer&.fail!(error)
raise error
end
end
@@ -0,0 +1,26 @@
class DataImports::Intercom::ContactsPageJob < DataImports::Intercom::BaseJob
def perform(data_import, starting_after = nil, run_id = nil)
return if skip_import?(data_import, run_id)
importer = importer_for(data_import, run_id)
return enqueue_conversations_or_finish(data_import, importer, run_id) if importer.contacts_completed?
result = importer.import_contacts_page(starting_after: starting_after)
return if skip_import?(data_import, run_id)
return self.class.perform_later(data_import, result.next_cursor, run_id) unless result.done?
enqueue_conversations_or_finish(data_import, importer, run_id)
rescue StandardError => e
fail_unexpected_error(importer, e)
end
private
def enqueue_conversations_or_finish(data_import, importer, run_id)
if importer.import_conversations? && !importer.conversations_completed?
DataImports::Intercom::ConversationsPageJob.perform_later(data_import, importer.cursor_for('conversations'), run_id)
else
importer.finish!
end
end
end
@@ -0,0 +1,16 @@
class DataImports::Intercom::ConversationsPageJob < DataImports::Intercom::BaseJob
def perform(data_import, starting_after = nil, run_id = nil)
return if skip_import?(data_import, run_id)
importer = importer_for(data_import, run_id)
return importer.finish! if importer.conversations_completed?
result = importer.import_conversations_page(starting_after: starting_after)
return if skip_import?(data_import, run_id)
return self.class.perform_later(data_import, result.next_cursor, run_id) unless result.done?
importer.finish!
rescue StandardError => e
fail_unexpected_error(importer, e)
end
end
@@ -0,0 +1,24 @@
class DataImports::Intercom::ImportJob < DataImports::Intercom::BaseJob
def perform(data_import, run_id = nil)
return if skip_import?(data_import, run_id)
importer = importer_for(data_import, run_id)
return unless importer.start!
enqueue_next_stage(data_import, importer, run_id)
rescue StandardError => e
fail_unexpected_error(importer, e)
end
private
def enqueue_next_stage(data_import, importer, run_id)
if importer.import_contacts? && !importer.contacts_completed?
DataImports::Intercom::ContactsPageJob.perform_later(data_import, importer.cursor_for('contacts'), run_id)
elsif importer.import_conversations? && !importer.conversations_completed?
DataImports::Intercom::ConversationsPageJob.perform_later(data_import, importer.cursor_for('conversations'), run_id)
else
importer.finish!
end
end
end
+85 -3
View File
@@ -3,33 +3,115 @@
# Table name: data_imports
#
# id :bigint not null, primary key
# abandoned_at :datetime
# access_token :text
# completed_at :datetime
# cursor :jsonb not null
# data_type :string not null
# import_types :jsonb not null
# last_error_at :datetime
# name :string
# processed_records :integer
# processing_errors :text
# source_metadata :jsonb not null
# source_provider :string
# source_type :string
# started_at :datetime
# stats :jsonb not null
# status :integer default("pending"), not null
# total_records :integer
# created_at :datetime not null
# updated_at :datetime not null
# account_id :bigint not null
# initiated_by_id :integer
#
# Indexes
#
# index_data_imports_on_account_id (account_id)
# index_data_imports_on_account_id (account_id)
# index_data_imports_on_initiated_by_id (initiated_by_id)
# index_data_imports_on_source_provider (source_provider)
#
class DataImport < ApplicationRecord
ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY = 'active_intercom_import_run_id'.freeze
LEGACY_DATA_TYPES = ['contacts'].freeze
INTEGRATION_DATA_TYPES = ['intercom'].freeze
IMPORT_TYPES = %w[contacts conversations].freeze
belongs_to :account
validates :data_type, inclusion: { in: ['contacts'], message: I18n.t('errors.data_import.data_type.invalid') }
enum status: { pending: 0, processing: 1, completed: 2, failed: 3 }
belongs_to :initiated_by, class_name: 'User', optional: true
encrypts :access_token if Chatwoot.encryption_configured?
has_many :items, class_name: 'DataImportItem', dependent: :destroy_async
has_many :mappings, class_name: 'DataImportMapping', dependent: :destroy_async
has_many :import_errors, class_name: 'DataImportError', dependent: :destroy_async
validates :data_type, inclusion: { in: LEGACY_DATA_TYPES + INTEGRATION_DATA_TYPES, message: I18n.t('errors.data_import.data_type.invalid') }
validates :access_token, presence: true, on: :create, if: :intercom_import?
validate :validate_import_types
enum status: { pending: 0, processing: 1, completed: 2, failed: 3, completed_with_errors: 6, abandoned: 7 }
scope :active_intercom, -> { where(data_type: 'intercom', source_provider: 'intercom', status: [:pending, :processing]) }
has_one_attached :import_file
has_one_attached :failed_records
after_create_commit :process_data_import
def legacy_contacts_csv_import?
data_type == 'contacts' && source_provider.blank?
end
def intercom_import?
data_type == 'intercom' && source_provider == 'intercom'
end
def restartable?
failed? || abandoned?
end
def abandonable?
intercom_import? && (pending? || processing?)
end
def abandon!
self.class.transaction do
abandonable_import = self.class.lock.find_by(
id: id,
data_type: 'intercom',
source_provider: 'intercom',
status: [:pending, :processing]
)
abandonable_import&.update!(status: :abandoned, abandoned_at: Time.current)
end
reload
end
def active_intercom_import_run_id
source_metadata.to_h[ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY]
end
def assign_active_intercom_import_run_id
self.source_metadata = source_metadata.to_h.merge(ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => SecureRandom.uuid)
active_intercom_import_run_id
end
private
def process_data_import
return unless legacy_contacts_csv_import?
# we wait for the file to be uploaded to the cloud
DataImportJob.set(wait: 1.minute).perform_later(self)
end
def validate_import_types
return if import_types.blank?
invalid_types = import_types - IMPORT_TYPES
return if invalid_types.blank?
errors.add(:import_types, "contains unsupported values: #{invalid_types.join(', ')}")
end
end
+33
View File
@@ -0,0 +1,33 @@
# == Schema Information
#
# Table name: data_import_errors
#
# id :bigint not null, primary key
# details :jsonb not null
# error_code :string not null
# message :text
# source_object_type :string
# created_at :datetime not null
# updated_at :datetime not null
# data_import_id :bigint not null
# data_import_item_id :bigint
# source_object_id :string
#
# Indexes
#
# idx_data_import_errors_on_source (source_object_type,source_object_id)
# index_data_import_errors_on_data_import_id (data_import_id)
# index_data_import_errors_on_data_import_item_id (data_import_item_id)
#
class DataImportError < ApplicationRecord
SKIP_LOG_KINDS = %w[failed skipped].freeze
belongs_to :data_import
belongs_to :data_import_item, optional: true
validates :error_code, presence: true
scope :skip_logs, -> { where("details ->> 'kind' IN (:kinds)", kinds: SKIP_LOG_KINDS) }
scope :failed, -> { where("details ->> 'kind' = ?", 'failed') }
scope :non_skip_logs, -> { where("details ->> 'kind' IS NULL OR details ->> 'kind' NOT IN (:kinds)", kinds: SKIP_LOG_KINDS) }
end
+35
View File
@@ -0,0 +1,35 @@
# == Schema Information
#
# Table name: data_import_items
#
# id :bigint not null, primary key
# attempt_count :integer default(0), not null
# chatwoot_record_type :string
# last_error_code :string
# last_error_message :text
# metadata :jsonb not null
# source_object_type :string not null
# source_provider :string not null
# status :integer default("pending"), not null
# created_at :datetime not null
# updated_at :datetime not null
# chatwoot_record_id :bigint
# data_import_id :bigint not null
# source_object_id :string not null
#
# Indexes
#
# idx_data_import_items_on_import_and_source (data_import_id,source_object_type,source_object_id) UNIQUE
# idx_data_import_items_on_record (chatwoot_record_type,chatwoot_record_id)
# idx_data_import_items_on_source (source_provider,source_object_type,source_object_id)
# index_data_import_items_on_data_import_id (data_import_id)
#
class DataImportItem < ApplicationRecord
belongs_to :data_import
has_many :import_errors, class_name: 'DataImportError', dependent: :destroy_async
validates :source_provider, :source_object_type, :source_object_id, presence: true
validates :source_object_id, uniqueness: { scope: [:data_import_id, :source_object_type] }
enum status: { pending: 0, processing: 1, imported: 2, skipped: 3, failed: 4 }
end
+33
View File
@@ -0,0 +1,33 @@
# == Schema Information
#
# Table name: data_import_mappings
#
# id :bigint not null, primary key
# chatwoot_record_type :string not null
# metadata :jsonb not null
# source_object_type :string not null
# source_provider :string not null
# created_at :datetime not null
# updated_at :datetime not null
# account_id :integer not null
# chatwoot_record_id :bigint not null
# data_import_id :bigint not null
# source_object_id :string not null
#
# Indexes
#
# idx_data_import_mappings_on_account_and_source (account_id,source_provider,source_object_type,source_object_id) UNIQUE
# idx_data_import_mappings_on_record (chatwoot_record_type,chatwoot_record_id)
# index_data_import_mappings_on_data_import_id (data_import_id)
#
class DataImportMapping < ApplicationRecord
belongs_to :data_import
belongs_to :account
validates :source_provider, :source_object_type, :source_object_id, :chatwoot_record_type, :chatwoot_record_id, presence: true
validates :source_object_id, uniqueness: { scope: [:account_id, :source_provider, :source_object_type] }
def chatwoot_record
chatwoot_record_type.constantize.find_by(id: chatwoot_record_id)
end
end
+41
View File
@@ -0,0 +1,41 @@
class DataImportPolicy < ApplicationPolicy
def index?
@account_user.administrator?
end
def show?
@account_user.administrator? && record.account_id == account.id
end
def create?
@account_user.administrator?
end
def validate_source?
create?
end
def start?
show?
end
def abandon?
show?
end
def skip_logs?
show?
end
def error_logs?
show?
end
class Scope < Scope
def resolve
return scope.where(account_id: account.id) if account_user.administrator?
scope.none
end
end
end
@@ -0,0 +1,84 @@
class DataImports::Intercom::ActivityContentBuilder
EVENT_KEYS = {
'assignment' => :assignment,
'assign_and_reopen' => :assign_and_reopen,
'open' => :open,
'close' => :close,
'snoozed' => :snoozed,
'participant_added' => :participant_added,
'participant_removed' => :participant_removed,
'conversation_attribute_updated_by_admin' => :conversation_attribute_updated,
'conversation_attribute_updated_by_user' => :conversation_attribute_updated,
'conversation_attribute_updated_by_workflow' => :conversation_attribute_updated,
'ticket_attribute_updated_by_admin' => :ticket_attribute_updated,
'ticket_state_updated_by_admin' => :ticket_state_updated,
'custom_action_started' => :custom_action_started,
'custom_action_finished' => :custom_action_finished,
'quick_reply' => :quick_reply
}.freeze
def initialize(part)
@part = part.to_h
end
def perform
append_body(translated_content)
end
private
def translated_content
key = EVENT_KEYS.fetch(event_type, :generic)
key = "#{key}_with_target" if target_aware_event?(key) && target_name.present?
I18n.t(
"data_imports.intercom.activities.#{key}",
actor: actor_name,
target: target_name,
event: event_type.tr('_', ' ')
)
end
def event_type
@part['part_type'].to_s
end
def actor_name
author = @part['author'].to_h
return author['name'] if author['name'].present?
case author['type']
when 'user', 'contact', 'lead'
'Contact'
when 'bot'
'Intercom automation'
else
automation_event? ? 'Intercom automation' : 'Intercom teammate'
end
end
def target_name
assigned_to = @part['assigned_to'].to_h
event_details = @part['event_details'].to_h
participant = event_details['participant'].to_h
assigned_to['name'].presence || participant['name'].presence || event_details['participant_name'].presence || event_details['name'].presence
end
def target_aware_event?(key)
%i[assignment assign_and_reopen participant_added participant_removed].include?(key)
end
def automation_event?
event_type.include?('workflow') || event_type.start_with?('custom_action')
end
def append_body(content)
fragment = Nokogiri::HTML5.fragment(@part['body'].to_s)
fragment.css('script, style').remove
body = fragment.text.squish
return content if body.blank? || content.downcase.include?(body.downcase)
"#{content}: #{body}"
end
end
@@ -0,0 +1,107 @@
class DataImports::Intercom::Client
class Error < StandardError
attr_reader :status, :body
def initialize(message, status: nil, body: nil)
super(message)
@status = status
@body = body
end
end
class AuthenticationError < Error; end
class RateLimitError < Error
attr_reader :retry_after
def initialize(message, retry_after: nil, **)
super(message, **)
@retry_after = retry_after
end
end
BASE_URL = 'https://api.intercom.io'.freeze
API_VERSION = '2.15'.freeze
DEFAULT_PER_PAGE = 50
def initialize(access_token:)
@access_token = access_token
end
def list_contacts(starting_after: nil, per_page: DEFAULT_PER_PAGE)
get('/contacts', query: pagination_query(starting_after, per_page))
end
def list_conversations(starting_after: nil, per_page: DEFAULT_PER_PAGE)
get('/conversations', query: pagination_query(starting_after, per_page))
end
def retrieve_conversation(id)
get("/conversations/#{id}")
end
def retrieve_contact(id)
get("/contacts/#{id}")
end
private
def pagination_query(starting_after, per_page)
{ per_page: per_page, starting_after: starting_after }.compact
end
def get(path, query: {})
response =
begin
HTTParty.get(
"#{BASE_URL}#{path}",
query: query,
headers: headers,
timeout: 30
)
rescue StandardError => e
raise Error.new(
"Intercom API request failed before receiving a response: #{e.message}",
body: { transport_error_class: e.class.name }
)
end
parse_response(response)
end
def headers
{
'Authorization' => "Bearer #{@access_token}",
'Accept' => 'application/json',
'Content-Type' => 'application/json',
'Intercom-Version' => API_VERSION
}
end
def parse_response(response)
body = parsed_body(response)
return body if response.success?
message = error_message(body, response)
case response.code
when 401, 403
raise AuthenticationError.new(message, status: response.code, body: body)
when 429
raise RateLimitError.new(message, status: response.code, body: body, retry_after: response.headers['retry-after'])
else
raise Error.new(message, status: response.code, body: body)
end
end
def parsed_body(response)
response.parsed_response.presence || {}
rescue JSON::ParserError
{}
end
def error_message(body, response)
errors = body.is_a?(Hash) ? body['errors'] : nil
first_error = errors.is_a?(Array) ? errors.first : nil
first_error&.dig('message').presence || "Intercom API request failed with status #{response.code}"
end
end
@@ -0,0 +1,67 @@
class DataImports::Intercom::CreationService
def initialize(account:, initiated_by:, source_params:)
@account = account
@initiated_by = initiated_by
@source_params = source_params.symbolize_keys
@access_token = @source_params[:access_token].to_s.strip
end
def perform
return if active_import?
totals = validate_source
@account.with_lock do
next if active_import?
@account.data_imports.new(attributes(totals)).tap do |data_import|
data_import.assign_active_intercom_import_run_id
data_import.save!
end
end
end
private
def validate_source
raise ArgumentError, 'Unsupported import source.' unless @source_params[:source_provider] == 'intercom'
DataImports::Intercom::CredentialsValidator.new(
access_token: @access_token,
import_types: import_types
).perform
end
def attributes(totals)
{
name: @source_params[:name].presence || 'Intercom import',
data_type: 'intercom',
source_type: 'api',
source_provider: 'intercom',
import_types: import_types,
initiated_by: @initiated_by,
access_token: @access_token,
stats: initial_stats(totals)
}
end
def import_types
return DataImports::Intercom::Importer::DEFAULT_IMPORT_TYPES unless @source_params.key?(:import_types)
Array(@source_params[:import_types]).compact_blank
end
def initial_stats(totals)
{
'contacts' => { 'imported' => 0, 'skipped' => 0 },
'conversations' => { 'imported' => 0, 'skipped' => 0 },
'messages' => { 'imported' => 0, 'skipped' => 0 },
'errors' => { 'count' => 0 }
}.tap do |stats|
totals.each { |type, total| stats[type]['total'] = total unless total.nil? }
end
end
def active_import?
@account.data_imports.active_intercom.exists?
end
end
@@ -0,0 +1,36 @@
class DataImports::Intercom::CredentialsValidator
def initialize(access_token:, import_types:)
@access_token = access_token.to_s.strip
@import_types = Array(import_types).compact_blank
end
def perform
validate_parameters!
{}.tap do |totals|
contacts_response = client.list_contacts(per_page: 1) if @import_types.intersect?(%w[contacts conversations])
totals['contacts'] = total_count(contacts_response) if @import_types.include?('contacts')
totals['conversations'] = total_count(client.list_conversations(per_page: 1)) if @import_types.include?('conversations')
end.compact
end
private
def validate_parameters!
raise ArgumentError, 'Intercom access key is required.' if @access_token.blank?
raise ArgumentError, 'Select at least one data type to import.' if @import_types.blank?
invalid_types = @import_types - DataImport::IMPORT_TYPES
return if invalid_types.blank?
raise ArgumentError, "Unsupported import types: #{invalid_types.join(', ')}"
end
def client
@client ||= DataImports::Intercom::Client.new(access_token: @access_token)
end
def total_count(response)
response['total_count'] if response.key?('total_count')
end
end
@@ -0,0 +1,991 @@
# rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Rails/SkipsModelValidations
class DataImports::Intercom::Importer
PageResult = Struct.new(:next_cursor, keyword_init: true) do
def done?
next_cursor.blank?
end
end
DEFAULT_IMPORT_TYPES = %w[contacts conversations].freeze
PROVIDER = 'intercom'.freeze
ALREADY_IMPORTED_ERROR_CODE = 'DataImports::Intercom::AlreadyImported'.freeze
SKIPPED_MESSAGE_ERROR_CODE = 'DataImports::Intercom::SkippedMessage'.freeze
TRUNCATED_PARTS_ERROR_CODE = 'DataImports::Intercom::TruncatedConversationParts'.freeze
E164_REGEX = /\A\+[1-9]\d{1,14}\z/
INTERCOM_NUMBER_REGEX = /\A[1-9]\d{1,14}\z/
REGULAR_MESSAGE_PART_TYPES = %w[comment note source].freeze
def initialize(data_import:, run_id: nil)
@data_import = data_import
@run_id = run_id
@account = data_import.account
@client = DataImports::Intercom::Client.new(access_token: data_import.access_token)
@placeholder_inboxes = DataImports::Intercom::PlaceholderInboxBuilder.new(account: @account)
@stats = default_stats.deep_merge(data_import.stats || {})
end
def perform
return unless start!
import_contacts if import_type?('contacts')
import_conversations if import_type?('conversations')
finish!
rescue StandardError => e
fail!(e)
raise
end
def start!
return if @data_import.reload.abandoned?
@data_import.update!(status: :processing, started_at: @data_import.started_at || Time.current)
end
def finish!
return if @data_import.reload.abandoned?
has_failures = @data_import.import_errors.non_skip_logs.exists? || @data_import.import_errors.failed.exists?
status = has_failures ? :completed_with_errors : :completed
@data_import.update!(
status: status,
completed_at: Time.current,
stats: @stats,
total_records: total_processed_records,
processed_records: total_successful_records
)
end
def fail!(error)
return if @data_import.reload.abandoned?
record_run_error(error)
@data_import.update!(status: :failed, last_error_at: Time.current)
end
def import_contacts_page(starting_after: cursor_for('contacts'))
response = @client.list_contacts(starting_after: starting_after)
update_stat_total('contacts', response['total_count']) if response['total_count'].present?
Array(response['data'] || response['contacts']).each do |contact|
break if import_stopped?
import_contact(contact)
end
return PageResult.new(next_cursor: nil) if import_stopped?
next_cursor = response.dig('pages', 'next', 'starting_after')
update_cursor('contacts', next_cursor)
PageResult.new(next_cursor: next_cursor)
end
def import_conversations_page(starting_after: cursor_for('conversations'))
response = @client.list_conversations(starting_after: starting_after)
update_stat_total('conversations', response['total_count']) if response['total_count'].present?
Array(response['data'] || response['conversations']).each do |conversation_summary|
break if import_stopped?
import_conversation_from_summary(conversation_summary)
end
return PageResult.new(next_cursor: nil) if import_stopped?
next_cursor = response.dig('pages', 'next', 'starting_after')
update_cursor('conversations', next_cursor)
PageResult.new(next_cursor: next_cursor)
end
def import_contacts?
import_type?('contacts')
end
def import_conversations?
import_type?('conversations')
end
def contacts_completed?
stage_completed?('contacts')
end
def conversations_completed?
stage_completed?('conversations')
end
def cursor_for(key)
@data_import.cursor&.dig(key, 'starting_after')
end
private
def import_contacts
cursor = cursor_for('contacts')
loop do
result = import_contacts_page(starting_after: cursor)
break if result.done?
cursor = result.next_cursor
end
end
def import_conversations
cursor = cursor_for('conversations')
loop do
result = import_conversations_page(starting_after: cursor)
break if result.done?
cursor = result.next_cursor
end
end
def import_conversation_from_summary(conversation_summary)
source_id = source_id_for(conversation_summary)
already_handled = item_handled?('conversation', source_id)
item = import_item('conversation', source_id, conversation_summary)
mapping = find_mapping('conversation', source_id)
conversation = @client.retrieve_conversation(source_id)
return if import_stopped?
update_message_total(item, conversation)
contact = import_contact(primary_conversation_contact(conversation), required_for_conversation: true)
source_type = conversation_source_type(conversation, conversation_summary)
inbox = @placeholder_inboxes.inbox_for(source_type)
contact_inbox = contact_inbox_for(contact, inbox)
mapped_conversation = mapping&.chatwoot_record
if mapped_conversation && mapping.data_import_id != @data_import.id
skip_already_imported_item(item, mapping, already_handled: already_handled)
import_source_message(conversation, mapped_conversation, contact)
import_conversation_parts(conversation, mapped_conversation, contact)
update_conversation_activity(mapped_conversation)
return
end
chatwoot_conversation = mapped_conversation || create_conversation(conversation, contact, contact_inbox, inbox, source_type)
if mapped_conversation
record_mapping('conversation', source_id, chatwoot_conversation, metadata: conversation_metadata(conversation, inbox, source_type))
end
item.update!(status: :imported, chatwoot_record_type: 'Conversation', chatwoot_record_id: chatwoot_conversation.id)
increment_stat('conversations', 'imported') unless already_handled
import_source_message(conversation, chatwoot_conversation, contact)
import_conversation_parts(conversation, chatwoot_conversation, contact)
update_conversation_activity(chatwoot_conversation)
rescue StandardError => e
raise if e.is_a?(DataImports::Intercom::Client::Error)
fail_item(item, e)
ensure
persist_stats
end
def import_stopped?
return true if @import_stopped
@data_import.reload
@import_stopped = @data_import.abandoned? || @data_import.completed? || @data_import.completed_with_errors? || stale_import_run?
end
def stale_import_run?
active_run_id = @data_import.active_intercom_import_run_id
@run_id.present? && active_run_id.present? && active_run_id != @run_id
end
def import_contact(contact_payload, required_for_conversation: false)
source_id = source_id_for(contact_payload)
if source_id.present? && (mapping = find_mapping('contact', source_id)) && (mapped_contact = mapping.chatwoot_record)
return reuse_mapped_contact(contact_payload, source_id, mapping, mapped_contact)
end
contact_payload = retrieve_contact_payload(contact_payload)
source_id = source_id_for(contact_payload)
already_handled = item_handled?('contact', source_id)
item = import_item('contact', source_id, contact_payload)
mapping = find_mapping('contact', source_id)
mapped_contact = mapping&.chatwoot_record
if mapped_contact && mapping.data_import_id != @data_import.id
skip_already_imported_item(item, mapping, already_handled: already_handled)
return mapped_contact
end
contact = Contact.transaction do
imported_contact = mapped_contact || find_existing_contact(contact_payload) || create_contact(contact_payload)
update_existing_contact(imported_contact, contact_payload)
record_mapping('contact', source_id, imported_contact, metadata: contact_metadata(contact_payload))
item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: imported_contact.id)
imported_contact
end
increment_stat('contacts', 'imported') unless already_handled
contact
rescue StandardError => e
raise if e.is_a?(DataImports::Intercom::Client::Error)
fail_item(item, e)
raise if required_for_conversation
ensure
persist_stats
end
def retrieve_contact_payload(contact_payload)
return contact_payload if contact_payload.blank?
return contact_payload if contact_payload['email'].present? || contact_payload['phone'].present? || contact_payload['name'].present?
return contact_payload if contact_payload['id'].blank?
@client.retrieve_contact(contact_payload['id'])
rescue DataImports::Intercom::Client::Error => e
raise unless e.status == 404
contact_payload
end
def create_contact(contact_payload)
attrs = contact_attributes(contact_payload).merge(created_at: timestamp_for(contact_payload['created_at']), updated_at: Time.current)
result = Contact.insert_all!([attrs], returning: %w[id])
Contact.find(result.rows.first.first)
rescue ActiveRecord::RecordNotUnique
find_existing_contact(contact_payload)
end
def reuse_mapped_contact(contact_payload, source_id, mapping, mapped_contact)
if mapping.data_import_id == @data_import.id
reconcile_current_run_contact(contact_payload, source_id, mapped_contact)
return mapped_contact
end
already_handled = item_handled?('contact', source_id)
item = import_item('contact', source_id, contact_payload)
skip_already_imported_item(item, mapping, already_handled: already_handled)
mapped_contact
end
def update_existing_contact(contact, contact_payload)
attrs = contact_attributes(contact_payload)
updates = {}
updates[:name] = attrs[:name] if contact.name.blank? && attrs[:name].present?
updates[:email] = attrs[:email] if contact_email_available?(contact, attrs[:email])
updates[:phone_number] = attrs[:phone_number] if contact_phone_number_available?(contact, attrs[:phone_number])
updates[:identifier] = attrs[:identifier] if contact.identifier.blank? && attrs[:identifier].present?
updates[:last_activity_at] = attrs[:last_activity_at] if contact.last_activity_at.blank? && attrs[:last_activity_at].present?
updates[:additional_attributes] = contact.additional_attributes.to_h.deep_merge(attrs[:additional_attributes])
updates[:custom_attributes] = contact.custom_attributes.to_h.deep_merge(attrs[:custom_attributes])
if contact.visitor? && attrs[:contact_type].present? && contact_resolved_after_update?(contact, updates)
updates[:contact_type] = attrs[:contact_type]
end
updates[:updated_at] = Time.current
contact.update_columns(updates) if updates.present?
contact.reload
end
def contact_email_available?(contact, email)
return false if contact.email.present? || email.blank?
@account.contacts.where.not(id: contact.id).where('LOWER(email) = ?', email.downcase).empty?
end
def contact_phone_number_available?(contact, phone_number)
return false if contact.phone_number.present? || phone_number.blank?
@account.contacts.where.not(id: contact.id).where(phone_number: phone_number).empty?
end
def contact_resolved_after_update?(contact, updates)
contact.email.present? || contact.phone_number.present? || updates[:email].present? || updates[:phone_number].present?
end
def find_existing_contact(contact_payload)
identifier = normalized_identifier(contact_payload)
email = normalized_email(contact_payload)
phone_number = normalized_phone(contact_payload)
if identifier.present?
contact = @account.contacts.find_by(identifier: identifier)
return contact if contact.present?
end
if email.present?
contact = @account.contacts.from_email(email)
return contact if contact.present?
end
return @account.contacts.find_by(phone_number: phone_number) if phone_number.present?
nil
end
def contact_attributes(contact_payload)
attrs = {
account_id: @account.id,
name: contact_payload['name'].presence || contact_payload['email'].presence || '',
email: normalized_email(contact_payload),
phone_number: normalized_phone(contact_payload),
identifier: normalized_identifier(contact_payload),
last_activity_at: contact_activity_at(contact_payload),
additional_attributes: {
source: {
provider: PROVIDER,
contact_id: contact_payload['id'],
external_id: contact_payload['external_id'],
raw_phone: contact_payload['phone']
}.compact
},
custom_attributes: {
intercom_contact_id: contact_payload['id'],
intercom_external_id: contact_payload['external_id']
}.compact
}
attrs[:contact_type] = Contact.contact_types[:lead] if attrs[:email].present? || attrs[:phone_number].present?
attrs
end
def create_conversation(conversation, contact, contact_inbox, inbox, source_type)
source_id = source_id_for(conversation)
metadata = conversation_metadata(conversation, inbox, source_type)
if (existing_conversation = @account.conversations.find_by(identifier: conversation_identifier(conversation)))
record_mapping('conversation', source_id, existing_conversation, metadata: metadata)
return existing_conversation
end
attrs = {
account_id: @account.id,
inbox_id: inbox.id,
status: Conversation.statuses['resolved'],
contact_id: contact.id,
contact_inbox_id: contact_inbox.id,
identifier: conversation_identifier(conversation),
additional_attributes: metadata,
custom_attributes: { intercom_conversation_id: source_id },
created_at: timestamp_for(conversation['created_at']),
updated_at: timestamp_for(conversation['updated_at']),
last_activity_at: timestamp_for(conversation['updated_at'])
}
Conversation.transaction do
result = Conversation.insert_all!([attrs], returning: %w[id])
chatwoot_conversation = Conversation.find(result.rows.first.first)
record_mapping('conversation', source_id, chatwoot_conversation, metadata: metadata)
chatwoot_conversation
end
rescue ActiveRecord::RecordNotUnique
@account.conversations.find_by!(identifier: conversation_identifier(conversation)).tap do |chatwoot_conversation|
record_mapping('conversation', source_id, chatwoot_conversation, metadata: metadata)
end
end
def import_source_message(conversation, chatwoot_conversation, contact)
source = conversation['source'].to_h
return unless source_message_importable?(source)
message_source_id = "conversation:#{source_id_for(conversation)}:source:#{source['id'].presence || 'initial'}"
source_part = source.merge('part_type' => 'source', 'created_at' => conversation['created_at'])
if (mapping = find_mapping('message', message_source_id)) && message_mapping_handled?(mapping, source_part)
if mapping.data_import_id == @data_import.id
reconcile_current_run_message_mapping(chatwoot_conversation, mapping, source_part)
return
end
skip_existing_message_mapping(chatwoot_conversation, mapping, source_part)
return
end
create_message(chatwoot_conversation, contact, source_part, message_source_id)
rescue StandardError => e
fail_message(chatwoot_conversation, message_source_id, source_part, e)
end
def import_conversation_parts(conversation, chatwoot_conversation, contact)
parts_payload = conversation['conversation_parts'].to_h
parts = Array(parts_payload['conversation_parts'])
record_truncated_conversation_parts(conversation, parts.size)
parts.each do |part|
message_source_id = "conversation:#{source_id_for(conversation)}:part:#{part['id']}"
if (mapping = find_mapping('message', message_source_id)) && message_mapping_handled?(mapping, part)
if mapping.data_import_id == @data_import.id
reconcile_current_run_message_mapping(chatwoot_conversation, mapping, part)
next
end
skip_existing_message_mapping(chatwoot_conversation, mapping, part)
next
end
create_message(chatwoot_conversation, contact, part, message_source_id)
rescue StandardError => e
fail_message(chatwoot_conversation, message_source_id, part, e)
end
end
def create_message(conversation, contact, part, message_source_id)
content = content_for(part)
return record_skipped_message(conversation, message_source_id, part) if content.blank?
attrs = message_attributes(conversation, contact, part, message_source_id, content)
message = nil
Message.transaction do
message = conversation.messages.find_by(source_id: attrs[:source_id])
unless message
result = Message.insert_all!([attrs], returning: %w[id])
message = Message.find(result.rows.first.first)
end
record_mapping('message', message_source_id, message, metadata: message_metadata(part))
end
increment_stat('messages', 'imported')
reindex_message_for_search(message)
message
end
def reindex_message_for_search(message)
return unless message.should_index?
message.__send__(:reindex_for_search)
rescue StandardError => e
Rails.logger.warn("Intercom import message reindex failed for message #{message.id}: #{e.class} - #{e.message}")
end
def record_skipped_message(conversation, message_source_id, part)
mapping = find_mapping('message', message_source_id)
if mapping
already_recorded = skip_log_recorded?('message', message_source_id, SKIPPED_MESSAGE_ERROR_CODE)
record_skipped_message_log(conversation, message_source_id, part)
increment_stat('messages', 'skipped') unless already_recorded
return mapping.chatwoot_record
end
DataImportMapping.create!(
account: @account,
data_import: @data_import,
source_provider: PROVIDER,
source_object_type: 'message',
source_object_id: message_source_id,
chatwoot_record_type: 'Conversation',
chatwoot_record_id: conversation.id,
metadata: message_metadata(part).merge(skipped: true, reason: 'blank_or_unsupported_intercom_part')
)
record_skipped_message_log(conversation, message_source_id, part)
increment_stat('messages', 'skipped')
end
def message_attributes(conversation, contact, part, message_source_id, content)
message_type = message_type_for(part)
created_at = timestamp_for(part['created_at'])
{
account_id: @account.id,
inbox_id: conversation.inbox_id,
conversation_id: conversation.id,
message_type: Message.message_types[message_type],
content_type: Message.content_types['text'],
content: content,
processed_message_content: content,
private: message_type != 'activity' && part['part_type'] == 'note',
status: Message.statuses['sent'],
sender_type: message_type == 'incoming' ? 'Contact' : nil,
sender_id: message_type == 'incoming' ? contact.id : nil,
source_id: "intercom:#{message_source_id}",
external_source_ids: { intercom: message_source_id },
content_attributes: {},
additional_attributes: message_metadata(part),
created_at: created_at,
updated_at: part['updated_at'].present? ? timestamp_for(part['updated_at']) : created_at
}
end
def message_type_for(part)
return 'activity' if activity_part?(part)
author_type = part.dig('author', 'type').to_s
return 'incoming' if %w[user contact lead].include?(author_type)
'outgoing'
end
def content_for(part)
return DataImports::Intercom::ActivityContentBuilder.new(part).perform if activity_part?(part)
message_content(part)
end
def activity_part?(part)
part_type = part['part_type'].to_s
part_type.present? && REGULAR_MESSAGE_PART_TYPES.exclude?(part_type)
end
def message_content(part)
body = sanitized_text(part['body'])
subject = sanitized_text(part['subject'])
attachments = Array(part['attachments'])
content = [subject, body].reject(&:blank?).join("\n\n")
return content if attachments.blank?
[content.presence, "[Intercom attachment skipped: #{attachments.size}]"].compact.join("\n\n")
end
def sanitized_text(value)
Rails::HTML5::FullSanitizer.new.sanitize(value.to_s).squish
end
def update_conversation_activity(conversation)
latest_message = conversation.messages.reorder(created_at: :desc).first
return if latest_message.blank?
conversation.update_columns(last_activity_at: latest_message.created_at, updated_at: Time.current)
end
def contact_inbox_for(contact, inbox)
ContactInbox.find_or_create_by!(contact: contact, inbox: inbox) do |contact_inbox|
contact_inbox.source_id = "intercom:#{contact.id}"
end
end
def primary_conversation_contact(conversation)
contacts = conversation.dig('contacts', 'contacts') || []
contacts.first || conversation.dig('source', 'author') || {}
end
def conversation_source_type(conversation, conversation_summary)
conversation.dig('source', 'type').presence ||
conversation.dig('first_contact_reply', 'type').presence ||
conversation_summary.dig('source', 'type').presence ||
conversation_summary.dig('first_contact_reply', 'type').presence
end
def normalized_identifier(contact_payload)
contact_payload['external_id'].presence
end
def normalized_email(contact_payload)
email = contact_payload['email'].to_s.strip.downcase
email.match?(Devise.email_regexp) ? email : nil
end
def normalized_phone(contact_payload)
phone = contact_payload['phone'].to_s.strip
phone = "+#{phone}" if phone.match?(INTERCOM_NUMBER_REGEX)
phone.match?(E164_REGEX) ? phone : nil
end
def contact_activity_at(contact_payload)
return timestamp_for(contact_payload['last_seen_at']) if contact_payload['last_seen_at'].present?
return timestamp_for(contact_payload['last_replied_at']) if contact_payload['last_replied_at'].present?
nil
end
def source_id_for(payload)
payload['id'].presence || payload['external_id'].presence || payload['email'].presence
end
def conversation_identifier(conversation)
"intercom:#{source_id_for(conversation)}"
end
def import_item(object_type, source_id, metadata)
@data_import.items.find_or_initialize_by(
source_provider: PROVIDER,
source_object_type: object_type,
source_object_id: source_id
).tap do |item|
item.status = :processing
item.attempt_count += 1
item.metadata = item.metadata.to_h.merge(metadata.to_h)
item.save!
end
end
def item_handled?(object_type, source_id)
@data_import.items.where(status: [:imported, :skipped]).exists?(
source_provider: PROVIDER,
source_object_type: object_type,
source_object_id: source_id
)
end
def find_mapping(object_type, source_id)
DataImportMapping.find_by(
account: @account,
source_provider: PROVIDER,
source_object_type: object_type,
source_object_id: source_id
)
end
def record_mapping(object_type, source_id, record, metadata: {})
DataImportMapping.find_or_initialize_by(
account: @account,
source_provider: PROVIDER,
source_object_type: object_type,
source_object_id: source_id
).tap do |mapping|
mapping.data_import = @data_import
mapping.chatwoot_record_type = record.class.name
mapping.chatwoot_record_id = record.id
mapping.metadata = metadata
mapping.save!
end
end
def reconcile_current_run_contact(contact_payload, source_id, mapped_contact)
item = @data_import.items.find_by(
source_provider: PROVIDER,
source_object_type: 'contact',
source_object_id: source_id
)
item = import_item('contact', source_id, contact_payload) unless item&.imported?
item.update!(status: :imported, chatwoot_record_type: 'Contact', chatwoot_record_id: mapped_contact.id)
reconcile_item_stats('contact')
end
def reconcile_item_stats(source_object_type)
items = @data_import.items.where(source_provider: PROVIDER, source_object_type: source_object_type)
group = stat_group_for(source_object_type)
@stats[group]['imported'] = items.imported.count
@stats[group]['skipped'] = items.skipped.count
persist_stats
end
def reconcile_current_run_message_mapping(conversation, mapping, part)
record_skipped_message_log(conversation, mapping.source_object_id, part) if mapping.metadata['skipped']
mappings = @data_import.mappings.where(source_provider: PROVIDER, source_object_type: 'message')
skipped_mappings = mappings.where("metadata ->> 'skipped' = ?", 'true').count
message_logs = @data_import.import_errors.where(source_object_type: 'message')
@stats['messages']['imported'] = mappings.count - skipped_mappings
@stats['messages']['skipped'] = message_logs.where("details ->> 'kind' = ?", 'skipped').count
persist_stats
end
def skip_already_imported_item(item, mapping, already_handled:)
item.update!(
status: :skipped,
chatwoot_record_type: mapping.chatwoot_record_type,
chatwoot_record_id: mapping.chatwoot_record_id,
last_error_code: ALREADY_IMPORTED_ERROR_CODE,
last_error_message: 'Already imported in a previous import.'
)
record_already_imported_log(
data_import_item: item,
source_object_type: item.source_object_type,
source_object_id: item.source_object_id,
mapping: mapping
)
increment_stat(stat_group_for(item.source_object_type), 'skipped') unless already_handled
end
def skip_existing_message_mapping(conversation, mapping, part)
if mapping.metadata['skipped']
already_recorded = skip_log_recorded?('message', mapping.source_object_id, SKIPPED_MESSAGE_ERROR_CODE)
record_skipped_message_log(conversation, mapping.source_object_id, part)
else
already_recorded = skip_log_recorded?('message', mapping.source_object_id, ALREADY_IMPORTED_ERROR_CODE)
record_already_imported_log(source_object_type: 'message', source_object_id: mapping.source_object_id, mapping: mapping)
end
increment_stat('messages', 'skipped') unless already_recorded
end
def message_mapping_handled?(mapping, part)
return false if mapping.metadata['skipped'] && activity_part?(part)
mapping.metadata['skipped'] || mapping.chatwoot_record.present?
end
def fail_item(item, error)
increment_stat('errors', 'count')
item&.update!(status: :failed, last_error_code: error.class.name, last_error_message: error.message)
record_skip_log(
data_import_item: item,
source_object_type: item&.source_object_type,
source_object_id: item&.source_object_id,
error_code: error.class.name,
message: error.message,
details: {
kind: 'failed',
source_provider: PROVIDER,
error_class: error.class.name
}
)
end
def fail_message(conversation, message_source_id, part, error)
increment_stat('errors', 'count')
record_skip_log(
source_object_type: 'message',
source_object_id: message_source_id,
error_code: error.class.name,
message: error.message,
details: message_metadata(part).merge(
kind: 'failed',
source_provider: PROVIDER,
error_class: error.class.name,
conversation_id: conversation.identifier
)
)
end
def record_skipped_message_log(conversation, message_source_id, part)
record_skip_log(
source_object_type: 'message',
source_object_id: message_source_id,
error_code: SKIPPED_MESSAGE_ERROR_CODE,
message: skipped_message_log_message(part),
details: skipped_message_details(conversation, part)
)
end
def record_already_imported_log(source_object_type:, source_object_id:, mapping:, data_import_item: nil)
record_skip_log(
data_import_item: data_import_item,
source_object_type: source_object_type,
source_object_id: source_object_id,
error_code: ALREADY_IMPORTED_ERROR_CODE,
message: 'Already imported in a previous import.',
details: {
kind: 'skipped',
reason: 'already_imported',
source_provider: PROVIDER,
previous_data_import_id: mapping.data_import_id,
chatwoot_record_type: mapping.chatwoot_record_type,
chatwoot_record_id: mapping.chatwoot_record_id
}
)
end
def record_truncated_conversation_parts(conversation, imported_parts_count)
total_parts_count = total_conversation_parts_count(conversation)
return if total_parts_count <= imported_parts_count
source_id = source_id_for(conversation)
already_recorded = @data_import.import_errors.exists?(
source_object_type: 'conversation',
source_object_id: source_id,
error_code: TRUNCATED_PARTS_ERROR_CODE
)
record_import_error(
source_object_type: 'conversation',
source_object_id: source_id,
error_code: TRUNCATED_PARTS_ERROR_CODE,
message: "Intercom returned #{imported_parts_count} of #{total_parts_count} conversation parts.",
details: {
kind: 'incomplete',
source_provider: PROVIDER,
imported_parts_count: imported_parts_count,
total_parts_count: total_parts_count
}
)
increment_stat('errors', 'count') unless already_recorded
end
def total_conversation_parts_count(conversation)
conversation_parts_total_count = conversation.dig('conversation_parts', 'total_count')
return conversation_parts_total_count.to_i if conversation_parts_total_count.present?
[
conversation.dig('statistics', 'count_conversation_parts'),
conversation.dig('statistics', 'count_conversations_parts')
].compact.map(&:to_i).max || 0
end
def source_message_importable?(source)
source['body'].present? || source['subject'].present? || source['attachments'].present?
end
def skipped_message_log_message(part)
"Skipped Intercom #{intercom_event_name(part)} event#{intercom_part_id_suffix(part)}: #{skipped_message_reason_details(part)}."
end
def skipped_message_details(conversation, part)
author = part['author'].to_h
message_metadata(part).merge(
{
kind: 'skipped',
reason: 'blank_or_unsupported_intercom_part',
reason_details: skipped_message_reason_details(part),
event_name: intercom_event_name(part),
event_type: part['part_type'],
author_type: author['type'],
author_name: author['name'],
conversation_id: conversation.identifier
}.compact
)
end
def skipped_message_reason_details(part)
return 'message body did not contain readable text after HTML sanitization' if part['body'].present?
return 'attachments are present but no importable message text was found' if Array(part['attachments']).present?
'no message body or attachments to import'
end
def intercom_event_name(part)
part['part_type'].to_s.tr('_', ' ').presence || 'message part'
end
def intercom_part_id_suffix(part)
part['id'].present? ? " #{part['id']}" : ''
end
def skip_log_recorded?(source_object_type, source_object_id, error_code)
@data_import.import_errors.skip_logs.exists?(
source_object_type: source_object_type,
source_object_id: source_object_id,
error_code: error_code
)
end
def record_run_error(error)
@data_import.import_errors.create!(
error_code: error.class.name,
message: error.message,
details: {
kind: 'run_error',
source_provider: PROVIDER,
error_class: error.class.name
}
)
end
def record_skip_log(attributes)
record_import_error(attributes)
end
def record_import_error(attributes)
@data_import.import_errors.find_or_initialize_by(
data_import_item: attributes[:data_import_item],
source_object_type: attributes[:source_object_type],
source_object_id: attributes[:source_object_id],
error_code: attributes[:error_code]
).tap do |import_error|
import_error.message = attributes[:message]
import_error.details = attributes[:details]
import_error.save!
end
end
def conversation_metadata(conversation, inbox, source_type)
{
source: {
provider: PROVIDER,
conversation_id: source_id_for(conversation),
source_type: source_type,
delivered_as: conversation.dig('source', 'delivered_as'),
source_url: conversation.dig('source', 'url'),
admin_assignee_id: conversation['admin_assignee_id'],
team_assignee_id: conversation['team_assignee_id'],
state: conversation['state'],
open: conversation['open'],
routing_method: 'source_bucket_api_inbox',
routed_inbox_id: inbox.id,
import_id: @data_import.id
}.compact
}
end
def contact_metadata(contact_payload)
{
source: {
provider: PROVIDER,
contact_id: contact_payload['id'],
external_id: contact_payload['external_id']
}.compact
}
end
def message_metadata(part)
{
source: {
provider: PROVIDER,
part_id: part['id'],
part_type: part['part_type'],
author: part['author'],
assigned_to: part['assigned_to'],
state: part['state'],
tags: part['tags'],
event_details: part['event_details'],
app_package_code: part['app_package_code'],
metadata: part['metadata'],
attachments: part['attachments'],
redacted: part['redacted']
}.compact
}
end
def timestamp_for(value)
return Time.current if value.blank?
Time.zone.at(value.to_i)
end
def update_cursor(key, cursor)
@data_import.cursor = @data_import.cursor.to_h.merge(
key => { starting_after: cursor, completed: cursor.blank?, updated_at: Time.current.iso8601 }
)
@data_import.save!
end
def stage_completed?(key)
@data_import.cursor&.dig(key, 'completed') == true
end
def import_type?(type)
import_types.include?(type)
end
def import_types
@import_types ||= (@data_import.import_types.presence || DEFAULT_IMPORT_TYPES)
end
def increment_stat(group, key)
@stats[group] ||= {}
@stats[group][key] = @stats[group][key].to_i + 1
end
def update_stat_total(group, total)
@stats[group] ||= {}
@stats[group]['total'] = total.to_i
persist_stats
end
def update_message_total(item, conversation)
parts = conversation['conversation_parts'].to_h
conversation_parts_total = if parts.key?('total_count')
parts['total_count'].to_i
else
Array(parts['conversation_parts']).size
end
contribution = conversation_parts_total
contribution += 1 if source_message_importable?(conversation['source'].to_h)
previous_contribution = item.metadata.to_h['message_total_contribution'].to_i
@stats['messages']['total'] = @stats['messages']['total'].to_i + contribution - previous_contribution
item.update!(metadata: item.metadata.to_h.merge('message_total_contribution' => contribution))
persist_stats
end
def stat_group_for(source_object_type)
"#{source_object_type}s"
end
def persist_stats
@data_import.update_columns(stats: @stats, updated_at: Time.current)
end
def default_stats
{
'contacts' => { 'imported' => 0, 'skipped' => 0 },
'conversations' => { 'imported' => 0, 'skipped' => 0 },
'messages' => { 'imported' => 0, 'skipped' => 0 },
'errors' => { 'count' => 0 }
}
end
def total_processed_records
total_successful_records +
@stats.fetch('contacts', {}).fetch('skipped', 0).to_i +
@stats.fetch('conversations', {}).fetch('skipped', 0).to_i +
@stats.fetch('messages', {}).fetch('skipped', 0).to_i +
@stats.fetch('errors', {}).fetch('count', 0).to_i
end
def total_successful_records
@stats.fetch('contacts', {}).fetch('imported', 0).to_i +
@stats.fetch('conversations', {}).fetch('imported', 0).to_i +
@stats.fetch('messages', {}).fetch('imported', 0).to_i
end
end
# rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Rails/SkipsModelValidations
@@ -0,0 +1,41 @@
class DataImports::Intercom::PlaceholderInboxBuilder
AGENT_REPLY_TIME_WINDOW_HOURS = 1
def initialize(account:)
@account = account
end
def inbox_for(source_type)
bucket = DataImports::Intercom::SourceBucket.for(source_type)
placeholder_inboxes[bucket[:key]] ||= create_placeholder_inbox(bucket)
end
private
def placeholder_inboxes
@placeholder_inboxes ||= @account.inboxes.includes(:channel).where(channel_type: 'Channel::Api').each_with_object({}) do |inbox, inboxes|
attrs = inbox.channel.additional_attributes || {}
next unless attrs['source_provider'] == 'intercom' && attrs['import_placeholder'] == true
inboxes[attrs['source_bucket']] = inbox
end
end
def create_placeholder_inbox(bucket)
channel = @account.api_channels.create!(
additional_attributes: {
source_provider: 'intercom',
source_bucket: bucket[:key],
import_placeholder: true,
agent_reply_time_window: AGENT_REPLY_TIME_WINDOW_HOURS
}
)
@account.inboxes.create!(
name: "Intercom Import - #{bucket[:name]}",
channel: channel,
enable_auto_assignment: false,
allow_messages_after_resolved: false
)
end
end
@@ -0,0 +1,55 @@
class DataImports::Intercom::RestartService
attr_reader :data_import
def initialize(account:, data_import:)
@account = account
@data_import = data_import
end
def perform
@account.with_lock do
@data_import.reload
next :render_show unless @data_import.restartable?
if (active_import = find_active_import)
@data_import = active_import
next :render_show
end
next :access_token_missing if @data_import.access_token.blank?
@data_import.assign_active_intercom_import_run_id
retained_skip_logs = @data_import.import_errors.where("details ->> 'kind' = ?", 'skipped')
@data_import.import_errors.where.not(id: retained_skip_logs.select(:id)).delete_all
@data_import.update!(restart_attributes(retained_skip_logs))
:enqueue
end
end
private
def find_active_import
@account.data_imports.active_intercom.first
end
def restart_attributes(retained_skip_logs)
{
status: :pending,
abandoned_at: nil,
completed_at: nil,
last_error_at: nil,
started_at: nil,
stats: restart_stats(retained_skip_logs)
}
end
def restart_stats(retained_skip_logs)
@data_import.stats.to_h.deep_dup.tap do |stats|
%w[contact conversation message].each do |object_type|
stats["#{object_type}s"] ||= {}
stats["#{object_type}s"]['skipped'] = retained_skip_logs.where(source_object_type: object_type).count
end
stats['errors'] = { 'count' => 0 }
end
end
end
@@ -0,0 +1,23 @@
class DataImports::Intercom::SourceBucket
BUCKETS = {
'email' => { key: 'email', name: 'Email' },
'instagram' => { key: 'instagram', name: 'Instagram' },
'facebook' => { key: 'facebook', name: 'Facebook' },
'sms' => { key: 'sms', name: 'SMS' },
'twitter' => { key: 'twitter', name: 'Twitter' },
'whatsapp' => { key: 'whatsapp', name: 'WhatsApp' },
'phone' => { key: 'phone', name: 'Phone' },
'phone_call' => { key: 'phone', name: 'Phone' },
'phone_switch' => { key: 'phone', name: 'Phone' },
'inapp' => { key: 'messenger', name: 'Messenger' },
'messenger' => { key: 'messenger', name: 'Messenger' },
'conversation' => { key: 'messenger', name: 'Messenger' },
'push' => { key: 'messenger', name: 'Messenger' }
}.freeze
DEFAULT_BUCKET = { key: 'unknown', name: 'Unknown' }.freeze
def self.for(source_type)
BUCKETS[source_type.to_s.downcase] || DEFAULT_BUCKET
end
end
@@ -0,0 +1,24 @@
json.id data_import.id
json.name data_import.name
json.data_type data_import.data_type
json.source_type data_import.source_type
json.source_provider data_import.source_provider
json.import_types data_import.import_types
json.status data_import.status
json.total_records data_import.total_records
json.processed_records data_import.processed_records
json.stats data_import.stats
json.cursor data_import.cursor
json.created_at data_import.created_at
json.updated_at data_import.updated_at
json.started_at data_import.started_at
json.completed_at data_import.completed_at
json.abandoned_at data_import.abandoned_at
json.initiated_by data_import.initiated_by&.slice(:id, :name, :email)
if @import_errors_counts
json.import_errors_count @import_errors_counts.fetch(data_import.id, 0)
json.skip_logs_count (@skip_logs_counts || {}).fetch(data_import.id, 0)
else
json.import_errors_count data_import.import_errors.non_skip_logs.count
json.skip_logs_count data_import.import_errors.skip_logs.count
end
@@ -0,0 +1,5 @@
json.payload do
json.array! @data_imports do |data_import|
json.partial! 'api/v1/accounts/data_imports/data_import', formats: [:json], data_import: data_import
end
end
@@ -0,0 +1,31 @@
json.partial! 'api/v1/accounts/data_imports/data_import', formats: [:json], data_import: @data_import
json.import_errors do
json.array! @import_errors_finder.import_errors do |import_error|
json.id import_error.id
json.error_code import_error.error_code
json.message import_error.message
json.source_object_type import_error.source_object_type
json.source_object_id import_error.source_object_id
json.details import_error.details
json.created_at import_error.created_at
end
end
json.skip_logs do
json.array! @skip_logs_finder.skip_logs do |skip_log|
json.id skip_log.id
json.kind skip_log.details['kind']
json.error_code skip_log.error_code
json.message skip_log.message
json.source_object_type skip_log.source_object_type
json.source_object_id skip_log.source_object_id
json.details skip_log.details
json.created_at skip_log.created_at
end
end
json.skip_logs_filters do
json.selected_source_object_type @skip_logs_finder.selected_source_object_type
json.counts_by_type @skip_logs_finder.counts_by_type
end
+4
View File
@@ -253,6 +253,10 @@
display_name: WhatsApp Manual Transfer
enabled: false
column: feature_flags_ext_1
- name: data_import
display_name: Data Import
enabled: false
column: feature_flags_ext_1
- name: api_and_webhooks
display_name: API and Webhooks
enabled: true
+22
View File
@@ -47,6 +47,28 @@ en:
saml_not_available: SAML authentication is not available in this installation.
inbox_deletetion_response: Your inbox deletion request will be processed in some time.
data_imports:
intercom:
activities:
assignment: '%{actor} assigned the conversation'
assignment_with_target: '%{actor} assigned the conversation to %{target}'
assign_and_reopen: '%{actor} assigned and reopened the conversation'
assign_and_reopen_with_target: '%{actor} assigned the conversation to %{target} and reopened it'
open: '%{actor} opened the conversation'
close: '%{actor} closed the conversation'
snoozed: '%{actor} snoozed the conversation'
participant_added: '%{actor} added a participant'
participant_added_with_target: '%{actor} added %{target} as a participant'
participant_removed: '%{actor} removed a participant'
participant_removed_with_target: '%{actor} removed %{target} as a participant'
conversation_attribute_updated: '%{actor} updated conversation attributes'
ticket_attribute_updated: '%{actor} updated ticket attributes'
ticket_state_updated: '%{actor} updated the ticket state'
custom_action_started: '%{actor} started a custom action'
custom_action_finished: '%{actor} finished a custom action'
quick_reply: '%{actor} used a quick reply'
generic: '%{actor} recorded %{event}'
profile_settings:
sessions:
cannot_revoke_current: You cannot revoke the current session.
+11
View File
@@ -222,6 +222,17 @@ Rails.application.routes.draw do
post :call, on: :member, to: 'calls#create' if ChatwootApp.enterprise?
end
end
resources :data_imports, only: [:index, :show, :create] do
collection do
post :validate_source
end
member do
post :start
post :abandon
get :error_logs
get :skip_logs
end
end
resources :csat_survey_responses, only: [:index] do
collection do
get :metrics
@@ -0,0 +1,31 @@
class ExpandDataImportsForIntercomImports < ActiveRecord::Migration[7.1]
def change
add_data_import_columns
add_data_import_indexes
end
private
def add_data_import_columns
change_table :data_imports, bulk: true do |t|
t.string :name
t.string :source_type
t.string :source_provider
t.jsonb :import_types, default: [], null: false
t.integer :initiated_by_id
t.text :access_token
t.jsonb :source_metadata, default: {}, null: false
t.jsonb :stats, default: {}, null: false
t.jsonb :cursor, default: {}, null: false
t.datetime :started_at
t.datetime :completed_at
t.datetime :abandoned_at
t.datetime :last_error_at
end
end
def add_data_import_indexes
add_index :data_imports, :initiated_by_id
add_index :data_imports, :source_provider
end
end
@@ -0,0 +1,35 @@
class CreateDataImportItems < ActiveRecord::Migration[7.1]
def change
create_data_import_items
add_data_import_item_indexes
end
private
def create_data_import_items
create_table :data_import_items do |t|
t.references :data_import, null: false, index: true
t.string :source_provider, null: false
t.string :source_object_type, null: false
t.string :source_object_id, null: false
t.integer :status, default: 0, null: false
t.string :chatwoot_record_type
t.bigint :chatwoot_record_id
t.integer :attempt_count, default: 0, null: false
t.string :last_error_code
t.text :last_error_message
t.jsonb :metadata, default: {}, null: false
t.timestamps
end
end
def add_data_import_item_indexes
add_index :data_import_items,
[:data_import_id, :source_object_type, :source_object_id],
unique: true,
name: 'idx_data_import_items_on_import_and_source'
add_index :data_import_items, [:chatwoot_record_type, :chatwoot_record_id], name: 'idx_data_import_items_on_record'
add_index :data_import_items, [:source_provider, :source_object_type, :source_object_id], name: 'idx_data_import_items_on_source'
end
end
@@ -0,0 +1,22 @@
class CreateDataImportMappings < ActiveRecord::Migration[7.1]
def change
create_table :data_import_mappings do |t|
t.integer :account_id, null: false
t.references :data_import, null: false, index: true
t.string :source_provider, null: false
t.string :source_object_type, null: false
t.string :source_object_id, null: false
t.string :chatwoot_record_type, null: false
t.bigint :chatwoot_record_id, null: false
t.jsonb :metadata, default: {}, null: false
t.timestamps
end
add_index :data_import_mappings,
[:account_id, :source_provider, :source_object_type, :source_object_id],
unique: true,
name: 'idx_data_import_mappings_on_account_and_source'
add_index :data_import_mappings, [:chatwoot_record_type, :chatwoot_record_id], name: 'idx_data_import_mappings_on_record'
end
end
@@ -0,0 +1,17 @@
class CreateDataImportErrors < ActiveRecord::Migration[7.1]
def change
create_table :data_import_errors do |t|
t.references :data_import, null: false, index: true
t.references :data_import_item, null: true, index: true
t.string :source_object_type
t.string :source_object_id
t.string :error_code, null: false
t.text :message
t.jsonb :details, default: {}, null: false
t.timestamps
end
add_index :data_import_errors, [:source_object_type, :source_object_id], name: 'idx_data_import_errors_on_source'
end
end
+66
View File
@@ -841,6 +841,57 @@ ActiveRecord::Schema[7.1].define(version: 2026_07_10_000000) do
t.index ["user_id"], name: "index_dashboard_apps_on_user_id"
end
create_table "data_import_errors", force: :cascade do |t|
t.bigint "data_import_id", null: false
t.bigint "data_import_item_id"
t.string "source_object_type"
t.string "source_object_id"
t.string "error_code", null: false
t.text "message"
t.jsonb "details", default: {}, null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["data_import_id"], name: "index_data_import_errors_on_data_import_id"
t.index ["data_import_item_id"], name: "index_data_import_errors_on_data_import_item_id"
t.index ["source_object_type", "source_object_id"], name: "idx_data_import_errors_on_source"
end
create_table "data_import_items", force: :cascade do |t|
t.bigint "data_import_id", null: false
t.string "source_provider", null: false
t.string "source_object_type", null: false
t.string "source_object_id", null: false
t.integer "status", default: 0, null: false
t.string "chatwoot_record_type"
t.bigint "chatwoot_record_id"
t.integer "attempt_count", default: 0, null: false
t.string "last_error_code"
t.text "last_error_message"
t.jsonb "metadata", default: {}, null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["chatwoot_record_type", "chatwoot_record_id"], name: "idx_data_import_items_on_record"
t.index ["data_import_id", "source_object_type", "source_object_id"], name: "idx_data_import_items_on_import_and_source", unique: true
t.index ["data_import_id"], name: "index_data_import_items_on_data_import_id"
t.index ["source_provider", "source_object_type", "source_object_id"], name: "idx_data_import_items_on_source"
end
create_table "data_import_mappings", force: :cascade do |t|
t.integer "account_id", null: false
t.bigint "data_import_id", null: false
t.string "source_provider", null: false
t.string "source_object_type", null: false
t.string "source_object_id", null: false
t.string "chatwoot_record_type", null: false
t.bigint "chatwoot_record_id", null: false
t.jsonb "metadata", default: {}, null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["account_id", "source_provider", "source_object_type", "source_object_id"], name: "idx_data_import_mappings_on_account_and_source", unique: true
t.index ["chatwoot_record_type", "chatwoot_record_id"], name: "idx_data_import_mappings_on_record"
t.index ["data_import_id"], name: "index_data_import_mappings_on_data_import_id"
end
create_table "data_imports", force: :cascade do |t|
t.bigint "account_id", null: false
t.string "data_type", null: false
@@ -850,7 +901,22 @@ ActiveRecord::Schema[7.1].define(version: 2026_07_10_000000) do
t.integer "processed_records"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "name"
t.string "source_type"
t.string "source_provider"
t.jsonb "import_types", default: [], null: false
t.integer "initiated_by_id"
t.text "access_token"
t.jsonb "source_metadata", default: {}, null: false
t.jsonb "stats", default: {}, null: false
t.jsonb "cursor", default: {}, null: false
t.datetime "started_at"
t.datetime "completed_at"
t.datetime "abandoned_at"
t.datetime "last_error_at"
t.index ["account_id"], name: "index_data_imports_on_account_id"
t.index ["initiated_by_id"], name: "index_data_imports_on_initiated_by_id"
t.index ["source_provider"], name: "index_data_imports_on_source_provider"
end
create_table "email_templates", force: :cascade do |t|
@@ -0,0 +1,199 @@
class Captain::AssistantMigration::DraftApplier
ASSISTANT_DESCRIPTION_LIMIT = 500
CONFIG_KEY = 'assistant_migration'.freeze
SCENARIO_DESCRIPTION_LIMIT = 500
ORIGINAL_VALUES_KEY = 'original_values'.freeze
pattr_initialize [:assistant!, :draft!, { dry_run: true }]
def perform
changes = build_changes
apply_changes(changes) unless dry_run
{
assistant_id: assistant.id,
dry_run: dry_run,
changes: changes
}
end
private
def build_changes
{
description: description_change,
response_guidelines: array_change(:response_guidelines, response_guidelines),
guardrails: array_change(:guardrails, guardrails),
config: config_change
}.compact
end
def apply_changes(changes)
assistant.transaction do
assistant.update!(assistant_update_attributes(changes)) if assistant_update_attributes(changes).present?
end
end
def assistant_update_attributes(changes)
{}.tap do |attributes|
attributes[:description] = changes.dig(:description, :to) if changes[:description].present?
attributes[:response_guidelines] = changes.dig(:response_guidelines, :to) if changes[:response_guidelines].present?
attributes[:guardrails] = changes.dig(:guardrails, :to) if changes[:guardrails].present?
attributes[:config] = changes.dig(:config, :to) if changes[:config].present?
end
end
def description_change
value = assistant_description_value
return if value.blank? || value == assistant.description
{ from: assistant.description, to: value }
end
def assistant_description_value
value = item_values(:business_product_context).join(' ').presence
return if value.blank?
raise ArgumentError, "Assistant description exceeds #{ASSISTANT_DESCRIPTION_LIMIT} characters" if value.length > ASSISTANT_DESCRIPTION_LIMIT
value
end
def response_guidelines
(item_values(:response_guidelines) + scenario_response_guidelines).uniq
end
def guardrails
item_values(:guardrails)
end
def array_change(field, values)
return if values.blank?
current = Array(assistant.public_send(field)).map(&:to_s)
return if current == values
{ from: current, to: values }
end
def config_change
updated_config = assistant.config.deep_dup
conversation_messages.each do |key, value|
next if value.blank?
next if updated_config[key].present?
updated_config[key] = value
end
updated_config[CONFIG_KEY] = migration_config
return if updated_config == assistant.config
{ from: assistant.config, to: updated_config }
end
def migration_config
existing_migration_config.merge(
ORIGINAL_VALUES_KEY => existing_original_values,
'scenario_candidates' => staged_scenario_candidates,
'faq_document_candidates' => normalized_faq_document_candidates,
'needs_review' => normalized_instruction_items(:needs_review)
)
end
def existing_migration_config
config = assistant.config[CONFIG_KEY]
config.is_a?(Hash) ? config : {}
end
def existing_original_values
existing_migration_config[ORIGINAL_VALUES_KEY].presence || original_values
end
def original_values
{
'name' => assistant.name,
'description' => assistant.description,
'config' => original_config,
'response_guidelines' => Array(assistant.response_guidelines),
'guardrails' => Array(assistant.guardrails)
}
end
def original_config
assistant.config.except(CONFIG_KEY)
end
def conversation_messages
messages = draft_hash.fetch(:conversation_messages, {})
messages = messages.deep_stringify_keys
{
'welcome_message' => messages['welcome_message'].to_s.strip,
'handoff_message' => messages['handoff_message'].to_s.strip,
'resolution_message' => messages['resolution_message'].to_s.strip
}
end
def staged_scenario_candidates
scenario_candidates.map do |candidate|
candidate.transform_keys(&:to_s)
end
end
def scenario_response_guidelines
scenario_candidates.filter_map { |candidate| candidate[:response_guideline].presence }
end
def scenario_tool_ids(tool_ids)
Array(tool_ids).filter_map { |tool_id| tool_id.to_s.squish.presence }.uniq
end
def scenario_candidates
Array(draft_hash[:scenario_candidates]).filter_map do |candidate|
normalized_scenario_candidate(candidate)
end
end
def normalized_scenario_candidate(candidate)
return unless candidate.is_a?(Hash)
candidate = candidate.deep_symbolize_keys
normalized_candidate = {
title: candidate[:title].to_s.squish,
description: candidate[:description].to_s.squish.truncate(SCENARIO_DESCRIPTION_LIMIT),
instruction: candidate[:instruction].to_s.squish,
response_guideline: candidate[:response_guideline].to_s.squish,
tool_ids: scenario_tool_ids(candidate[:tool_ids])
}
return if normalized_candidate.values_at(:title, :description, :instruction).any?(&:blank?)
normalized_candidate
end
def item_values(key)
Array(draft_hash[key]).filter_map do |item|
item.to_s.squish.presence
end.uniq
end
def normalized_instruction_items(key)
item_values(key)
end
def normalized_faq_document_candidates
Array(draft_hash[:faq_document_candidates]).map do |candidate|
raise ArgumentError, 'FAQ document candidates must be question and answer objects' unless candidate.is_a?(Hash)
candidate = candidate.deep_symbolize_keys
question = candidate[:question].to_s.squish
answer = candidate[:answer].to_s.squish
raise ArgumentError, 'FAQ document candidates must include a question and answer' if question.blank? || answer.blank?
{ 'question' => question, 'answer' => answer }
end.uniq
end
def draft_hash
@draft_hash ||= draft.deep_symbolize_keys
end
end
@@ -0,0 +1,148 @@
class Captain::AssistantMigration::InstructionClassifier < Captain::BaseTaskService
RESPONSE_SCHEMA = Captain::AssistantMigration::InstructionClassifierSchema
CLASSIFIER_MODEL = 'gpt-5.2'.freeze
MAX_INSTRUCTIONS_LENGTH = 20_000
pattr_initialize [:assistant!]
def perform
response = make_api_call(model: CLASSIFIER_MODEL, messages: messages, schema: RESPONSE_SCHEMA)
return error_response(response) if response[:error]
{
assistant: assistant_metadata,
draft: normalized_payload(response[:message]),
usage: response[:usage],
request_messages: response[:request_messages]
}
end
private
def account
assistant.account
end
def messages
[
{ role: 'system', content: system_prompt },
{ role: 'user', content: user_prompt }
]
end
def system_prompt
Captain::PromptRenderer.render('instruction_classifier')
end
def user_prompt
JSON.pretty_generate(assistant_payload)
end
def assistant_payload # rubocop:disable Metrics/AbcSize
{
assistant_id: assistant.id,
account_id: assistant.account_id,
account_name: assistant.account.name,
name: assistant.name,
description: assistant.description,
product_name: assistant.config['product_name'],
instructions: truncated_instructions,
welcome_message: assistant.config['welcome_message'],
handoff_message: assistant.config['handoff_message'],
resolution_message: assistant.config['resolution_message'],
existing_response_guidelines: assistant.response_guidelines || [],
existing_guardrails: assistant.guardrails || [],
existing_scenarios: existing_scenarios,
available_agent_tools: available_agent_tools,
feature_settings: feature_settings
}
end
def truncated_instructions
instructions = assistant.config['instructions'].to_s
return instructions if instructions.length <= MAX_INSTRUCTIONS_LENGTH
"#{instructions.first(MAX_INSTRUCTIONS_LENGTH)}\n\n[TRUNCATED]"
end
def existing_scenarios
assistant.scenarios.map do |scenario|
{
id: scenario.id,
title: scenario.title,
description: scenario.description,
instruction: scenario.instruction,
enabled: scenario.enabled
}
end
end
def available_agent_tools
tools = assistant.respond_to?(:available_agent_tools) ? assistant.available_agent_tools : Captain::Assistant.built_in_agent_tools
tools.map { |tool| tool.slice(:id, :title, :description) }
end
def feature_settings
assistant.config.slice(
'feature_faq',
'feature_memory',
'feature_citation',
'feature_contact_attributes',
'temperature'
)
end
def normalized_payload(message)
payload = message.is_a?(Hash) ? message.deep_symbolize_keys : {}
payload.reverse_merge(
business_product_context: [],
response_guidelines: [],
guardrails: [],
scenario_candidates: [],
conversation_messages: {},
faq_document_candidates: [],
needs_review: [],
classification_notes: []
)
end
def assistant_metadata # rubocop:disable Metrics/AbcSize
{
id: assistant.id,
name: assistant.name,
account_id: assistant.account_id,
account_name: assistant.account.name,
inbox_count: assistant.captain_inboxes.size,
instruction_length: assistant.config['instructions'].to_s.length,
original_instructions: assistant.config['instructions'].to_s,
welcome_message: assistant.config['welcome_message'].to_s,
handoff_message: assistant.config['handoff_message'].to_s,
resolution_message: assistant.config['resolution_message'].to_s
}
end
def error_response(response)
{
assistant: assistant_metadata,
error: response[:error],
error_code: response[:error_code],
request_messages: response[:request_messages]
}
end
def event_name
'assistant_migration_instruction_classifier'
end
def captain_tasks_enabled?
true
end
def counts_toward_usage?
false
end
def build_follow_up_context?
false
end
end
@@ -0,0 +1,91 @@
class Captain::AssistantMigration::InstructionClassifierSchema < RubyLLM::Schema
DESCRIPTION_LENGTH_LIMIT = 500
def self.instruction_items(field_name, description:, max_items: 20)
array field_name,
description: "#{description} Return plain standalone sentences without numbering, bullets, or section labels.",
max_items: max_items,
of: :string
end
array :business_product_context,
description: "Single compact root assistant description for the root orchestrator prompt, maximum #{DESCRIPTION_LENGTH_LIMIT} characters: " \
'preserve the existing assistant description and enrich it only with relevant business/product context from the ' \
'custom instructions. Include assistant identity, product scope, high-level mission, and high-level source/routing ' \
'priorities only. Do not include workflows, procedures, attribute glossaries, policy details, or long inventories. ' \
'Return complete plain prose without numbering, bullets, section labels, or a truncated final sentence.',
min_items: 1,
max_items: 1 do
string max_length: DESCRIPTION_LENGTH_LIMIT
end
instruction_items :response_guidelines,
description: 'Tone, language, answer length, formatting, and clarification behavior.',
max_items: 20
instruction_items :guardrails,
description: 'Refusal rules, escalation boundaries, source boundaries, safety limits, and things the assistant must not do.',
max_items: 20
array :scenario_candidates,
description: 'Review-stage specialized-agent candidates. These are also temporarily flattened into response guidelines.',
max_items: 15 do
object do
string :title,
description: 'Short scenario agent title for a distinct user-intent workflow.',
max_length: 80
string :description,
description: 'When this specialized scenario should be used. This is shown to the orchestrator for routing.',
max_length: 500
string :instruction,
description: 'How the specialized agent should handle the workflow. Include only evidence-backed markdown tool links. ' \
'Do not include confidence labels or review notes.',
max_length: 2000
string :response_guideline,
description: 'Same-language, customer-visible response guideline that preserves this scenario behavior when flattened. ' \
'Do not include tool syntax, tool names, labels, private-note instructions, or internal implementation details.',
max_length: 1000
array :tool_ids,
description: 'Available tool IDs explicitly referenced in instruction using markdown links. Empty when no tools are required.',
max_items: 10,
of: :string
end
end
object :conversation_messages, description: 'Exact globally reusable customer-facing message copy found in instructions. ' \
'Leave empty for conditional, placeholder, or workflow-specific copy.' do
string :welcome_message, description: 'Exact globally reusable initial greeting copy from instructions, or empty string. ' \
'Do not convert an instruction about greeting into message copy.',
max_length: 1000
string :handoff_message,
description: 'Exact globally reusable human-handoff message copy from instructions, or empty string. ' \
'Do not use scenario-specific, team-specific, placeholder, or conditional handoff copy.',
max_length: 1000
string :resolution_message,
description: 'Exact globally reusable resolution/closing message copy from instructions, or empty string. ' \
'Do not use conditional or placeholder closing copy.',
max_length: 1000
end
array :faq_document_candidates,
description: 'Pending FAQ candidates for factual or product-specific knowledge such as pricing, policy, setup, troubleshooting, ' \
'or operational details. These candidates remain inactive until reviewed and approved.',
max_items: 25 do
object do
string :question,
description: 'Natural, standalone customer question about factual product or business knowledge.',
max_length: 255
string :answer,
description: 'Self-contained factual answer using only the existing instructions. Do not include assistant behavior, ' \
'tool use, or message copy. Preserve exact values, conditions, and exceptions.',
max_length: 2000
end
end
instruction_items :needs_review,
description: 'Unclear, conflicting, risky, duplicated, or uncertain content that needs human review. ' \
'Include the reason in the item text.',
max_items: 20
array :classification_notes, description: 'Short notes about important migration decisions or risks.', max_items: 10, of: :string
end
@@ -0,0 +1,137 @@
You are migrating Captain assistant instructions into a structured configuration.
Classify the existing assistant instructions into these sections:
1. Business/Product Context
2. Response Guidelines
3. Guardrails
4. Scenario Candidates
5. Conversation Messages
6. FAQs/Documents Candidates
7. Needs Review
## General Rules
- Preserve behavior as closely as possible.
- Do not duplicate the same content across sections.
- Return clean migrated values only. Do not include source excerpts, source labels, citations, or "Source:" text in any migrated field.
- Do not rewrite customer-facing message copy unless necessary to classify an exact copy from instructions.
- Do not include confidence labels, review labels, bracketed reviewer comments, or schema labels inside migrated values.
- For Business/Product Context, Response Guidelines, and Guardrails, return each item as a plain standalone sentence.
Do not prefix items with numbers, bullets, section labels, or list markers such as "1.", "-", or "*".
- When several instructions share the same trigger, condition, or subject, combine them into one concise item instead
of repeating the same trigger across multiple items. Preserve every required action, prohibition, and routing
outcome from the source instruction when combining.
- If unsure, place content in Needs Review and include the reason in that item.
- Return data that matches the provided schema.
## Business/Product Context
- Business/Product Context maps to the root assistant description and is injected into the root orchestrator prompt.
- Return exactly one Business/Product Context item.
- Start with the existing assistant description and preserve its meaning.
- Enrich it only with relevant business or product context found in the custom instructions.
- Produce one coherent description rather than appending a second context block or repeating the existing description.
- Keep it at most 500 characters because that is the assistant description limit in the UI and model.
- Prefer roughly 300-450 characters when the source needs detail, leaving room below the hard limit.
- Finish the description cleanly. Never end mid-word, mid-clause, after an opening bracket, or with a dangling separator.
- Make it a compact summary of assistant identity, product scope, high-level mission, and high-level source or routing priorities.
- Do not include detailed workflows, step-by-step procedures, long support-scope inventories, attribute glossaries,
policy details, scenario-specific handling, tool instructions, or customer-facing message copy.
## Conversation Messages
- Existing welcome_message, handoff_message, and resolution_message config values are provided separately.
- Treat welcome_message, handoff_message, and resolution_message as conversation message config fields.
- Extract exact welcome, handoff, or resolution message copy from instructions into conversation_messages when present.
- Only classify handoff copy as conversation_messages.handoff_message when it is generic enough to reuse for any human handoff.
- If handoff copy is scenario-specific, keep it inside that scenario instruction; if it is only a rule about when or how to hand off, classify it as a Response Guideline or Guardrail.
- Do not extract a conversation message from an instruction about what to say, from a placeholder template,
from conditional copy, from role/team-specific copy, or from text that only applies inside one workflow.
- If a message contains placeholders such as a blank name, team name, bracketed variable, business-hours state,
or dynamic runtime condition, do not place it in conversation_messages. Keep it in the relevant workflow or
Needs Review.
- Do not copy message values from existing config into conversation_messages.
- Do not decide whether existing config values should be overwritten. Migration code handles applying extracted
conversation_messages only when the corresponding config value is blank.
## Scenario Candidates
- In the current architecture, a scenario becomes a specialized sub-agent with its own title, description,
instructions, and optional tools.
- During this migration, scenario candidates are also temporarily flattened into response guidelines so existing
assistant behavior is preserved before scenario records are created.
- For every scenario candidate, write a response_guideline that is the flattened version of that scenario for
the root assistant's response guidelines.
- The response_guideline must be in the same language as the original scenario or source instruction.
- The response_guideline must preserve the intended customer-visible behavior, trigger, information to collect,
and routing/escalation outcome.
- The response_guideline must not include tool syntax, tool:// links, markdown tool links, tool names, label
updates, priority updates, private-note instructions, custom-tool instructions, or internal implementation details.
- If the scenario uses internal tools such as labels, priorities, private notes, or custom tools, describe only
the customer-visible behavior and expected routing/escalation outcome in response_guideline.
- If human handoff is needed, describe it in natural language such as route/escalate/transfer to a human; do not
mention the handoff tool in response_guideline.
- Keep scenario titles, descriptions, instructions, and response_guidelines clear, self-contained, and reviewable.
- Only create scenario candidates for distinct user-intent workflows that should be routed to a specialized agent.
A candidate must be narrow enough to become a named specialist assistant with domain-specific handling instructions.
- Good scenario candidates include multi-step intake workflows, qualification flows, specialized troubleshooting
workflows, booking flows, lead-capture flows, recommendation flows, fulfillment workflows, or tool-use procedures
for a specific user intent.
- A scenario candidate should answer "yes" to this test: would a named specialist sub-agent improve handling
beyond the base assistant's global FAQ, guardrail, response-guideline, and human-handoff behavior?
- Do not create scenario candidates for global escalation rules, generic handoff policy, missing-information
behavior, source-boundary rules, refusal rules, tone, formatting, answer length, or one-step fallback behavior.
- Do not create scenario candidates whose main purpose is to escalate or hand off. "Identify the trigger, avoid
guessing, tell the user support will review, and hand off" is a guardrail/handoff boundary, not a scenario,
even though it contains multiple statements.
- Do create scenario candidates when the instructions define a concrete intake, qualification, troubleshooting,
booking, lead-capture, recommendation, or fulfillment workflow, even when the workflow eventually hands off
to a human.
- Do not create scenario candidates for simple routing triggers such as "user asks for a human", "immediately
hand off this category", or "route sales questions to the sales team" when there is no concrete workflow to run.
- Handoff behavior is a scenario candidate only when part of a larger intake, qualification, or specialized handling workflow.
- Global rules like "if not in docs, escalate", "ask one clarifying question", "do not answer account-specific
questions", or "tell the user support will review" belong in Guardrails or Response Guidelines, not Scenario Candidates.
- Broad buckets like "account-specific issue escalation", "unknown question escalation", "contact support",
"fallback to human", or "documentation unavailable" are not scenario candidates.
## Tool Use
- If a scenario candidate requires tools, reference the available tool explicitly inside the scenario instruction
using markdown tool links such as [Handoff to Human](tool://handoff).
- Use only tool IDs listed in available_agent_tools. If a needed tool is unavailable or the workflow depends on
unavailable runtime data such as FAQ relevance scores or business-hours status, place it in Needs Review instead.
- Do not map an unavailable named tool to a different available tool. For example, do not treat FAQ Lookup as
Product Search, Order Status, website browsing, pricing lookup, agent availability, business-hours detection,
ticket creation, or custom-attribute assignment unless the instructions explicitly say that the available
tool provides that behavior.
- If a workflow cannot run without an unavailable tool or runtime signal, do not create a tool-backed scenario
for it. Preserve the instruction in Needs Review with the missing capability named.
## FAQs/Documents Candidates
- Convert factual or product-specific knowledge into pending FAQ candidates with a natural customer question and a self-contained answer.
- FAQ candidates are review-stage data only. They are not active assistant knowledge until a human reviews and approves them.
- Use only facts stated in the existing instructions. Do not invent, generalize, update, or fill in missing details.
- Preserve exact prices, limits, dates, time zones, conditions, exceptions, product names, and operational details in the answer.
- Write each question as a standalone question a customer might naturally ask. Make it specific enough to retrieve the corresponding answer.
- Write each answer so it fully answers its question without relying on another FAQ candidate or surrounding context.
- Split unrelated facts into separate candidates. Keep related conditions and exceptions together when separating them would make an answer incomplete.
- Do not create FAQ candidates about what the assistant should say or do, how it should use sources or tools, when it should route or escalate,
or which exact message it should send. Classify those as Response Guidelines, Guardrails, Scenario Candidates, Conversation Messages,
or Needs Review as appropriate.
- FAQ questions must ask about the product or business, not about the assistant. Do not write questions such as "What should the assistant answer?",
"What should I say?", "Which source should the assistant use?", or "Which tool should be called?".
- FAQ answers must contain customer-facing knowledge, not instructions to call tools, inspect internal data, update records, transfer conversations,
or follow internal workflows.
- When factual sources conflict and the instructions do not explicitly establish which fact overrides the others, put the conflict in Needs Review
instead of creating an FAQ candidate. Use an explicitly stated override or superseding fact when one is present.
- Only factual or product-specific knowledge should become FAQs/Documents candidates.
- Generic capability statements such as "answer product questions", "help with billing",
"troubleshoot common issues", or "direct to documentation" are not FAQ/document candidates.
Put them in Business/Product Context or Response Guidelines when useful.
- Product facts, pricing, policies, setup steps, troubleshooting facts, support hours, emergency contacts,
and operational details should become pending FAQ candidates, not Response Guidelines or trusted approved knowledge.
- Do not create FAQ/document candidates for topic labels or unsupported capabilities when the factual content is
missing. Put "pricing details are needed", "same-day delivery schedule details are needed", or similar gaps in
Needs Review instead.
+278
View File
@@ -0,0 +1,278 @@
require 'json'
require 'fileutils'
require 'csv'
# rubocop:disable Metrics/BlockLength
namespace :captain do
namespace :assistant_migration do
desc 'Generate structured migration drafts. Usage: rake captain:assistant_migration:generate IDS=1,2,3 LIMIT=50 ' \
'OUTPUT=tmp/captain_migration.jsonl'
task generate: :environment do
assistants = CaptainAssistantMigrationTask.assistants
output_path = ENV.fetch('OUTPUT', Rails.root.join('tmp/captain_assistant_migration_drafts.jsonl').to_s)
FileUtils.mkdir_p(File.dirname(output_path))
processed = 0
File.open(output_path, 'w') do |file|
CaptainAssistantMigrationTask.each_assistant(assistants) do |assistant|
result = Captain::AssistantMigration::InstructionClassifier.new(assistant: assistant).perform
file.puts(JSON.generate(result))
processed += 1
puts "Generated migration draft for assistant #{assistant.id} (#{processed}/#{CaptainAssistantMigrationTask.assistant_count(assistants)})"
end
end
puts "Wrote #{processed} migration drafts to #{output_path}"
end
desc 'Apply reviewed migration drafts. Usage: rake captain:assistant_migration:apply INPUT=tmp/reviewed.jsonl DRY_RUN=true'
task apply: :environment do
input_path = ENV.fetch('INPUT')
dry_run = CaptainAssistantMigrationTask.truthy?('DRY_RUN', default: true)
results = CaptainAssistantMigrationTask.apply_drafts(
input_path: input_path,
dry_run: dry_run
)
results.each { |result| puts(JSON.generate(result)) }
puts "Processed #{results.size} migration drafts from #{input_path}"
puts 'Dry run only. Re-run with DRY_RUN=false to write changes.' if dry_run
end
desc 'Restore conversation message config from migration backup. Usage: rake captain:assistant_migration:restore_messages IDS=1,2 DRY_RUN=true'
task restore_messages: :environment do
dry_run = CaptainAssistantMigrationTask.truthy?('DRY_RUN', default: true)
results = CaptainAssistantMigrationTask.restore_conversation_messages(dry_run: dry_run)
results.each { |result| puts(JSON.generate(result)) }
puts "Processed #{results.size} assistant message restores"
puts 'Dry run only. Re-run with DRY_RUN=false to restore conversation messages.' if dry_run
end
end
end
# rubocop:enable Metrics/BlockLength
# rubocop:disable Style/OneClassPerFile
class CaptainAssistantMigrationTask
CsvAccount = Struct.new(:id, :name, keyword_init: true) do
def captain_models
{}
end
def conversations
CsvRelation.new
end
end
CsvAssociation = Struct.new(:inbox_count, keyword_init: true) do
def size
inbox_count
end
end
class CsvRelation
def find_by(*)
nil
end
def exists?
false
end
end
CsvAssistant = Struct.new(
:id,
:name,
:account_id,
:account,
:description,
:config,
:response_guidelines,
:guardrails,
:captain_inboxes,
:scenarios,
keyword_init: true
)
class << self
def assistants
return csv_assistants if ENV['CSV_INPUT'].present?
scope = Captain::Assistant.includes(:account, :captain_inboxes, :scenarios)
ids = ENV.fetch('IDS', '').split(',').filter_map { |id| id.strip.presence }
scope = scope.where(id: ids) if ids.any?
scope = migration_eligible_scope(scope).order(:id)
limit = ENV.fetch('LIMIT', 50).to_i
limit.positive? ? scope.limit(limit) : scope
end
def each_assistant(assistants, &)
return assistants.find_each(&) if assistants.respond_to?(:find_each)
assistants.each(&)
end
def assistant_count(assistants)
assistants.respond_to?(:size) ? assistants.size : assistants.count
end
def restore_conversation_messages(dry_run:)
ENV.fetch('IDS').split(',').filter_map { |id| id.strip.presence }.map do |assistant_id|
assistant = Captain::Assistant.find(assistant_id)
restore_conversation_messages_for(assistant, dry_run: dry_run)
rescue ActiveRecord::RecordNotFound
{ assistant_id: assistant_id, error: 'Assistant not found' }
end
end
def apply_drafts(input_path:, dry_run:)
File.readlines(input_path, chomp: true).filter_map.with_index(1) do |line, line_number|
next if line.blank?
apply_draft(JSON.parse(line), line_number: line_number, dry_run: dry_run)
rescue JSON::ParserError => e
{ line_number: line_number, error: "Invalid JSON: #{e.message}" }
end
end
def apply_draft(payload, line_number:, dry_run:)
return { line_number: line_number, skipped: true, reason: payload['error'] } if payload['error'].present?
assistant_id = payload.dig('assistant', 'id') || payload['assistant_id']
assistant = Captain::Assistant.find(assistant_id)
return skipped_result(line_number, assistant_id, 'Assistant is not a V1 migration candidate') unless migration_candidate?(assistant)
draft = payload['draft'] || payload
Captain::AssistantMigration::DraftApplier.new(
assistant: assistant,
draft: draft,
dry_run: dry_run
).perform.merge(line_number: line_number)
rescue ActiveRecord::RecordNotFound
{ line_number: line_number, assistant_id: assistant_id, error: 'Assistant not found' }
end
def truthy?(key, default:)
value = ENV.fetch(key, nil)
return default if value.nil?
value.to_s.downcase.in?(%w[1 true yes y])
end
private
def restore_conversation_messages_for(assistant, dry_run:)
original_config = assistant.config.dig(
Captain::AssistantMigration::DraftApplier::CONFIG_KEY,
Captain::AssistantMigration::DraftApplier::ORIGINAL_VALUES_KEY,
'config'
)
return skipped_result(nil, assistant.id, 'No stored migration original config found') if original_config.nil?
config, changes = restored_message_config(assistant.config.deep_dup, original_config)
assistant.update!(config: config) if !dry_run && changes.present?
{ assistant_id: assistant.id, dry_run: dry_run, changes: changes }
end
def restored_message_config(config, original_config)
changes = {}
%w[welcome_message handoff_message resolution_message].each do |key|
original_present = original_config.key?(key)
next if config[key] == original_config[key] && config.key?(key) == original_present
changes[key] = { from: config[key], to: original_config[key] }
original_present ? config[key] = original_config[key] : config.delete(key)
end
[config, changes]
end
def skipped_result(line_number, assistant_id, reason)
{
line_number: line_number,
assistant_id: assistant_id,
skipped: true,
reason: reason
}
end
def migration_eligible_scope(scope)
scope.left_outer_joins(:scenarios)
.joins(:captain_inboxes)
.where("NULLIF(captain_assistants.config->>'instructions', '') IS NOT NULL")
.where("captain_assistants.response_guidelines IS NULL OR captain_assistants.response_guidelines = '[]'::jsonb")
.where("captain_assistants.guardrails IS NULL OR captain_assistants.guardrails = '[]'::jsonb")
.where(captain_scenarios: { id: nil })
.distinct
end
def migration_candidate?(assistant)
assistant.config['instructions'].present? &&
assistant.captain_inboxes.size.positive? &&
Array(assistant.response_guidelines).blank? &&
Array(assistant.guardrails).blank? &&
!scenarios_exist?(assistant)
end
def scenarios_exist?(assistant)
scenarios = assistant.scenarios
return scenarios.exists? if scenarios.respond_to?(:exists?)
scenarios.present?
end
def csv_assistants # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
rows = CSV.read(ENV.fetch('CSV_INPUT'), headers: true)
ids = ENV.fetch('IDS', '').split(',').filter_map { |id| id.strip.presence }
status = ENV.fetch('STATUS', '').presence
assistants = rows.filter_map do |row|
next if ids.any? && ids.exclude?(row['id'].to_s)
next if status.present? && row['status'].to_s != status
assistant = csv_assistant(row)
next unless migration_candidate?(assistant)
assistant
end
limit = ENV.fetch('LIMIT', 50).to_i
limit.positive? ? assistants.first(limit) : assistants
end
def csv_assistant(row)
config = parse_json(row['config'], fallback: {})
CsvAssistant.new(
id: normalize_integer(row['id']),
name: row['name'].to_s,
account_id: normalize_integer(row['account_id']),
account: CsvAccount.new(id: normalize_integer(row['account_id']), name: row['account_name'].to_s),
description: row['description'].to_s,
config: config,
response_guidelines: parse_json(row['response_guidelines'], fallback: []),
guardrails: parse_json(row['guardrails'], fallback: []),
captain_inboxes: CsvAssociation.new(inbox_count: normalize_integer(row['inbox_count'])),
scenarios: []
)
end
def parse_json(value, fallback:)
return fallback if value.blank?
JSON.parse(value)
rescue JSON::ParserError
fallback
end
def normalize_integer(value)
value.to_s.delete(',').to_i
end
end
end
# rubocop:enable Style/OneClassPerFile
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.9 KiB

@@ -0,0 +1,116 @@
require 'rails_helper'
RSpec.describe Captain::AssistantMigration::DraftApplier do
let(:account) { create(:account) }
let(:assistant) do
create(
:captain_assistant,
account: account,
config: { 'product_name' => 'Test Product', 'instructions' => 'Legacy V1 custom instructions.' },
response_guidelines: [],
guardrails: []
)
end
let(:scenario_candidate) do
{
'title' => 'Billing Investigation',
'description' => 'Use when a customer reports an account-specific billing issue.',
'instruction' => 'Collect the invoice number and summarize the issue before escalating.',
'response_guideline' => 'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.',
'tool_ids' => []
}
end
let(:faq_document_candidate) do
{
'question' => 'When is support available?',
'answer' => 'Support is available Monday to Friday.'
}
end
let(:draft) do
{
business_product_context: ['Support assistant for Test Product.'],
response_guidelines: ['Be concise.'],
guardrails: ['Do not guess.'],
conversation_messages: {},
scenario_candidates: [scenario_candidate],
faq_document_candidates: [faq_document_candidate],
needs_review: ['Pricing details are missing because factual details are absent from the source instructions.']
}
end
describe '#perform' do
it 'reports staged scenario candidates in dry run without writing to the assistant' do
result = described_class.new(assistant: assistant, draft: draft, dry_run: true).perform
expect(result.dig(:changes, :config, :to, 'assistant_migration', 'scenario_candidates')).to eq([scenario_candidate])
expect(result.dig(:changes, :response_guidelines, :to)).to include(
'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.'
)
expect(assistant.reload.config).not_to have_key('assistant_migration')
expect(assistant.scenarios.count).to eq(0)
end
it 'stores scenario candidates in assistant config and flattens them into response guidelines' do
described_class.new(assistant: assistant, draft: draft, dry_run: false).perform
assistant.reload
expect(assistant.config.dig('assistant_migration', 'scenario_candidates')).to eq([scenario_candidate])
expect(assistant.config.dig('assistant_migration', 'faq_document_candidates')).to contain_exactly(faq_document_candidate)
expect(assistant.config.dig('assistant_migration', 'needs_review')).to contain_exactly(
'Pricing details are missing because factual details are absent from the source instructions.'
)
expect(assistant.response_guidelines).to include(
'For account-specific billing issues, collect the invoice number and summarize the issue before escalating.'
)
expect(assistant.response_guidelines).not_to include(faq_document_candidate['answer'])
expect(assistant.scenarios.count).to eq(0)
end
it 'rejects stale drafts whose FAQ candidates use the old string format' do
stale_draft = draft.merge(faq_document_candidates: ['Support is available Monday to Friday.'])
expect do
described_class.new(assistant: assistant, draft: stale_draft, dry_run: false).perform
end.to raise_error(ArgumentError, 'FAQ document candidates must be question and answer objects')
expect(assistant.reload.config).not_to have_key('assistant_migration')
end
it 'preserves original values in migration config before applying classifier output' do
assistant.update!(
description: 'Existing assistant description.',
response_guidelines: ['Use plain language.'],
guardrails: ['Do not disclose internal notes.']
)
described_class.new(assistant: assistant, draft: draft, dry_run: false).perform
assistant.reload
expect(assistant.description).to eq('Support assistant for Test Product.')
expect(assistant.response_guidelines).to include('Be concise.')
expect(assistant.guardrails).to eq(['Do not guess.'])
expect(assistant.config.dig('assistant_migration', 'original_values')).to include(
'name' => assistant.name,
'description' => 'Existing assistant description.',
'config' => { 'product_name' => 'Test Product', 'instructions' => 'Legacy V1 custom instructions.' },
'response_guidelines' => ['Use plain language.'],
'guardrails' => ['Do not disclose internal notes.']
)
end
it 'rejects an oversized assistant description from a stale draft' do
long_context = 'This assistant supports a very broad product surface with many long details. ' * 10
original_description = assistant.description
expect do
described_class.new(
assistant: assistant,
draft: draft.merge(business_product_context: [long_context]),
dry_run: false
).perform
end.to raise_error(ArgumentError, 'Assistant description exceeds 500 characters')
expect(assistant.reload.description).to eq(original_description)
end
end
end
+9
View File
@@ -3,5 +3,14 @@ FactoryBot.define do
data_type { 'contacts' }
import_file { Rack::Test::UploadedFile.new(Rails.root.join('spec/assets/contacts.csv'), 'text/csv') }
account
trait :intercom do
data_type { 'intercom' }
source_type { 'api' }
source_provider { 'intercom' }
import_types { %w[contacts conversations] }
access_token { 'intercom-token' }
import_file { nil }
end
end
end
@@ -0,0 +1,26 @@
require 'rails_helper'
RSpec.describe DataImportErrorFinder do
let(:data_import) { create(:data_import, :intercom) }
it 'returns only the latest five non-skip errors' do
6.times do |index|
data_import.import_errors.create!(
error_code: 'Intercom::Error',
source_object_id: "error_#{index}",
details: { kind: 'run_error' },
created_at: Time.zone.at(index)
)
end
data_import.import_errors.create!(
error_code: 'Intercom::Skipped',
source_object_id: 'skipped_error',
details: { kind: 'skipped' },
created_at: Time.zone.at(10)
)
errors = described_class.new(data_import).import_errors
expect(errors.pluck(:source_object_id)).to eq(%w[error_5 error_4 error_3 error_2 error_1])
end
end
@@ -0,0 +1,38 @@
require 'rails_helper'
RSpec.describe DataImportSkipLogFinder do
let(:data_import) { create(:data_import, :intercom) }
before do
6.times do |index|
data_import.import_errors.create!(
error_code: 'Intercom::Skipped',
source_object_type: 'message',
source_object_id: "message_#{index}",
details: { kind: 'skipped' },
created_at: Time.zone.at(index)
)
end
data_import.import_errors.create!(
error_code: 'Intercom::Skipped',
source_object_type: 'contact',
source_object_id: 'contact_1',
details: { kind: 'skipped' }
)
end
it 'filters skip logs and returns only the latest five', :aggregate_failures do
finder = described_class.new(data_import, skip_logs_type: 'message')
expect(finder.skip_logs.pluck(:source_object_id)).to eq(%w[message_5 message_4 message_3 message_2 message_1])
expect(finder.selected_source_object_type).to eq('message')
expect(finder.counts_by_type).to include('message' => 6, 'contact' => 1)
end
it 'ignores unsupported source object filters' do
finder = described_class.new(data_import, skip_logs_type: 'company')
expect(finder.selected_source_object_type).to be_nil
expect(finder.skip_logs.size).to eq(5)
end
end
@@ -0,0 +1,218 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::ImportJob do
let(:account) { create(:account) }
let(:data_import) do
create(
:data_import, :intercom,
account: account,
import_types: %w[contacts conversations]
)
end
let(:importer) { instance_double(DataImports::Intercom::Importer) }
let(:run_id) { 'intercom-run-1' }
before do
account.enable_features!('data_import')
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
allow(DataImports::Intercom::Importer).to receive(:new).with(data_import: data_import, run_id: run_id).and_return(importer)
end
describe DataImports::Intercom::BaseJob do
it 'checks rate limit retry before the generic client retry' do
expect(described_class.rescue_handlers.last.first).to eq('DataImports::Intercom::Client::RateLimitError')
end
end
describe DataImports::Intercom::ImportJob do
it 'starts the import and enqueues the first contacts page' do
allow(importer).to receive_messages(start!: true, import_contacts?: true, contacts_completed?: false, cursor_for: 'contact-cursor')
expect do
described_class.perform_now(data_import, run_id)
end.to have_enqueued_job(DataImports::Intercom::ContactsPageJob).with(data_import, 'contact-cursor', run_id).on_queue('low')
expect(importer).to have_received(:start!)
end
it 'resumes at conversations when contacts are already completed' do
allow(importer).to receive_messages(
start!: true,
import_contacts?: true,
contacts_completed?: true,
import_conversations?: true,
conversations_completed?: false
)
allow(importer).to receive(:cursor_for).with('conversations').and_return('conversation-cursor')
expect do
described_class.perform_now(data_import, run_id)
end.to have_enqueued_job(DataImports::Intercom::ConversationsPageJob).with(data_import, 'conversation-cursor', run_id)
end
it 'finishes immediately when every requested stage is already complete' do
allow(importer).to receive_messages(
start!: true,
import_contacts?: true,
contacts_completed?: true,
import_conversations?: true,
conversations_completed?: true,
finish!: true
)
expect do
described_class.perform_now(data_import, run_id)
end.not_to have_enqueued_job
expect(importer).to have_received(:finish!)
end
it 'skips stale import jobs from an earlier run' do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
expect(DataImports::Intercom::Importer).not_to receive(:new)
described_class.perform_now(data_import, 'old-run')
end
end
describe DataImports::Intercom::ContactsPageJob do
it 'hands off to conversations when a retry finds contacts already completed' do
allow(importer).to receive_messages(
contacts_completed?: true,
import_conversations?: true,
conversations_completed?: false
)
allow(importer).to receive(:cursor_for).with('conversations').and_return('conversation-cursor')
expect(importer).not_to receive(:import_contacts_page)
expect do
described_class.perform_now(data_import, 'completed-contact-cursor', run_id)
end.to have_enqueued_job(DataImports::Intercom::ConversationsPageJob).with(data_import, 'conversation-cursor', run_id)
end
it 'imports one contacts page and enqueues the next contacts page' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: 'next-contact-cursor')
allow(importer).to receive_messages(contacts_completed?: false)
allow(importer).to receive(:import_contacts_page).with(starting_after: 'current-contact-cursor').and_return(result)
expect do
described_class.perform_now(data_import, 'current-contact-cursor', run_id)
end.to have_enqueued_job(described_class).with(data_import, 'next-contact-cursor', run_id)
end
it 'hands off to conversations after the final contacts page' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: nil)
allow(importer).to receive_messages(
contacts_completed?: false,
import_conversations?: true,
conversations_completed?: false
)
allow(importer).to receive(:import_contacts_page).with(starting_after: nil).and_return(result)
allow(importer).to receive(:cursor_for).with('conversations').and_return(nil)
expect do
described_class.perform_now(data_import, nil, run_id)
end.to have_enqueued_job(DataImports::Intercom::ConversationsPageJob).with(data_import, nil, run_id)
end
it 'finishes after the final contacts page when conversations are not requested' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: nil)
allow(importer).to receive_messages(contacts_completed?: false, import_conversations?: false, finish!: true)
allow(importer).to receive(:import_contacts_page).with(starting_after: nil).and_return(result)
expect do
described_class.perform_now(data_import, nil, run_id)
end.not_to have_enqueued_job
expect(importer).to have_received(:finish!)
end
it 'skips stale page jobs from an earlier run' do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
expect(DataImports::Intercom::Importer).not_to receive(:new)
described_class.perform_now(data_import, 'current-contact-cursor', 'old-run')
end
it 'skips failed page jobs from backend retries' do
data_import.update!(status: :failed)
expect(DataImports::Intercom::Importer).not_to receive(:new)
described_class.perform_now(data_import, 'current-contact-cursor', run_id)
end
it 'does not enqueue another stage when the page import becomes stale' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: nil)
allow(importer).to receive_messages(contacts_completed?: false, finish!: true)
allow(importer).to receive(:import_contacts_page).with(starting_after: 'current-contact-cursor') do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
result
end
expect do
described_class.perform_now(data_import, 'current-contact-cursor', run_id)
end.not_to have_enqueued_job
expect(importer).not_to have_received(:finish!)
end
end
describe DataImports::Intercom::ConversationsPageJob do
it 'finishes when a retry finds conversations already completed' do
allow(importer).to receive_messages(conversations_completed?: true, finish!: true)
expect(importer).not_to receive(:import_conversations_page)
described_class.perform_now(data_import, 'completed-conversation-cursor', run_id)
expect(importer).to have_received(:finish!)
end
it 'imports one conversations page and enqueues the next conversations page' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: 'next-conversation-cursor')
allow(importer).to receive_messages(conversations_completed?: false)
allow(importer).to receive(:import_conversations_page).with(starting_after: 'current-conversation-cursor').and_return(result)
expect do
described_class.perform_now(data_import, 'current-conversation-cursor', run_id)
end.to have_enqueued_job(described_class).with(data_import, 'next-conversation-cursor', run_id)
end
it 'finishes after the final conversations page' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: nil)
allow(importer).to receive_messages(conversations_completed?: false, finish!: true)
allow(importer).to receive(:import_conversations_page).with(starting_after: nil).and_return(result)
expect do
described_class.perform_now(data_import, nil, run_id)
end.not_to have_enqueued_job
expect(importer).to have_received(:finish!)
end
it 'skips stale page jobs from an earlier run' do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
expect(DataImports::Intercom::Importer).not_to receive(:new)
described_class.perform_now(data_import, 'current-conversation-cursor', 'old-run')
end
it 'does not finish when the page import becomes stale' do
result = DataImports::Intercom::Importer::PageResult.new(next_cursor: nil)
allow(importer).to receive_messages(conversations_completed?: false, finish!: true)
allow(importer).to receive(:import_conversations_page).with(starting_after: 'current-conversation-cursor') do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
result
end
expect do
described_class.perform_now(data_import, 'current-conversation-cursor', run_id)
end.not_to have_enqueued_job
expect(importer).not_to have_received(:finish!)
end
end
end
+6 -2
View File
@@ -109,6 +109,8 @@ RSpec.describe Account do
it 'configures the account feature flag extension column' do
expect(described_class.flag_columns).to include('feature_flags', 'feature_flags_ext_1')
expect(described_class.flag_mapping['feature_flags_ext_1']).to eq(feature_whatsapp_manual_transfer: 1, feature_api_and_webhooks: 1 << 1)
expect(described_class.flag_mapping['feature_flags_ext_1'][:feature_whatsapp_manual_transfer]).to eq(1)
expect(described_class.flag_mapping['feature_flags_ext_1'][:feature_data_import]).to eq(2)
end
it 'keeps existing feature flags on the original column' do
@@ -117,15 +119,17 @@ RSpec.describe Account do
end
it 'keeps bulk selected feature assignment compatible with existing feature names' do
account.selected_feature_flags = [:feature_ip_lookup, :feature_assignment_v2, :feature_advanced_assignment]
account.selected_feature_flags = [:feature_ip_lookup, :feature_assignment_v2, :feature_advanced_assignment, :feature_data_import]
expect(account).to be_feature_ip_lookup
expect(account).to be_feature_assignment_v2
expect(account).to be_feature_advanced_assignment
expect(account).to be_feature_data_import
expect(account.selected_feature_flags).to contain_exactly(
:feature_ip_lookup,
:feature_assignment_v2,
:feature_advanced_assignment
:feature_advanced_assignment,
:feature_data_import
)
end
end
+44
View File
@@ -11,6 +11,18 @@ RSpec.describe DataImport do
end
end
describe 'access token encryption' do
it 'encrypts the Intercom access token at rest' do
skip('encryption keys missing; see run_mfa_spec workflow') unless Chatwoot.encryption_configured?
data_import = create(:data_import, :intercom, access_token: 'intercom-secret')
stored_value = data_import.reload.read_attribute_before_type_cast(:access_token).to_s
expect(stored_value).not_to include('intercom-secret')
expect(data_import.access_token).to eq('intercom-secret')
end
end
describe 'callbacks' do
let(:data_import) { build(:data_import) }
@@ -20,4 +32,36 @@ RSpec.describe DataImport do
end.to have_enqueued_job(DataImportJob).with(data_import).on_queue('low')
end
end
describe '#abandon!' do
let(:account) { create(:account) }
let(:data_import) do
create(
:data_import, :intercom,
account: account,
status: :processing
)
end
before do
account.enable_features!('data_import')
end
it 'abandons active Intercom imports', :aggregate_failures do
data_import.abandon!
expect(data_import).to be_abandoned
expect(data_import.abandoned_at).to be_present
end
it 'does not overwrite terminal status from a stale instance', :aggregate_failures do
stale_import = described_class.find(data_import.id)
data_import.update!(status: :completed, completed_at: 1.minute.ago)
stale_import.abandon!
expect(data_import.reload).to be_completed
expect(data_import.abandoned_at).to be_nil
end
end
end
@@ -0,0 +1,438 @@
require 'rails_helper'
RSpec.describe 'Data Imports API', type: :request do
let(:account) { create(:account) }
let(:admin) { create(:user, account: account, role: :administrator) }
let(:validator) { instance_double(DataImports::Intercom::CredentialsValidator, perform: { 'contacts' => 12, 'conversations' => 8 }) }
before do
account.enable_features!('data_import')
allow(DataImports::Intercom::CredentialsValidator).to receive(:new).and_return(validator)
end
describe 'POST /api/v1/accounts/:account_id/data_imports/validate_source' do
it 'validates the selected Intercom source and returns discovered totals' do
post validate_source_api_v1_account_data_imports_url(account_id: account.id),
params: {
source_provider: 'intercom', access_token: 'intercom-token', import_types: %w[contacts conversations]
},
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(response.parsed_body).to eq('valid' => true, 'totals' => { 'contacts' => 12, 'conversations' => 8 })
end
it 'returns a safe validation error' do
allow(validator).to receive(:perform).and_raise(DataImports::Intercom::Client::AuthenticationError, 'provider response')
post validate_source_api_v1_account_data_imports_url(account_id: account.id),
params: { source_provider: 'intercom', access_token: 'invalid', import_types: %w[contacts] },
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:unprocessable_entity)
expect(response.parsed_body).to eq(
'valid' => false,
'message' => 'We could not validate this Intercom access key. Check the key and its permissions.'
)
end
end
describe 'POST /api/v1/accounts/:account_id/data_imports' do
it 'returns unauthorized and does not enqueue imports when data import is disabled' do
account.disable_features!('data_import')
expect do
post api_v1_account_data_imports_url(account_id: account.id),
params: {
name: 'Migration run', source_provider: 'intercom', access_token: 'intercom-token',
import_types: %w[contacts conversations]
},
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:unauthorized)
expect(account.data_imports).to be_empty
end
it 'creates and enqueues an Intercom import', :aggregate_failures do
expect do
post api_v1_account_data_imports_url(account_id: account.id),
params: {
name: 'Migration run', source_provider: 'intercom', access_token: 'intercom-token',
import_types: %w[contacts conversations]
},
headers: admin.create_new_auth_token,
as: :json
end.to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:ok)
data_import = account.data_imports.last
expect(data_import).to have_attributes(
name: 'Migration run',
data_type: 'intercom',
source_type: 'api',
source_provider: 'intercom',
initiated_by_id: admin.id
)
expect(data_import.access_token).to eq('intercom-token')
expect(data_import.import_types).to eq(%w[contacts conversations])
expect(data_import.stats).to include(
'contacts' => include('total' => 12),
'conversations' => include('total' => 8)
)
expect(response.parsed_body['source_provider']).to eq('intercom')
expect(response.parsed_body).not_to have_key('access_token')
end
it 'rejects creation while another Intercom import is active' do
active_import = create(
:data_import, :intercom,
account: account,
status: :processing
)
expect do
post api_v1_account_data_imports_url(account_id: account.id),
params: {
name: 'Second run', source_provider: 'intercom', access_token: 'intercom-token',
import_types: %w[contacts conversations]
},
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:unprocessable_entity)
expect(response.parsed_body['message']).to eq('Another data import is already in progress.')
expect(account.data_imports.where(data_type: 'intercom', source_provider: 'intercom').count).to eq(1)
expect(active_import.reload).to be_processing
end
it 'rejects unsupported import types instead of silently importing everything' do
allow(validator).to receive(:perform).and_raise(ArgumentError, 'Unsupported import types: companies')
expect do
post api_v1_account_data_imports_url(account_id: account.id),
params: {
name: 'Migration run', source_provider: 'intercom', access_token: 'intercom-token', import_types: %w[companies]
},
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:unprocessable_entity)
expect(response.parsed_body['message']).to eq('Unsupported import types: companies')
expect(account.data_imports).to be_empty
end
end
describe 'POST /api/v1/accounts/:account_id/data_imports/:id/start' do
let(:data_import) { create(:data_import, :intercom, account: account) }
it 'restarts abandoned imports' do
data_import.update!(
status: :abandoned,
abandoned_at: 1.hour.ago,
source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'previous-run' }
)
data_import.import_errors.create!(
error_code: 'StandardError',
message: 'old run error',
details: { kind: 'run_error' }
)
data_import.import_errors.create!(
error_code: DataImports::Intercom::Importer::ALREADY_IMPORTED_ERROR_CODE,
message: 'old skip log',
details: { kind: 'skipped' }
)
expect do
post start_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.to have_enqueued_job(DataImports::Intercom::ImportJob).with(data_import, a_kind_of(String))
expect(response).to have_http_status(:ok)
expect(data_import.reload).to be_pending
expect(data_import.abandoned_at).to be_nil
expect(data_import.started_at).to be_nil
expect(data_import.active_intercom_import_run_id).not_to eq('previous-run')
expect(data_import.import_errors.pluck(:error_code)).to eq([DataImports::Intercom::Importer::ALREADY_IMPORTED_ERROR_CODE])
end
it 'does not enqueue duplicate jobs for active imports' do
data_import.update!(status: :processing)
expect do
post start_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:ok)
expect(data_import.reload).to be_processing
end
it 'returns the active Intercom import instead of restarting another import' do
data_import.update!(status: :abandoned, abandoned_at: 1.hour.ago)
active_import = create(
:data_import, :intercom,
account: account,
status: :processing
)
expect do
post start_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:ok)
expect(response.parsed_body['id']).to eq(active_import.id)
expect(data_import.reload).to be_abandoned
end
it 'does not restart imports when the stored access key is unavailable' do
data_import.update!(status: :abandoned, abandoned_at: 1.hour.ago, access_token: nil)
expect do
post start_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
end.not_to have_enqueued_job(DataImports::Intercom::ImportJob)
expect(response).to have_http_status(:unprocessable_entity)
expect(response.parsed_body['message']).to eq('The Intercom access key for this import is unavailable.')
expect(data_import.reload).to be_abandoned
end
end
describe 'POST /api/v1/accounts/:account_id/data_imports/:id/abandon' do
let(:data_import) { create(:data_import, :intercom, account: account) }
it 'abandons active imports' do
data_import.update!(status: :processing)
post abandon_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(data_import.reload).to be_abandoned
expect(data_import.abandoned_at).to be_present
end
it 'does not rewrite completed imports as abandoned' do
data_import.update!(status: :completed, completed_at: 1.hour.ago)
post abandon_api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(data_import.reload).to be_completed
expect(data_import.abandoned_at).to be_nil
end
it 'does not abandon legacy contact imports' do
legacy_import = create(:data_import, account: account, status: :processing)
post abandon_api_v1_account_data_import_url(account_id: account.id, id: legacy_import.id),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(legacy_import.reload).to be_processing
expect(legacy_import.abandoned_at).to be_nil
end
end
describe 'GET /api/v1/accounts/:account_id/data_imports/:id' do
let(:data_import) do
create(
:data_import, :intercom,
account: account,
name: 'July Intercom migration',
initiated_by: admin
)
end
it 'returns import details with recent errors' do
data_import.import_errors.create!(
error_code: 'Intercom::RateLimited',
message: 'Rate limited',
source_object_type: 'conversation',
source_object_id: 'conversation_1'
)
data_import.import_errors.create!(
error_code: 'DataImports::Intercom::SkippedMessage',
message: 'Skipped blank message',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:blank_part',
details: { kind: 'skipped', reason: 'blank_or_unsupported_intercom_part' }
)
get api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(response.parsed_body).to include(
'id' => data_import.id,
'name' => 'July Intercom migration',
'source_provider' => 'intercom',
'import_errors_count' => 1,
'skip_logs_count' => 1
)
expect(response.parsed_body['import_errors'].first).to include(
'error_code' => 'Intercom::RateLimited',
'message' => 'Rate limited',
'source_object_type' => 'conversation',
'source_object_id' => 'conversation_1'
)
expect(response.parsed_body['skip_logs'].first).to include(
'kind' => 'skipped',
'error_code' => 'DataImports::Intercom::SkippedMessage',
'source_object_type' => 'message',
'source_object_id' => 'conversation:conversation_1:part:blank_part'
)
end
it 'returns the latest five skip logs' do
16.times do |index|
data_import.import_errors.create!(
error_code: 'DataImports::Intercom::AlreadyImported',
message: 'Already imported in a previous import.',
source_object_type: 'message',
source_object_id: "message_#{index}",
details: { kind: 'skipped', reason: 'already_imported' },
created_at: Time.zone.at(index)
)
end
get api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(response.parsed_body['skip_logs'].pluck('source_object_id')).to eq(%w[message_15 message_14 message_13 message_12 message_11])
expect(response.parsed_body).not_to have_key('skip_logs_pagination')
end
it 'filters skip logs by source object type with counts for each type' do
3.times do |index|
data_import.import_errors.create!(
error_code: 'DataImports::Intercom::AlreadyImported',
message: 'Already imported in a previous import.',
source_object_type: 'contact',
source_object_id: "contact_#{index}",
details: { kind: 'skipped', reason: 'already_imported' }
)
end
2.times do |index|
data_import.import_errors.create!(
error_code: 'DataImports::Intercom::AlreadyImported',
message: 'Already imported in a previous import.',
source_object_type: 'message',
source_object_id: "message_#{index}",
details: { kind: 'skipped', reason: 'already_imported' }
)
end
get api_v1_account_data_import_url(account_id: account.id, id: data_import.id, skip_logs_type: 'contact'),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(response.parsed_body['skip_logs'].pluck('source_object_type').uniq).to eq(['contact'])
expect(response.parsed_body['skip_logs_filters']).to include(
'selected_source_object_type' => 'contact',
'counts_by_type' => include('contact' => 3, 'message' => 2)
)
end
it 'returns the latest five error logs' do
16.times do |index|
data_import.import_errors.create!(
error_code: 'Intercom::RateLimited',
message: 'Rate limited',
source_object_type: 'conversation',
source_object_id: "conversation_#{index}",
created_at: Time.zone.at(index)
)
end
get api_v1_account_data_import_url(account_id: account.id, id: data_import.id),
headers: admin.create_new_auth_token,
as: :json
expect(response).to have_http_status(:ok)
expect(response.parsed_body['import_errors'].pluck('source_object_id')).to eq(
%w[conversation_15 conversation_14 conversation_13 conversation_12 conversation_11]
)
expect(response.parsed_body).not_to have_key('import_errors_pagination')
end
end
describe 'GET /api/v1/accounts/:account_id/data_imports/:id/error_logs.csv' do
let(:data_import) do
create(
:data_import, :intercom,
account: account,
initiated_by: admin
)
end
it 'downloads all error logs as CSV' do
6.times do |index|
data_import.import_errors.create!(
error_code: 'Intercom::RateLimited',
message: 'Rate limited',
source_object_type: 'conversation',
source_object_id: "conversation_#{index}",
details: { kind: 'run_error' }
)
end
get error_logs_api_v1_account_data_import_url(account_id: account.id, id: data_import.id, format: :csv),
headers: admin.create_new_auth_token
expect(response).to have_http_status(:ok)
expect(response.media_type).to eq('text/csv')
expect(response.body).to include('source_object_type,source_object_id')
expect(response.body).to include('conversation,conversation_0,Intercom::RateLimited,Rate limited')
expect(response.body).to include('conversation,conversation_5,Intercom::RateLimited,Rate limited')
expect(response.body.lines.size).to eq(7)
end
end
describe 'GET /api/v1/accounts/:account_id/data_imports/:id/skip_logs.csv' do
let(:data_import) do
create(
:data_import, :intercom,
account: account,
initiated_by: admin
)
end
it 'downloads skip logs as CSV' do
data_import.import_errors.create!(
error_code: 'DataImports::Intercom::SkippedMessage',
message: 'Skipped blank message',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:blank_part',
details: { kind: 'skipped', reason: 'blank_or_unsupported_intercom_part' }
)
get skip_logs_api_v1_account_data_import_url(account_id: account.id, id: data_import.id, format: :csv),
headers: admin.create_new_auth_token
expect(response).to have_http_status(:ok)
expect(response.media_type).to eq('text/csv')
expect(response.body).to include('source_object_type,source_object_id')
expect(response.body).to include('message,conversation:conversation_1:part:blank_part')
end
end
end
@@ -0,0 +1,51 @@
require 'rails_helper'
event_content = {
'assignment' => 'Avery assigned the conversation to Support',
'assign_and_reopen' => 'Avery assigned the conversation to Support and reopened it',
'open' => 'Avery opened the conversation',
'close' => 'Avery closed the conversation',
'snoozed' => 'Avery snoozed the conversation',
'participant_added' => 'Avery added Support as a participant',
'participant_removed' => 'Avery removed Support as a participant',
'conversation_attribute_updated_by_admin' => 'Avery updated conversation attributes',
'conversation_attribute_updated_by_user' => 'Avery updated conversation attributes',
'conversation_attribute_updated_by_workflow' => 'Avery updated conversation attributes',
'ticket_attribute_updated_by_admin' => 'Avery updated ticket attributes',
'ticket_state_updated_by_admin' => 'Avery updated the ticket state',
'custom_action_started' => 'Avery started a custom action',
'custom_action_finished' => 'Avery finished a custom action',
'quick_reply' => 'Avery used a quick reply'
}.freeze
RSpec.describe DataImports::Intercom::ActivityContentBuilder do
event_content.each do |part_type, expected_content|
it "builds readable content for #{part_type}" do
part = {
'part_type' => part_type,
'author' => { 'type' => 'admin', 'name' => 'Avery' },
'assigned_to' => { 'name' => 'Support' }
}
expect(described_class.new(part).perform).to eq(expected_content)
end
end
it 'uses a humanized fallback for unknown future event types' do
part = { 'part_type' => 'journey_stage_changed', 'author' => { 'type' => 'bot' } }
expect(described_class.new(part).perform).to eq('Intercom automation recorded journey stage changed')
end
it 'appends sanitized body context' do
part = {
'part_type' => 'close',
'author' => { 'type' => 'admin' },
'body' => '<p>Customer confirmed <strong>resolution</strong></p><script>alert(1)</script>'
}
expect(described_class.new(part).perform).to eq(
'Intercom teammate closed the conversation: Customer confirmed resolution'
)
end
end
@@ -0,0 +1,16 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::Client do
let(:client) { described_class.new(access_token: 'intercom-token') }
describe '#list_contacts' do
it 'wraps transport failures in a retryable client error', :aggregate_failures do
allow(HTTParty).to receive(:get).and_raise(SocketError, 'getaddrinfo failed')
expect { client.list_contacts }.to raise_error(DataImports::Intercom::Client::Error) do |error|
expect(error.message).to eq('Intercom API request failed before receiving a response: getaddrinfo failed')
expect(error.body).to include(transport_error_class: 'SocketError')
end
end
end
end
@@ -0,0 +1,52 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::CreationService do
let(:account) { create(:account) }
let(:user) { create(:user, account: account) }
let(:validator) { instance_double(DataImports::Intercom::CredentialsValidator, perform: { 'contacts' => 12 }) }
before do
allow(DataImports::Intercom::CredentialsValidator).to receive(:new).and_return(validator)
end
it 'validates and creates an import with its credentials and totals', :aggregate_failures do
data_import = described_class.new(
account: account,
initiated_by: user,
source_params: {
name: 'Migration run',
source_provider: 'intercom',
access_token: ' intercom-token ',
import_types: %w[contacts]
}
).perform
expect(data_import).to have_attributes(
name: 'Migration run',
source_type: 'api',
source_provider: 'intercom',
import_types: %w[contacts],
access_token: 'intercom-token',
initiated_by_id: user.id
)
expect(data_import.stats.dig('contacts', 'total')).to eq(12)
expect(data_import.active_intercom_import_run_id).to be_present
end
it 'returns no import without validating when another import is active' do
create(:data_import, :intercom, account: account, status: :processing)
data_import = described_class.new(
account: account,
initiated_by: user,
source_params: {
name: 'Second run',
source_provider: 'intercom',
access_token: 'intercom-token'
}
).perform
expect(data_import).to be_nil
expect(validator).not_to have_received(:perform)
end
end
@@ -0,0 +1,54 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::CredentialsValidator do
let(:client) { instance_double(DataImports::Intercom::Client) }
before do
allow(DataImports::Intercom::Client).to receive(:new).with(access_token: 'intercom-token').and_return(client)
allow(client).to receive(:list_contacts)
allow(client).to receive(:list_conversations)
end
it 'validates and counts only contacts when conversations are not selected' do
allow(client).to receive(:list_contacts).with(per_page: 1).and_return('total_count' => 42)
totals = described_class.new(access_token: ' intercom-token ', import_types: %w[contacts]).perform
expect(totals).to eq('contacts' => 42)
expect(client).not_to have_received(:list_conversations)
end
it 'validates contact access and counts only conversations when contacts are not selected' do
allow(client).to receive(:list_contacts).with(per_page: 1).and_return('total_count' => 42)
allow(client).to receive(:list_conversations).with(per_page: 1).and_return('total_count' => 17)
totals = described_class.new(access_token: 'intercom-token', import_types: %w[conversations]).perform
expect(totals).to eq('conversations' => 17)
expect(client).to have_received(:list_contacts).with(per_page: 1)
end
it 'keeps an undiscovered total absent' do
allow(client).to receive(:list_contacts).with(per_page: 1).and_return('data' => [])
totals = described_class.new(access_token: 'intercom-token', import_types: %w[contacts]).perform
expect(totals).to be_empty
end
it 'preserves a known zero total' do
allow(client).to receive(:list_contacts).with(per_page: 1).and_return('total_count' => 0)
totals = described_class.new(access_token: 'intercom-token', import_types: %w[contacts]).perform
expect(totals).to eq('contacts' => 0)
end
it 'rejects an empty access key before calling Intercom' do
expect do
described_class.new(access_token: '', import_types: %w[contacts]).perform
end.to raise_error(ArgumentError, 'Intercom access key is required.')
expect(DataImports::Intercom::Client).not_to have_received(:new)
end
end
@@ -0,0 +1,968 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::Importer do
let(:account) { create(:account) }
let(:data_import) do
create(
:data_import, :intercom,
account: account
)
end
let(:client) { instance_double(DataImports::Intercom::Client) }
let(:contact_payload) do
{
'id' => 'contact_1',
'external_id' => 'external_1',
'email' => 'CUSTOMER@Example.com',
'phone' => '15551234567',
'name' => 'Customer One',
'created_at' => 1_700_000_000,
'updated_at' => 1_700_000_100
}
end
let(:conversation_payload) do
{
'id' => 'conversation_1',
'created_at' => 1_700_000_000,
'updated_at' => 1_700_000_200,
'state' => 'closed',
'open' => false,
'admin_assignee_id' => 123,
'team_assignee_id' => 456,
'contacts' => { 'contacts' => [{ 'id' => 'contact_1' }] },
'source' => {
'id' => 'source_1',
'type' => 'email',
'delivered_as' => 'customer_initiated',
'subject' => 'Need help',
'body' => '<p>Hello there</p>',
'author' => { 'type' => 'user', 'id' => 'contact_1', 'email' => 'CUSTOMER@example.com' }
},
'conversation_parts' => {
'conversation_parts' => [
{
'id' => 'part_1',
'part_type' => 'comment',
'body' => '<p>Admin reply</p>',
'created_at' => 1_700_000_100,
'updated_at' => 1_700_000_100,
'author' => { 'type' => 'admin', 'id' => 'admin_1' },
'attachments' => []
},
{
'id' => 'part_2',
'part_type' => 'note',
'body' => '<strong>Internal note</strong>',
'created_at' => 1_700_000_150,
'updated_at' => 1_700_000_150,
'author' => { 'type' => 'admin', 'id' => 'admin_1' },
'attachments' => []
}
]
}
}
end
before do
account.enable_features!('data_import')
allow(DataImports::Intercom::Client).to receive(:new).with(access_token: 'intercom-token').and_return(client)
allow(client).to receive(:list_contacts).with(starting_after: nil).and_return(
'data' => [contact_payload],
'total_count' => 1,
'pages' => { 'next' => nil }
)
allow(client).to receive(:list_conversations).with(starting_after: nil).and_return(
'conversations' => [{ 'id' => 'conversation_1' }],
'total_count' => 1,
'pages' => { 'next' => nil }
)
allow(client).to receive(:retrieve_conversation).with('conversation_1').and_return(conversation_payload)
allow(client).to receive(:retrieve_contact).with('contact_1').and_return(contact_payload)
end
it 'imports contacts, conversations, messages, and source-bucket inboxes without normal message creation callbacks', :aggregate_failures do
described_class.new(data_import: data_import).perform
contact = account.contacts.find_by!(email: 'customer@example.com')
expect(contact.name).to eq('Customer One')
expect(contact.phone_number).to eq('+15551234567')
expect(contact).to be_lead
expect(contact.custom_attributes).to include('intercom_contact_id' => 'contact_1')
inbox = account.inboxes.find_by!(name: 'Intercom Import - Email')
expect(inbox.channel.additional_attributes).to include('source_bucket' => 'email', 'import_placeholder' => true)
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
expect(conversation).to have_attributes(
status: 'resolved',
inbox_id: inbox.id,
contact_id: contact.id
)
expect(conversation.additional_attributes.dig('source', 'routing_method')).to eq('source_bucket_api_inbox')
expect(conversation.messages.order(:created_at).pluck(:content)).to eq(["Need help\n\nHello there", 'Admin reply', 'Internal note'])
expect(conversation.messages.order(:created_at).map(&:message_type)).to eq(%w[incoming outgoing outgoing])
expect(conversation.messages.order(:created_at).last.private).to be(true)
expect(data_import.reload).to be_completed
expect(data_import.stats).to include(
'contacts' => include('imported' => 1, 'skipped' => 0, 'total' => 1),
'conversations' => include('imported' => 1, 'skipped' => 0, 'total' => 1),
'messages' => include('imported' => 3, 'skipped' => 0, 'total' => 3),
'errors' => { 'count' => 0 }
)
expect(data_import.processed_records).to eq(5)
expect(data_import.items.imported.count).to eq(2)
expect(DataImportMapping.where(data_import: data_import).count).to eq(5)
end
it 'imports historical records without dispatching record events or outbound side effects', :aggregate_failures do
dispatched_events = []
allow(Rails.configuration.dispatcher).to receive(:dispatch) do |event_name, *_args|
dispatched_events << event_name
end
clear_enqueued_jobs
described_class.new(data_import: data_import).perform
record_events = [
Events::Types::CONTACT_CREATED,
Events::Types::CONTACT_UPDATED,
Events::Types::CONVERSATION_CREATED,
Events::Types::CONVERSATION_UPDATED,
Events::Types::CONVERSATION_STATUS_CHANGED,
Events::Types::ASSIGNEE_CHANGED,
Events::Types::TEAM_CHANGED,
Events::Types::MESSAGE_CREATED,
Events::Types::FIRST_REPLY_CREATED,
Events::Types::REPLY_CREATED
]
side_effect_jobs = [SendReplyJob, EventDispatcherJob, ActionCableBroadcastJob, WebhookJob, HookJob]
expect(dispatched_events & record_events).to be_empty
expect(enqueued_jobs.pluck(:job) & side_effect_jobs).to be_empty
expect(Notification.where(account: account)).to be_empty
end
context 'when Intercom contact activity timestamps are available' do
let(:contact_payload) do
super().merge('last_seen_at' => 1_700_000_050, 'last_replied_at' => 1_700_000_090)
end
it 'prefers last_seen_at for contact activity' do
described_class.new(data_import: data_import).import_contacts_page
contact = account.contacts.find_by!(email: 'customer@example.com')
expect(contact.last_activity_at).to eq(Time.zone.at(1_700_000_050))
end
end
context 'when Intercom contact last_seen_at is unavailable' do
let(:contact_payload) do
super().merge('last_seen_at' => nil, 'last_replied_at' => 1_700_000_090)
end
it 'falls back to last_replied_at for contact activity' do
described_class.new(data_import: data_import).import_contacts_page
contact = account.contacts.find_by!(email: 'customer@example.com')
expect(contact.last_activity_at).to eq(Time.zone.at(1_700_000_090))
end
end
it 'leaves contact activity blank when Intercom activity timestamps are unavailable' do
described_class.new(data_import: data_import).import_contacts_page
contact = account.contacts.find_by!(email: 'customer@example.com')
expect(contact.last_activity_at).to be_nil
end
it 'updates message totals by delta when a conversation page is retried' do
importer = described_class.new(data_import: data_import)
importer.import_conversations_page
importer.import_conversations_page
expect(data_import.reload.stats.dig('messages', 'total')).to eq(3)
item = data_import.items.find_by!(source_object_type: 'conversation', source_object_id: 'conversation_1')
expect(item.metadata['message_total_contribution']).to eq(3)
end
it 'reconciles imported message stats from same-run mappings on retry' do
described_class.new(data_import: data_import).import_conversations_page
stats = data_import.reload.stats.deep_dup
stats['messages']['imported'] = 0
data_import.update!(stats: stats)
described_class.new(data_import: data_import).import_conversations_page
expect(data_import.reload.stats.dig('messages', 'imported')).to eq(3)
end
it 'indexes imported messages for advanced search' do
allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(true)
allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(false)
reindexed_message_ids = []
original_reindex_for_search = Message.instance_method(:reindex_for_search)
Message.define_method(:reindex_for_search) { reindexed_message_ids << id }
Message.__send__(:private, :reindex_for_search)
described_class.new(data_import: data_import).perform
expect(reindexed_message_ids).to match_array(Message.where(account_id: account.id).pluck(:id))
ensure
Message.define_method(:reindex_for_search, original_reindex_for_search)
Message.__send__(:private, :reindex_for_search)
end
it 'keeps imported messages successful when search reindexing fails', :aggregate_failures do
allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(true)
allow(ChatwootApp).to receive(:chatwoot_cloud?).and_return(false)
# rubocop:disable RSpec/AnyInstance
allow_any_instance_of(Message).to receive(:reindex_for_search).and_raise(StandardError, 'search unavailable')
# rubocop:enable RSpec/AnyInstance
described_class.new(data_import: data_import).perform
message = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:source:source_1')
mapping = data_import.mappings.find_by!(source_object_type: 'message', source_object_id: 'conversation:conversation_1:source:source_1')
expect(mapping.chatwoot_record).to eq(message)
expect(data_import.reload).to be_completed
expect(data_import.import_errors.exists?).to be(false)
expect(data_import.stats.dig('messages', 'imported')).to eq(3)
end
describe '#start!' do
it 'does not overwrite an import abandoned by another process', :aggregate_failures do
importer = described_class.new(data_import: data_import)
DataImport.find(data_import.id).update!(
status: :abandoned,
abandoned_at: Time.current
)
expect(importer.start!).to be_nil
expect(data_import.reload).to be_abandoned
expect(data_import.started_at).to be_nil
end
end
describe '#perform' do
it 'stops when the import was abandoned before processing starts' do
importer = described_class.new(data_import: data_import)
DataImport.find(data_import.id).update!(
status: :abandoned,
abandoned_at: Time.current
)
expect(client).not_to receive(:list_contacts)
importer.perform
expect(data_import.reload).to be_abandoned
end
end
describe '#import_conversations_page' do
it 'stops an in-flight page when a newer import run takes over', :aggregate_failures do
run_id = 'intercom-run-1'
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => run_id })
allow(client).to receive(:list_conversations).with(starting_after: nil).and_return(
'conversations' => [{ 'id' => 'conversation_1' }, { 'id' => 'conversation_2' }],
'pages' => { 'next' => { 'starting_after' => 'next-conversation-cursor' } }
)
allow(client).to receive(:retrieve_conversation).with('conversation_1') do
data_import.update!(source_metadata: { DataImport::ACTIVE_INTERCOM_IMPORT_RUN_ID_KEY => 'new-run' })
conversation_payload
end
result = described_class.new(data_import: data_import, run_id: run_id).import_conversations_page
expect(result).to be_done
expect(client).not_to have_received(:retrieve_conversation).with('conversation_2')
expect(account.conversations.where(identifier: 'intercom:conversation_1')).to be_empty
expect(account.contacts.where(email: 'customer@example.com')).to be_empty
expect(data_import.reload.cursor.dig('conversations', 'starting_after')).to be_nil
end
it 'rolls back a newly inserted conversation when mapping persistence fails', :aggregate_failures do
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
raise StandardError, 'mapping failed' if object_type == 'conversation'
method.call(object_type, source_id, record, metadata: metadata)
end
importer.import_conversations_page
expect(account.conversations.where(identifier: 'intercom:conversation_1')).to be_empty
item = data_import.items.find_by!(source_object_type: 'conversation', source_object_id: 'conversation_1')
expect(item).to be_failed
expect(item.last_error_message).to eq('mapping failed')
end
it 'rolls back a newly inserted contact when mapping persistence fails', :aggregate_failures do
sparse_contact = contact_payload.slice('id', 'name', 'created_at', 'updated_at')
allow(client).to receive(:retrieve_contact).with('contact_1').and_return(sparse_contact)
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
raise StandardError, 'mapping failed' if object_type == 'contact'
method.call(object_type, source_id, record, metadata: metadata)
end
importer.import_conversations_page
expect(account.contacts.where(name: 'Customer One')).to be_empty
expect(data_import.mappings.where(source_object_type: 'contact', source_object_id: 'contact_1')).to be_empty
contact_item = data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(contact_item).to be_failed
expect(contact_item.last_error_message).to eq('mapping failed')
end
it 'rolls back a newly inserted message when mapping persistence fails', :aggregate_failures do
importer = described_class.new(data_import: data_import)
allow(importer).to receive(:record_mapping).and_wrap_original do |method, object_type, source_id, record, metadata:|
raise StandardError, 'mapping failed' if object_type == 'message'
method.call(object_type, source_id, record, metadata: metadata)
end
importer.import_conversations_page
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
expect(conversation.messages.where(source_id: 'intercom:conversation:conversation_1:source:source_1')).to be_empty
error = data_import.import_errors.find_by!(
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:source:source_1'
)
expect(error).to have_attributes(error_code: 'StandardError', message: 'mapping failed')
end
end
describe '#finish!' do
it 'does not overwrite an import abandoned by another process' do
data_import.update!(status: :processing)
importer = described_class.new(data_import: data_import)
DataImport.find(data_import.id).update!(
status: :abandoned,
abandoned_at: Time.current
)
importer.finish!
expect(data_import.reload).to be_abandoned
expect(data_import.completed_at).to be_nil
end
end
describe '#fail!' do
it 'does not overwrite an import abandoned by another process', :aggregate_failures do
data_import.update!(status: :processing)
importer = described_class.new(data_import: data_import)
DataImport.find(data_import.id).update!(
status: :abandoned,
abandoned_at: Time.current
)
importer.fail!(StandardError.new('boom'))
expect(data_import.reload).to be_abandoned
expect(data_import.last_error_at).to be_nil
expect(data_import.import_errors.exists?).to be(false)
end
end
context 'when the Intercom records were imported by an earlier run' do
let(:next_data_import) do
create(
:data_import, :intercom,
account: account
)
end
it 'records the already mapped records as skipped for the current import run', :aggregate_failures do
described_class.new(data_import: data_import).perform
described_class.new(data_import: next_data_import).perform
expect(next_data_import.reload.stats).to include(
'contacts' => include('imported' => 0, 'skipped' => 1, 'total' => 1),
'conversations' => include('imported' => 0, 'skipped' => 1, 'total' => 1),
'messages' => include('imported' => 0, 'skipped' => 3, 'total' => 3),
'errors' => { 'count' => 0 }
)
expect(next_data_import).to be_completed
expect(next_data_import.total_records).to eq(5)
expect(next_data_import.processed_records).to eq(0)
expect(next_data_import.items.skipped.count).to eq(2)
expect(next_data_import.import_errors.skip_logs.group(:source_object_type).count).to eq(
'contact' => 1,
'conversation' => 1,
'message' => 3
)
expect(next_data_import.import_errors.skip_logs.pluck(:details).map { |details| details['reason'] }.uniq).to eq(['already_imported'])
end
it 'recreates messages when existing message mappings point to deleted records', :aggregate_failures do
described_class.new(data_import: data_import).perform
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
Message.where(conversation_id: conversation.id).delete_all
described_class.new(data_import: next_data_import).perform
expect(conversation.reload.messages.pluck(:source_id)).to match_array(
%w[
intercom:conversation:conversation_1:source:source_1
intercom:conversation:conversation_1:part:part_1
intercom:conversation:conversation_1:part:part_2
]
)
expect(next_data_import.reload.stats).to include(
'contacts' => include('imported' => 0, 'skipped' => 1, 'total' => 1),
'conversations' => include('imported' => 0, 'skipped' => 1, 'total' => 1),
'messages' => include('imported' => 3, 'skipped' => 0, 'total' => 3),
'errors' => { 'count' => 0 }
)
expect(next_data_import.import_errors.skip_logs.where(source_object_type: 'message')).to be_empty
message_mappings = DataImportMapping.where(account: account, source_provider: 'intercom', source_object_type: 'message')
expect(message_mappings.filter_map(&:chatwoot_record).count).to eq(3)
end
it 'updates conversation activity when a later import adds new messages to the mapped conversation', :aggregate_failures do
new_part = {
'id' => 'part_3',
'part_type' => 'comment',
'body' => '<p>Follow-up reply</p>',
'created_at' => 1_700_000_300,
'updated_at' => 1_700_000_300,
'author' => { 'type' => 'admin', 'id' => 'admin_1' },
'attachments' => []
}
updated_conversation_payload = conversation_payload.deep_dup
updated_conversation_payload['updated_at'] = 1_700_000_300
updated_conversation_payload['conversation_parts']['conversation_parts'] << new_part
allow(client).to receive(:retrieve_conversation).with('conversation_1').and_return(
conversation_payload,
updated_conversation_payload
)
described_class.new(data_import: data_import).perform
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
described_class.new(data_import: next_data_import).perform
expect(conversation.reload.last_activity_at).to eq(Time.zone.at(1_700_000_300))
expect(conversation.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:part_3').content).to eq('Follow-up reply')
end
end
context 'when a conversation references an already mapped contact' do
it 'reuses the mapped contact without hydrating the sparse reference' do
described_class.new(data_import: data_import).import_contacts_page
expect(client).not_to receive(:retrieve_contact)
described_class.new(data_import: data_import).import_conversations_page
end
end
context 'when a same-run contact mapping outlives its item progress' do
let!(:mapped_contact) { create(:contact, account: account) }
before do
DataImportMapping.create!(
account: account,
data_import: data_import,
source_provider: 'intercom',
source_object_type: 'contact',
source_object_id: 'contact_1',
chatwoot_record_type: 'Contact',
chatwoot_record_id: mapped_contact.id,
metadata: {}
)
data_import.items.create!(
source_provider: 'intercom',
source_object_type: 'contact',
source_object_id: 'contact_1',
status: :processing,
metadata: contact_payload
)
end
it 'repairs the item and imported count on retry', :aggregate_failures do
described_class.new(data_import: data_import).import_contacts_page
item = data_import.items.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(item).to be_imported
expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: mapped_contact.id)
expect(data_import.reload.stats.dig('contacts', 'imported')).to eq(1)
end
end
context 'when an existing contact has the same email but a different external id' do
let(:contact_payload) do
super().merge('last_replied_at' => 1_700_000_090)
end
let!(:existing_contact) { create(:contact, account: account, email: 'customer@example.com', identifier: nil) }
it 'updates the existing contact instead of creating a duplicate', :aggregate_failures do
described_class.new(data_import: data_import).import_contacts_page
expect(existing_contact.reload.identifier).to eq('external_1')
expect(existing_contact.last_activity_at).to eq(Time.zone.at(1_700_000_090))
expect(account.contacts.where(email: 'customer@example.com').count).to eq(1)
item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id)
end
end
context 'when an existing contact has the same phone but a different external id' do
let(:contact_payload) do
super().merge('email' => nil)
end
let!(:existing_contact) { create(:contact, account: account, phone_number: '+15551234567', identifier: nil) }
it 'updates the existing contact instead of creating a duplicate', :aggregate_failures do
described_class.new(data_import: data_import).import_contacts_page
expect(existing_contact.reload.identifier).to eq('external_1')
expect(account.contacts.where(phone_number: '+15551234567').count).to eq(1)
item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id)
end
end
context 'when an existing contact has the same phone but Intercom sends a new email' do
let!(:existing_contact) { create(:contact, account: account, phone_number: '+15551234567', identifier: nil) }
it 'falls through to the phone match after the email lookup misses', :aggregate_failures do
described_class.new(data_import: data_import).import_contacts_page
expect(existing_contact.reload.email).to eq('customer@example.com')
expect(existing_contact.identifier).to eq('external_1')
expect(account.contacts.where(phone_number: '+15551234567').count).to eq(1)
item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id)
end
end
context 'when an existing visitor contact matches the Intercom external id' do
let!(:existing_contact) { create(:contact, account: account, identifier: 'external_1') }
it 'promotes the contact to a lead when adding email or phone', :aggregate_failures do
expect(existing_contact).to be_visitor
described_class.new(data_import: data_import).import_contacts_page
expect(existing_contact.reload).to be_lead
expect(existing_contact.email).to eq('customer@example.com')
expect(existing_contact.phone_number).to eq('+15551234567')
end
end
context 'when an identifier match has contact details owned by another contact' do
let!(:existing_contact) { create(:contact, account: account, identifier: 'external_1') }
let!(:email_owner) { create(:contact, account: account, email: 'customer@example.com') }
let!(:phone_owner) { create(:contact, account: account, phone_number: '+15551234567') }
it 'does not copy the conflicting email or phone number', :aggregate_failures do
described_class.new(data_import: data_import).import_contacts_page
expect(existing_contact.reload.email).to be_nil
expect(existing_contact.phone_number).to be_nil
expect(existing_contact).to be_visitor
expect(email_owner.reload.email).to eq('customer@example.com')
expect(phone_owner.reload.phone_number).to eq('+15551234567')
expect(account.contacts.where(email: 'customer@example.com').count).to eq(1)
expect(account.contacts.where(phone_number: '+15551234567').count).to eq(1)
item = data_import.items.imported.find_by!(source_object_type: 'contact', source_object_id: 'contact_1')
expect(item).to have_attributes(chatwoot_record_type: 'Contact', chatwoot_record_id: existing_contact.id)
end
end
context 'when Intercom rate limits a conversation detail request' do
before do
allow(client).to receive(:retrieve_conversation).with('conversation_1').and_raise(
DataImports::Intercom::Client::RateLimitError.new('rate limited', status: 429)
)
end
it 're-raises the provider error so the page job can retry', :aggregate_failures do
expect { described_class.new(data_import: data_import).import_conversations_page }
.to raise_error(DataImports::Intercom::Client::RateLimitError)
item = data_import.items.find_by!(source_object_type: 'conversation', source_object_id: 'conversation_1')
expect(item).to be_processing
expect(data_import.import_errors.exists?).to be(false)
end
end
context 'when Intercom rate limits a contact hydration request' do
before do
allow(client).to receive(:retrieve_contact).with('contact_1').and_raise(
DataImports::Intercom::Client::RateLimitError.new('rate limited', status: 429)
)
end
it 're-raises the provider error instead of importing a sparse contact', :aggregate_failures do
expect { described_class.new(data_import: data_import).import_conversations_page }
.to raise_error(DataImports::Intercom::Client::RateLimitError)
expect(data_import.items.exists?(source_object_type: 'contact')).to be(false)
expect(data_import.import_errors.exists?).to be(false)
end
end
context 'when Intercom no longer has a sparse contact referenced by a conversation' do
before do
allow(client).to receive(:retrieve_contact).with('contact_1').and_raise(
DataImports::Intercom::Client::Error.new('not found', status: 404)
)
end
it 'falls back to the conversation contact reference', :aggregate_failures do
expect { described_class.new(data_import: data_import).import_conversations_page }.not_to raise_error
expect(data_import.items.imported.exists?(source_object_type: 'contact', source_object_id: 'contact_1')).to be(true)
expect(data_import.import_errors.exists?).to be(false)
end
end
context 'when the Intercom source message only has attachments' do
let(:conversation_payload) do
super().deep_merge(
'source' => {
'subject' => nil,
'body' => nil,
'attachments' => [{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }]
},
'conversation_parts' => {
'conversation_parts' => []
}
)
end
it 'imports the source message attachment placeholder', :aggregate_failures do
described_class.new(data_import: data_import).perform
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
expect(conversation.messages.pluck(:content)).to eq(['[Intercom attachment skipped: 1]'])
expect(conversation.messages.first.additional_attributes.dig('source', 'attachments')).to eq(
[{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }]
)
expect(data_import.reload.stats.dig('messages', 'imported')).to eq(1)
end
end
context 'when the Intercom source message has text and attachments' do
let(:conversation_payload) do
super().deep_merge(
'source' => {
'attachments' => [{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }]
}
)
end
it 'adds an attachment omission marker to the imported message', :aggregate_failures do
described_class.new(data_import: data_import).perform
message = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:source:source_1')
expect(message.content).to eq("Need help\n\nHello there\n\n[Intercom attachment skipped: 1]")
expect(message.additional_attributes.dig('source', 'attachments')).to eq(
[{ 'name' => 'invoice.pdf', 'url' => 'https://example.com/invoice.pdf' }]
)
expect(data_import.reload.stats.dig('messages', 'skipped')).to eq(0)
end
end
context 'when Intercom omits the conversation source' do
let(:conversation_payload) do
super().merge(
'source' => nil,
'first_contact_reply' => {
'type' => 'whatsapp',
'created_at' => 1_700_000_000,
'url' => nil
}
)
end
it 'routes the conversation from the first contact reply type', :aggregate_failures do
described_class.new(data_import: data_import).perform
inbox = account.inboxes.find_by!(name: 'Intercom Import - WhatsApp')
conversation = account.conversations.find_by!(identifier: 'intercom:conversation_1')
expect(conversation.inbox).to eq(inbox)
expect(conversation.additional_attributes.dig('source', 'source_type')).to eq('whatsapp')
end
end
context 'when an Intercom chat message part cannot be imported' do
let(:conversation_payload) do
super().deep_merge(
'conversation_parts' => {
'conversation_parts' => [
{
'id' => 'blank_part',
'part_type' => 'comment',
'body' => nil,
'created_at' => 1_700_000_175,
'updated_at' => 1_700_000_175,
'author' => { 'type' => 'admin', 'id' => 'admin_1' },
'attachments' => []
}
]
}
)
end
it 'records a skip log with the Intercom message source id', :aggregate_failures do
described_class.new(data_import: data_import).perform
skip_log = data_import.import_errors.skip_logs.find_by!(source_object_type: 'message')
expect(skip_log).to have_attributes(
source_object_id: 'conversation:conversation_1:part:blank_part',
error_code: 'DataImports::Intercom::SkippedMessage',
message: 'Skipped Intercom comment event blank_part: no message body or attachments to import.'
)
expect(skip_log.details).to include(
'kind' => 'skipped',
'reason' => 'blank_or_unsupported_intercom_part',
'reason_details' => 'no message body or attachments to import',
'event_name' => 'comment',
'event_type' => 'comment',
'author_type' => 'admin'
)
expect(data_import.reload.stats.dig('messages', 'skipped')).to eq(1)
end
it 'records the skip log again for a later import run', :aggregate_failures do
described_class.new(data_import: data_import).perform
next_data_import = create(
:data_import, :intercom,
account: account
)
described_class.new(data_import: next_data_import).perform
skip_log = next_data_import.import_errors.skip_logs.find_by!(
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:blank_part',
error_code: 'DataImports::Intercom::SkippedMessage'
)
expect(skip_log).to have_attributes(
source_object_id: 'conversation:conversation_1:part:blank_part',
error_code: 'DataImports::Intercom::SkippedMessage'
)
expect(next_data_import.reload.stats.dig('messages', 'skipped')).to eq(2)
end
it 'reconciles a same-run skipped mapping and missing skip log on retry', :aggregate_failures do
described_class.new(data_import: data_import).import_conversations_page
data_import.import_errors.where(source_object_type: 'message').delete_all
stats = data_import.reload.stats.deep_dup
stats['messages']['skipped'] = 0
data_import.update!(stats: stats)
described_class.new(data_import: data_import).import_conversations_page
expect(data_import.reload.stats.dig('messages', 'skipped')).to eq(1)
expect(data_import.import_errors.skip_logs.exists?(source_object_id: 'conversation:conversation_1:part:blank_part')).to be(true)
end
it 'repairs a previously skipped mapping when the part is now an activity', :aggregate_failures do
described_class.new(data_import: data_import).perform
previous_skip_log = data_import.import_errors.skip_logs.find_by!(source_object_id: 'conversation:conversation_1:part:blank_part')
conversation_payload.dig('conversation_parts', 'conversation_parts').first.merge!(
'part_type' => 'assignment',
'assigned_to' => { 'name' => 'Support' }
)
next_data_import = create(:data_import, :intercom, account: account)
described_class.new(data_import: next_data_import).perform
activity = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:blank_part')
mapping = DataImportMapping.find_by!(
account: account,
source_provider: 'intercom',
source_object_type: 'message',
source_object_id: 'conversation:conversation_1:part:blank_part'
)
expect(activity).to be_activity
expect(activity.content).to eq('Intercom teammate assigned the conversation to Support')
expect(mapping.chatwoot_record).to eq(activity)
expect(data_import.import_errors.skip_logs).to include(previous_skip_log)
expect(next_data_import.import_errors.skip_logs.where(source_object_id: mapping.source_object_id)).to be_empty
end
end
context 'when Intercom returns bodyless lifecycle events' do
let(:conversation_payload) do
super().deep_merge(
'conversation_parts' => {
'total_count' => 1,
'conversation_parts' => [
{
'id' => 'assignment_part',
'part_type' => 'assignment',
'body' => nil,
'created_at' => 1_700_000_175,
'author' => { 'type' => 'admin', 'name' => 'Avery' },
'assigned_to' => { 'type' => 'team', 'name' => 'Support' },
'state' => 'open',
'tags' => { 'tags' => [{ 'name' => 'priority' }] },
'event_details' => { 'source' => 'workflow' },
'app_package_code' => 'workflow'
}
]
}
)
end
it 'imports events as public activity messages with source metadata', :aggregate_failures do
described_class.new(data_import: data_import).perform
activity = account.messages.find_by!(source_id: 'intercom:conversation:conversation_1:part:assignment_part')
expect(activity).to have_attributes(
message_type: 'activity',
content: 'Avery assigned the conversation to Support',
private: false,
sender: nil,
created_at: Time.zone.at(1_700_000_175)
)
expect(activity.additional_attributes['source']).to include(
'part_type' => 'assignment',
'assigned_to' => include('name' => 'Support'),
'state' => 'open',
'event_details' => include('source' => 'workflow'),
'app_package_code' => 'workflow'
)
expect(data_import.reload.stats['messages']).to include('imported' => 2, 'skipped' => 0, 'total' => 2)
expect(data_import.import_errors.skip_logs).to be_empty
end
end
context 'when Intercom omits older conversation parts from the retrieved conversation' do
let(:conversation_payload) do
super().deep_merge(
'conversation_parts' => {
'total_count' => 503
},
'statistics' => {
'count_conversation_parts' => 503
}
)
end
it 'records an incomplete import error and completes with errors', :aggregate_failures do
described_class.new(data_import: data_import).perform
error = data_import.import_errors.non_skip_logs.find_by!(
source_object_type: 'conversation',
source_object_id: 'conversation_1',
error_code: 'DataImports::Intercom::TruncatedConversationParts'
)
expect(error.message).to eq('Intercom returned 2 of 503 conversation parts.')
expect(error.details).to include(
'kind' => 'incomplete',
'imported_parts_count' => 2,
'total_parts_count' => 503
)
expect(data_import.reload).to be_completed_with_errors
expect(data_import.stats.dig('errors', 'count')).to eq(1)
end
end
context 'when the conversation parts total matches the returned parts' do
let(:conversation_payload) do
super().deep_merge(
'conversation_parts' => {
'total_count' => 2
},
'statistics' => {
'count_conversation_parts' => 2
}
)
end
it 'does not record a truncated parts error', :aggregate_failures do
described_class.new(data_import: data_import).perform
expect(data_import.import_errors.non_skip_logs).to be_empty
expect(data_import.reload).to be_completed
expect(data_import.stats.dig('errors', 'count')).to eq(0)
end
end
context 'when Intercom statistics count is higher than the conversation parts total' do
let(:conversation_payload) do
super().deep_merge(
'source' => {},
'conversation_parts' => {
'total_count' => 2
},
'statistics' => {
'count_conversation_parts' => 3
}
)
end
it 'trusts the returned conversation parts total over the statistics counter', :aggregate_failures do
described_class.new(data_import: data_import).perform
expect(data_import.import_errors.non_skip_logs).to be_empty
expect(data_import.reload).to be_completed
expect(data_import.stats.dig('errors', 'count')).to eq(0)
end
end
context 'when a specific Intercom message part fails to persist' do
let(:conversation_payload) do
super().deep_merge(
'conversation_parts' => {
'conversation_parts' => [
{
'id' => 'bad_part',
'part_type' => 'comment',
'body' => '<p>Message that cannot be stored</p>',
'created_at' => 1_700_000_175,
'updated_at' => 1_700_000_175,
'author' => { 'type' => 'admin', 'id' => 'admin_1' },
'attachments' => []
}
]
}
)
end
before do
allow(Message).to receive(:insert_all!).and_wrap_original do |method, records, **kwargs|
raise ActiveRecord::StatementInvalid, 'bad message' if records.first[:source_id] == 'intercom:conversation:conversation_1:part:bad_part'
method.call(records, **kwargs)
end
end
it 'records a skip log with the Intercom message part id', :aggregate_failures do
described_class.new(data_import: data_import).perform
skip_log = data_import.import_errors.skip_logs.find_by!(source_object_type: 'message')
expect(skip_log).to have_attributes(
source_object_id: 'conversation:conversation_1:part:bad_part',
error_code: 'ActiveRecord::StatementInvalid',
message: 'bad message'
)
expect(skip_log.details).to include(
'kind' => 'failed',
'conversation_id' => 'intercom:conversation_1'
)
expect(data_import.reload).to be_completed_with_errors
expect(data_import.stats.dig('errors', 'count')).to eq(1)
end
end
end
@@ -0,0 +1,33 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::PlaceholderInboxBuilder do
let(:account) { create(:account) }
describe '#inbox_for' do
it 'creates a source-bucket API inbox for an Intercom conversation source' do
inbox = described_class.new(account: account).inbox_for('email')
expect(inbox.name).to eq('Intercom Import - Email')
expect(inbox.channel).to be_a(Channel::Api)
expect(inbox.enable_auto_assignment).to be(false)
expect(inbox.allow_messages_after_resolved).to be(false)
expect(inbox.channel.additional_attributes).to include(
'source_provider' => 'intercom',
'source_bucket' => 'email',
'import_placeholder' => true,
'agent_reply_time_window' => 1
)
end
it 'reuses an existing placeholder inbox for the same source bucket' do
builder = described_class.new(account: account)
first_inbox = builder.inbox_for('phone_call')
expect(account).not_to receive(:inboxes)
second_inbox = builder.inbox_for('phone_switch')
expect(second_inbox).to eq(first_inbox)
expect(Inbox.where(account: account, channel_type: 'Channel::Api').count).to eq(1)
end
end
end
@@ -0,0 +1,64 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::RestartService do
let(:account) { create(:account) }
let(:data_import) { create(:data_import, :intercom, account: account, status: :abandoned, abandoned_at: 1.hour.ago) }
it 'prepares a failed or abandoned import for another run', :aggregate_failures do
data_import.update!(
stats: {
'contacts' => { 'imported' => 1, 'skipped' => 9, 'total' => 10 },
'conversations' => { 'imported' => 2, 'skipped' => 8, 'total' => 10 },
'messages' => { 'imported' => 3, 'skipped' => 7, 'total' => 10 },
'errors' => { 'count' => 6 }
}
)
data_import.import_errors.create!(error_code: 'StandardError', message: 'old run error')
data_import.import_errors.create!(
error_code: 'ContactFailed',
message: 'old contact error',
source_object_type: 'contact',
details: { kind: 'failed' }
)
retained_skip_log = data_import.import_errors.create!(
error_code: DataImports::Intercom::Importer::ALREADY_IMPORTED_ERROR_CODE,
message: 'old skip log',
source_object_type: 'contact',
details: { kind: 'skipped' }
)
previous_run_id = data_import.assign_active_intercom_import_run_id
data_import.save!
service = described_class.new(account: account, data_import: data_import)
expect(service.perform).to eq(:enqueue)
expect(service.data_import).to be_pending
expect(service.data_import.abandoned_at).to be_nil
expect(service.data_import.started_at).to be_nil
expect(service.data_import.active_intercom_import_run_id).not_to eq(previous_run_id)
expect(service.data_import.import_errors).to contain_exactly(retained_skip_log)
expect(service.data_import.stats).to eq(
'contacts' => { 'imported' => 1, 'skipped' => 1, 'total' => 10 },
'conversations' => { 'imported' => 2, 'skipped' => 0, 'total' => 10 },
'messages' => { 'imported' => 3, 'skipped' => 0, 'total' => 10 },
'errors' => { 'count' => 0 }
)
end
it 'returns the active import instead of restarting another import', :aggregate_failures do
active_import = create(:data_import, :intercom, account: account, status: :processing)
service = described_class.new(account: account, data_import: data_import)
expect(service.perform).to eq(:render_show)
expect(service.data_import).to eq(active_import)
expect(data_import.reload).to be_abandoned
end
it 'does not restart when the stored access token is missing' do
data_import.update!(access_token: nil)
result = described_class.new(account: account, data_import: data_import).perform
expect(result).to eq(:access_token_missing)
expect(data_import.reload).to be_abandoned
end
end
@@ -0,0 +1,17 @@
require 'rails_helper'
RSpec.describe DataImports::Intercom::SourceBucket do
describe '.for' do
it 'maps Intercom source types to Chatwoot inbox buckets' do
expect(described_class.for('email')).to eq({ key: 'email', name: 'Email' })
expect(described_class.for('phone_switch')).to eq({ key: 'phone', name: 'Phone' })
expect(described_class.for('inapp')).to eq({ key: 'messenger', name: 'Messenger' })
expect(described_class.for('messenger')).to eq({ key: 'messenger', name: 'Messenger' })
expect(described_class.for('push')).to eq({ key: 'messenger', name: 'Messenger' })
end
it 'uses an unknown bucket for unsupported source types' do
expect(described_class.for('unsupported_source')).to eq({ key: 'unknown', name: 'Unknown' })
end
end
end