Merge remote-tracking branch 'origin/develop' into assignment_v2/assignment_service
This commit is contained in:
Tanmay Sharma
@@ -62,6 +62,10 @@ gem 'redis-namespace'
|
||||
# super fast record imports in bulk
|
||||
gem 'activerecord-import'
|
||||
|
||||
gem 'searchkick'
|
||||
gem 'opensearch-ruby'
|
||||
gem 'faraday_middleware-aws-sigv4'
|
||||
|
||||
##--- gems for server & infra configuration ---##
|
||||
gem 'dotenv-rails', '>= 3.0.0'
|
||||
gem 'foreman'
|
||||
|
||||
@@ -299,6 +299,9 @@ GEM
|
||||
net-http-persistent (~> 4.0)
|
||||
faraday-retry (2.2.1)
|
||||
faraday (~> 2.0)
|
||||
faraday_middleware-aws-sigv4 (1.0.1)
|
||||
aws-sigv4 (~> 1.0)
|
||||
faraday (>= 2.0, < 3)
|
||||
fast-mcp (1.5.0)
|
||||
addressable (~> 2.8)
|
||||
base64
|
||||
@@ -601,6 +604,9 @@ GEM
|
||||
omniauth-rails_csrf_protection (1.0.2)
|
||||
actionpack (>= 4.2)
|
||||
omniauth (~> 2.0)
|
||||
opensearch-ruby (3.4.0)
|
||||
faraday (>= 1.0, < 3)
|
||||
multi_json (>= 1.0)
|
||||
openssl (3.2.0)
|
||||
orm_adapter (0.5.0)
|
||||
os (1.1.4)
|
||||
@@ -802,6 +808,9 @@ GEM
|
||||
parser
|
||||
scss_lint (0.60.0)
|
||||
sass (~> 3.5, >= 3.5.5)
|
||||
searchkick (5.5.2)
|
||||
activemodel (>= 7.1)
|
||||
hashie
|
||||
securerandom (0.4.1)
|
||||
seed_dump (3.3.1)
|
||||
activerecord (>= 4)
|
||||
@@ -996,6 +1005,7 @@ DEPENDENCIES
|
||||
facebook-messenger
|
||||
factory_bot_rails (>= 6.4.3)
|
||||
faker
|
||||
faraday_middleware-aws-sigv4
|
||||
fcm
|
||||
flag_shih_tzu
|
||||
foreman
|
||||
@@ -1036,6 +1046,7 @@ DEPENDENCIES
|
||||
omniauth-google-oauth2 (>= 1.1.3)
|
||||
omniauth-oauth2
|
||||
omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2)
|
||||
opensearch-ruby
|
||||
pg
|
||||
pg_search
|
||||
pgvector
|
||||
@@ -1064,6 +1075,7 @@ DEPENDENCIES
|
||||
ruby_llm-schema
|
||||
scout_apm
|
||||
scss_lint
|
||||
searchkick
|
||||
seed_dump
|
||||
sentry-rails (>= 5.19.0)
|
||||
sentry-ruby
|
||||
|
||||
@@ -4,3 +4,7 @@
|
||||
require_relative 'config/application'
|
||||
|
||||
Rails.application.load_tasks
|
||||
|
||||
# Load Enterprise Edition rake tasks if they exist
|
||||
enterprise_tasks_path = Rails.root.join('enterprise/lib/tasks.rb').to_s
|
||||
require enterprise_tasks_path if File.exist?(enterprise_tasks_path)
|
||||
|
||||
@@ -1,25 +1,49 @@
|
||||
<script setup>
|
||||
import { ref } from 'vue';
|
||||
import { ref, nextTick, onMounted } from 'vue';
|
||||
|
||||
const emit = defineEmits(['send']);
|
||||
const message = ref('');
|
||||
const textareaRef = ref(null);
|
||||
|
||||
const adjustHeight = () => {
|
||||
if (!textareaRef.value) return;
|
||||
|
||||
// Reset height to auto to get the correct scrollHeight
|
||||
textareaRef.value.style.height = 'auto';
|
||||
// Set the height to the scrollHeight
|
||||
textareaRef.value.style.height = `${textareaRef.value.scrollHeight}px`;
|
||||
};
|
||||
|
||||
const sendMessage = () => {
|
||||
if (message.value.trim()) {
|
||||
emit('send', message.value);
|
||||
message.value = '';
|
||||
// Reset textarea height after sending
|
||||
nextTick(() => {
|
||||
adjustHeight();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleInput = () => {
|
||||
nextTick(adjustHeight);
|
||||
};
|
||||
|
||||
onMounted(() => {
|
||||
nextTick(adjustHeight);
|
||||
});
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<form class="relative" @submit.prevent="sendMessage">
|
||||
<input
|
||||
<textarea
|
||||
ref="textareaRef"
|
||||
v-model="message"
|
||||
type="text"
|
||||
:placeholder="$t('CAPTAIN.COPILOT.SEND_MESSAGE')"
|
||||
class="w-full reset-base bg-n-alpha-3 ltr:pl-4 ltr:pr-12 rtl:pl-12 rtl:pr-4 py-3 text-n-slate-11 text-sm border border-n-weak rounded-lg focus:outline-none focus:ring-1 focus:ring-n-blue-11 focus:border-n-blue-11"
|
||||
@keyup.enter="sendMessage"
|
||||
class="w-full reset-base bg-n-alpha-3 ltr:pl-4 ltr:pr-12 rtl:pl-12 rtl:pr-4 py-3 text-sm border border-n-weak rounded-lg focus:outline-0 focus:outline-none focus:ring-2 focus:ring-n-blue-11 focus:border-n-blue-11 resize-none overflow-hidden max-h-[200px] mb-0 text-n-slate-12"
|
||||
rows="1"
|
||||
@input="handleInput"
|
||||
@keydown.enter.exact.prevent="sendMessage"
|
||||
/>
|
||||
<button
|
||||
class="absolute ltr:right-1 rtl:left-1 top-1/2 -translate-y-1/2 h-9 w-10 flex items-center justify-center text-n-slate-11 hover:text-n-blue-11"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script setup>
|
||||
import { ref, computed, onMounted } from 'vue';
|
||||
import { useAlert } from 'dashboard/composables';
|
||||
import { useStore } from 'dashboard/composables/store';
|
||||
import Copilot from 'dashboard/components-next/copilot/Copilot.vue';
|
||||
import { useMapGetter } from 'dashboard/composables/store';
|
||||
@@ -100,20 +101,24 @@ const handleReset = () => {
|
||||
};
|
||||
|
||||
const sendMessage = async message => {
|
||||
if (selectedCopilotThreadId.value) {
|
||||
await store.dispatch('copilotMessages/create', {
|
||||
assistant_id: activeAssistant.value.id,
|
||||
conversation_id: currentChat.value?.id,
|
||||
threadId: selectedCopilotThreadId.value,
|
||||
message,
|
||||
});
|
||||
} else {
|
||||
const response = await store.dispatch('copilotThreads/create', {
|
||||
assistant_id: activeAssistant.value.id,
|
||||
conversation_id: currentChat.value?.id,
|
||||
message,
|
||||
});
|
||||
selectedCopilotThreadId.value = response.id;
|
||||
try {
|
||||
if (selectedCopilotThreadId.value) {
|
||||
await store.dispatch('copilotMessages/create', {
|
||||
assistant_id: activeAssistant.value.id,
|
||||
conversation_id: currentChat.value?.id,
|
||||
threadId: selectedCopilotThreadId.value,
|
||||
message,
|
||||
});
|
||||
} else {
|
||||
const response = await store.dispatch('copilotThreads/create', {
|
||||
assistant_id: activeAssistant.value.id,
|
||||
conversation_id: currentChat.value?.id,
|
||||
message,
|
||||
});
|
||||
selectedCopilotThreadId.value = response.id;
|
||||
}
|
||||
} catch (error) {
|
||||
useAlert(error.message);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup>
|
||||
import { ref, useTemplateRef, onMounted, watch, nextTick } from 'vue';
|
||||
import { ref, useTemplateRef, onMounted, watch, nextTick, computed } from 'vue';
|
||||
import { useMessageFormatter } from 'shared/composables/useMessageFormatter';
|
||||
import ReadMore from './ReadMore.vue';
|
||||
|
||||
@@ -8,9 +8,9 @@ const props = defineProps({
|
||||
type: String,
|
||||
default: '',
|
||||
},
|
||||
content: {
|
||||
type: String,
|
||||
default: '',
|
||||
message: {
|
||||
type: Object,
|
||||
default: () => ({}),
|
||||
},
|
||||
searchTerm: {
|
||||
type: String,
|
||||
@@ -18,6 +18,24 @@ const props = defineProps({
|
||||
},
|
||||
});
|
||||
|
||||
const messageContent = computed(() => {
|
||||
// We perform search on either content or email subject or transcribed text
|
||||
if (props.message.content) {
|
||||
return props.message.content;
|
||||
}
|
||||
|
||||
const { content_attributes = {} } = props.message;
|
||||
const { email = {} } = content_attributes || {};
|
||||
if (email.subject) {
|
||||
return email.subject;
|
||||
}
|
||||
|
||||
const audioAttachment = props.message.attachments.find(
|
||||
attachment => attachment.file_type === 'audio'
|
||||
);
|
||||
return audioAttachment?.transcribed_text || '';
|
||||
});
|
||||
|
||||
const { highlightContent } = useMessageFormatter();
|
||||
|
||||
const messageContainer = useTemplateRef('messageContainer');
|
||||
@@ -38,7 +56,8 @@ const escapeHtml = html => {
|
||||
return p.innerText;
|
||||
};
|
||||
|
||||
const prepareContent = (content = '') => {
|
||||
const prepareContent = () => {
|
||||
const content = messageContent.value || '';
|
||||
const escapedText = escapeHtml(content);
|
||||
return highlightContent(
|
||||
escapedText,
|
||||
@@ -65,7 +84,7 @@ onMounted(() => {
|
||||
{{ $t('SEARCH.WROTE') }}
|
||||
</p>
|
||||
<ReadMore :shrink="isOverflowing" @expand="isOverflowing = false">
|
||||
<div v-dompurify-html="prepareContent(content)" class="message-content" />
|
||||
<div v-dompurify-html="prepareContent()" class="message-content" />
|
||||
</ReadMore>
|
||||
</blockquote>
|
||||
</template>
|
||||
@@ -74,6 +93,7 @@ onMounted(() => {
|
||||
.message {
|
||||
@apply py-0 px-2 mt-2;
|
||||
}
|
||||
|
||||
.message-content::v-deep p,
|
||||
.message-content::v-deep li::marker {
|
||||
@apply text-n-slate-11 mb-1;
|
||||
|
||||
@@ -54,7 +54,7 @@ const getName = message => {
|
||||
>
|
||||
<MessageContent
|
||||
:author="getName(message)"
|
||||
:content="message.content"
|
||||
:message="message"
|
||||
:search-term="query"
|
||||
/>
|
||||
</SearchResultConversationItem>
|
||||
|
||||
@@ -131,7 +131,13 @@ async function generateQRCodes() {
|
||||
|
||||
// WhatsApp (both Cloud and Twilio)
|
||||
if (currentInbox.value.phone_number && isAWhatsAppChannel.value) {
|
||||
await generateQRCode('whatsapp', currentInbox.value.phone_number);
|
||||
// For Twilio WhatsApp, phone_number format is "whatsapp:+1234567890"
|
||||
// Extract just the phone number part for QR code generation
|
||||
const phoneNumber = currentInbox.value.phone_number.replace(
|
||||
'whatsapp:',
|
||||
''
|
||||
);
|
||||
await generateQRCode('whatsapp', phoneNumber);
|
||||
}
|
||||
|
||||
// Facebook Messenger
|
||||
@@ -230,47 +236,47 @@ onMounted(() => {
|
||||
</div>
|
||||
<div
|
||||
v-if="isAWhatsAppChannel && qrCodes.whatsapp"
|
||||
class="flex flex-col items-center mt-8 gap-3"
|
||||
class="flex flex-col gap-3 items-center mt-8"
|
||||
>
|
||||
<p class="mt-2 text-sm text-n-slate-9">
|
||||
{{ $t('INBOX_MGMT.FINISH.WHATSAPP_QR_INSTRUCTION') }}
|
||||
</p>
|
||||
<div class="outline-1 outline-n-strong outline rounded-lg shadow">
|
||||
<div class="rounded-lg shadow outline-1 outline-n-strong outline">
|
||||
<img
|
||||
:src="qrCodes.whatsapp"
|
||||
alt="WhatsApp QR Code"
|
||||
class="size-48 dark:invert rounded-lg"
|
||||
class="rounded-lg size-48 dark:invert"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
v-if="isAFacebookInbox && qrCodes.messenger"
|
||||
class="flex flex-col items-center mt-8 gap-3"
|
||||
class="flex flex-col gap-3 items-center mt-8"
|
||||
>
|
||||
<p class="mt-2 text-sm text-n-slate-9">
|
||||
{{ $t('INBOX_MGMT.FINISH.MESSENGER_QR_INSTRUCTION') }}
|
||||
</p>
|
||||
<div class="outline-1 outline-n-strong outline rounded-lg shadow">
|
||||
<div class="rounded-lg shadow outline-1 outline-n-strong outline">
|
||||
<img
|
||||
:src="qrCodes.messenger"
|
||||
alt="Messenger QR Code"
|
||||
class="size-48 dark:invert rounded-lg"
|
||||
class="rounded-lg size-48 dark:invert"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
v-if="isATelegramChannel && qrCodes.telegram"
|
||||
class="flex flex-col items-center mt-8 gap-4"
|
||||
class="flex flex-col gap-4 items-center mt-8"
|
||||
>
|
||||
<p class="mt-2 text-sm text-n-slate-9">
|
||||
{{ $t('INBOX_MGMT.FINISH.TELEGRAM_QR_INSTRUCTION') }}
|
||||
</p>
|
||||
|
||||
<div class="outline-1 outline-n-strong outline rounded-lg shadow">
|
||||
<div class="rounded-lg shadow outline-1 outline-n-strong outline">
|
||||
<img
|
||||
:src="qrCodes.telegram"
|
||||
alt="Telegram QR Code"
|
||||
class="size-48 dark:invert rounded-lg"
|
||||
class="rounded-lg size-48 dark:invert"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+21
-2
@@ -39,6 +39,8 @@
|
||||
#
|
||||
|
||||
class Message < ApplicationRecord
|
||||
searchkick callbacks: :async if ChatwootApp.advanced_search_allowed?
|
||||
|
||||
include MessageFilterHelpers
|
||||
include Liquidable
|
||||
NUMBER_OF_PERMITTED_ATTACHMENTS = 15
|
||||
@@ -139,14 +141,23 @@ class Message < ApplicationRecord
|
||||
data = attributes.symbolize_keys.merge(
|
||||
created_at: created_at.to_i,
|
||||
message_type: message_type_before_type_cast,
|
||||
conversation_id: conversation.display_id,
|
||||
conversation: conversation_push_event_data
|
||||
conversation_id: conversation&.display_id,
|
||||
conversation: conversation.present? ? conversation_push_event_data : nil
|
||||
)
|
||||
data[:echo_id] = echo_id if echo_id.present?
|
||||
data[:attachments] = attachments.map(&:push_event_data) if attachments.present?
|
||||
merge_sender_attributes(data)
|
||||
end
|
||||
|
||||
def search_data
|
||||
data = attributes.symbolize_keys
|
||||
data[:conversation] = conversation.present? ? conversation_push_event_data : nil
|
||||
data[:attachments] = attachments.map(&:push_event_data) if attachments.present?
|
||||
data[:sender] = sender.push_event_data if sender
|
||||
data[:inbox] = inbox
|
||||
data
|
||||
end
|
||||
|
||||
def conversation_push_event_data
|
||||
{
|
||||
assignee_id: conversation.assignee_id,
|
||||
@@ -228,6 +239,14 @@ class Message < ApplicationRecord
|
||||
previous_changes: previous_changes)
|
||||
end
|
||||
|
||||
def should_index?
|
||||
return false unless ChatwootApp.advanced_search_allowed?
|
||||
return false unless account.feature_enabled?('advanced_search')
|
||||
return false unless incoming? || outgoing?
|
||||
|
||||
true
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def prevent_message_flooding
|
||||
|
||||
@@ -43,11 +43,19 @@ class SearchService
|
||||
def filter_messages
|
||||
@messages = if use_gin_search
|
||||
filter_messages_with_gin
|
||||
elsif should_run_advanced_search?
|
||||
advanced_search
|
||||
else
|
||||
filter_messages_with_like
|
||||
end
|
||||
end
|
||||
|
||||
def should_run_advanced_search?
|
||||
ChatwootApp.advanced_search_allowed? && current_account.feature_enabled?('advanced_search')
|
||||
end
|
||||
|
||||
def advanced_search; end
|
||||
|
||||
def filter_messages_with_gin
|
||||
base_query = message_base_query
|
||||
|
||||
@@ -115,3 +123,5 @@ class SearchService
|
||||
.per(15)
|
||||
end
|
||||
end
|
||||
|
||||
SearchService.prepend_mod_with('SearchService')
|
||||
|
||||
@@ -1,12 +1 @@
|
||||
json.id message.id
|
||||
json.content message.content
|
||||
json.message_type message.message_type_before_type_cast
|
||||
json.content_type message.content_type
|
||||
json.source_id message.source_id
|
||||
json.inbox_id message.inbox_id
|
||||
json.conversation_id message.conversation.try(:display_id)
|
||||
json.created_at message.created_at.to_i
|
||||
json.sender message.sender.push_event_data if message.sender
|
||||
json.inbox do
|
||||
json.partial! 'inbox', formats: [:json], inbox: message.inbox if message.inbox.present? && message.try(:inbox).present?
|
||||
end
|
||||
json.partial! 'api/v1/models/message', message: message
|
||||
|
||||
+6
-1
@@ -194,7 +194,12 @@
|
||||
- name: twilio_content_templates
|
||||
display_name: Twilio Content Templates
|
||||
enabled: false
|
||||
- name: advanced_search
|
||||
display_name: Advanced Search
|
||||
enabled: false
|
||||
premium: true
|
||||
chatwoot_internal: true
|
||||
- name: assignment_v2
|
||||
display_name: Assignment V2
|
||||
enabled: false
|
||||
chatwoot_internal: false
|
||||
chatwoot_internal: true
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
Searchkick.queue_name = :async_database_migration if ENV.fetch('OPENSEARCH_URL', '').present?
|
||||
|
||||
access_key_id = ENV.fetch('OPENSEARCH_AWS_ACCESS_KEY_ID', '')
|
||||
secret_access_key = ENV.fetch('OPENSEARCH_AWS_SECRET_ACCESS_KEY', '')
|
||||
|
||||
if access_key_id.present? && secret_access_key.present?
|
||||
region = ENV.fetch('OPENSEARCH_AWS_REGION', 'us-east-1')
|
||||
|
||||
Searchkick.aws_credentials = {
|
||||
access_key_id: access_key_id,
|
||||
secret_access_key: secret_access_key,
|
||||
region: region
|
||||
}
|
||||
end
|
||||
@@ -292,6 +292,26 @@ en:
|
||||
completed_tool_call: 'Completed %{function_name} tool call'
|
||||
invalid_tool_call: 'Invalid tool call'
|
||||
tool_not_available: 'Tool not available'
|
||||
documents:
|
||||
limit_exceeded: 'Document limit exceeded'
|
||||
pdf_format_error: 'must be a PDF file'
|
||||
pdf_size_error: 'must be less than 10MB'
|
||||
pdf_upload_failed: 'Failed to upload PDF to OpenAI'
|
||||
pdf_upload_success: 'PDF uploaded successfully with file_id: %{file_id}'
|
||||
pdf_processing_failed: 'Failed to process PDF document %{document_id}: %{error}'
|
||||
pdf_processing_success: 'Successfully processed PDF document %{document_id}'
|
||||
faq_generation_complete: 'FAQ generation complete. Total FAQs created: %{count}'
|
||||
using_paginated_faq: 'Using paginated FAQ generation for document %{document_id}'
|
||||
using_standard_faq: 'Using standard FAQ generation for document %{document_id}'
|
||||
response_creation_error: 'Error in creating response document: %{error}'
|
||||
missing_openai_file_id: 'Document must have openai_file_id for paginated processing'
|
||||
openai_api_error: 'OpenAI API Error: %{error}'
|
||||
starting_paginated_faq: 'Starting paginated FAQ generation (%{pages_per_chunk} pages per chunk)'
|
||||
stopping_faq_generation: 'Stopping processing. Reason: %{reason}'
|
||||
paginated_faq_complete: 'Paginated generation complete. Total FAQs: %{total_faqs}, Pages processed: %{pages_processed}'
|
||||
processing_pages: 'Processing pages %{start}-%{end} (iteration %{iteration})'
|
||||
chunk_generated: 'Chunk generated %{chunk_faqs} FAQs. Total so far: %{total_faqs}'
|
||||
page_processing_error: 'Error processing pages %{start}-%{end}: %{error}'
|
||||
public_portal:
|
||||
search:
|
||||
search_placeholder: Search for article by title or body...
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
class AddMetadataToCaptainDocuments < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
add_column :captain_documents, :metadata, :jsonb, default: {}
|
||||
end
|
||||
end
|
||||
@@ -320,6 +320,7 @@ ActiveRecord::Schema[7.1].define(version: 2025_08_22_061042) do
|
||||
t.datetime "created_at", null: false
|
||||
t.datetime "updated_at", null: false
|
||||
t.integer "status", default: 0, null: false
|
||||
t.jsonb "metadata", default: {}
|
||||
t.index ["account_id"], name: "index_captain_documents_on_account_id"
|
||||
t.index ["assistant_id", "external_link"], name: "index_captain_documents_on_assistant_id_and_external_link", unique: true
|
||||
t.index ["assistant_id"], name: "index_captain_documents_on_assistant_id"
|
||||
|
||||
@@ -25,6 +25,8 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
@document.save!
|
||||
rescue Captain::Document::LimitExceededError => e
|
||||
render_could_not_create_error(e.message)
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
render_could_not_create_error(e.record.errors.full_messages.join(', '))
|
||||
end
|
||||
|
||||
def destroy
|
||||
@@ -55,6 +57,6 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
end
|
||||
|
||||
def document_params
|
||||
params.require(:document).permit(:name, :external_link, :assistant_id)
|
||||
params.require(:document).permit(:name, :external_link, :assistant_id, :pdf_file)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -2,7 +2,9 @@ class Captain::Documents::CrawlJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(document)
|
||||
if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
|
||||
if document.pdf_document?
|
||||
perform_pdf_processing(document)
|
||||
elsif InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
|
||||
perform_firecrawl_crawl(document)
|
||||
else
|
||||
perform_simple_crawl(document)
|
||||
@@ -13,6 +15,14 @@ class Captain::Documents::CrawlJob < ApplicationJob
|
||||
|
||||
include Captain::FirecrawlHelper
|
||||
|
||||
def perform_pdf_processing(document)
|
||||
Captain::Llm::PdfProcessingService.new(document).process
|
||||
document.update!(status: :available)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error I18n.t('captain.documents.pdf_processing_failed', document_id: document.id, error: e.message)
|
||||
raise # Re-raise to let job framework handle retry logic
|
||||
end
|
||||
|
||||
def perform_simple_crawl(document)
|
||||
page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links
|
||||
|
||||
|
||||
@@ -1,17 +1,65 @@
|
||||
class Captain::Documents::ResponseBuilderJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(document)
|
||||
def perform(document, options = {})
|
||||
reset_previous_responses(document)
|
||||
|
||||
faqs = Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name).generate
|
||||
faqs.each do |faq|
|
||||
create_response(faq, document)
|
||||
end
|
||||
faqs = generate_faqs(document, options)
|
||||
create_responses_from_faqs(faqs, document)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_faqs(document, options)
|
||||
if should_use_pagination?(document)
|
||||
generate_paginated_faqs(document, options)
|
||||
else
|
||||
generate_standard_faqs(document)
|
||||
end
|
||||
end
|
||||
|
||||
def generate_paginated_faqs(document, options)
|
||||
service = build_paginated_service(document, options)
|
||||
faqs = service.generate
|
||||
store_paginated_metadata(document, service)
|
||||
faqs
|
||||
end
|
||||
|
||||
def generate_standard_faqs(document)
|
||||
Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name).generate
|
||||
end
|
||||
|
||||
def build_paginated_service(document, options)
|
||||
Captain::Llm::PaginatedFaqGeneratorService.new(
|
||||
document,
|
||||
pages_per_chunk: options[:pages_per_chunk],
|
||||
max_pages: options[:max_pages]
|
||||
)
|
||||
end
|
||||
|
||||
def store_paginated_metadata(document, service)
|
||||
document.update!(
|
||||
metadata: (document.metadata || {}).merge(
|
||||
'faq_generation' => {
|
||||
'method' => 'paginated',
|
||||
'pages_processed' => service.total_pages_processed,
|
||||
'iterations' => service.iterations_completed,
|
||||
'timestamp' => Time.current.iso8601
|
||||
}
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
def create_responses_from_faqs(faqs, document)
|
||||
faqs.each { |faq| create_response(faq, document) }
|
||||
end
|
||||
|
||||
def should_use_pagination?(document)
|
||||
# Auto-detect when to use pagination
|
||||
# For now, use pagination for PDFs with OpenAI file ID
|
||||
document.pdf_document? && document.openai_file_id.present?
|
||||
end
|
||||
|
||||
def reset_previous_responses(response_document)
|
||||
response_document.responses.destroy_all
|
||||
end
|
||||
@@ -24,6 +72,6 @@ class Captain::Documents::ResponseBuilderJob < ApplicationJob
|
||||
documentable: document
|
||||
)
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
Rails.logger.error "Error in creating response document: #{e.message}"
|
||||
Rails.logger.error I18n.t('captain.documents.response_creation_error', error: e.message)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
# id :bigint not null, primary key
|
||||
# content :text
|
||||
# external_link :string not null
|
||||
# metadata :jsonb
|
||||
# name :string
|
||||
# status :integer default("in_progress"), not null
|
||||
# created_at :datetime not null
|
||||
@@ -26,11 +27,16 @@ class Captain::Document < ApplicationRecord
|
||||
belongs_to :assistant, class_name: 'Captain::Assistant'
|
||||
has_many :responses, class_name: 'Captain::AssistantResponse', dependent: :destroy, as: :documentable
|
||||
belongs_to :account
|
||||
has_one_attached :pdf_file
|
||||
|
||||
validates :external_link, presence: true
|
||||
validates :external_link, uniqueness: { scope: :assistant_id }
|
||||
validates :external_link, presence: true, unless: -> { pdf_file.attached? }
|
||||
validates :external_link, uniqueness: { scope: :assistant_id }, allow_blank: true
|
||||
validates :content, length: { maximum: 200_000 }
|
||||
validates :pdf_file, presence: true, if: :pdf_document?
|
||||
validate :validate_pdf_format, if: :pdf_document?
|
||||
validate :validate_file_attachment, if: -> { pdf_file.attached? }
|
||||
before_validation :ensure_account_id
|
||||
before_validation :set_external_link_for_pdf
|
||||
|
||||
enum status: {
|
||||
in_progress: 0,
|
||||
@@ -41,12 +47,44 @@ class Captain::Document < ApplicationRecord
|
||||
after_create_commit :enqueue_crawl_job
|
||||
after_create_commit :update_document_usage
|
||||
after_destroy :update_document_usage
|
||||
after_commit :enqueue_response_builder_job
|
||||
after_commit :enqueue_response_builder_job, on: :update, if: :should_enqueue_response_builder?
|
||||
scope :ordered, -> { order(created_at: :desc) }
|
||||
|
||||
scope :for_account, ->(account_id) { where(account_id: account_id) }
|
||||
scope :for_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
|
||||
|
||||
def pdf_document?
|
||||
return true if pdf_file.attached? && pdf_file.blob.content_type == 'application/pdf'
|
||||
|
||||
external_link&.ends_with?('.pdf')
|
||||
end
|
||||
|
||||
def content_type
|
||||
pdf_file.blob.content_type if pdf_file.attached?
|
||||
end
|
||||
|
||||
def file_size
|
||||
pdf_file.blob.byte_size if pdf_file.attached?
|
||||
end
|
||||
|
||||
def openai_file_id
|
||||
metadata&.dig('openai_file_id')
|
||||
end
|
||||
|
||||
def store_openai_file_id(file_id)
|
||||
update!(metadata: (metadata || {}).merge('openai_file_id' => file_id))
|
||||
end
|
||||
|
||||
def display_url
|
||||
return external_link if external_link.present? && !external_link.start_with?('PDF:')
|
||||
|
||||
if pdf_file.attached?
|
||||
Rails.application.routes.url_helpers.rails_blob_url(pdf_file, only_path: false)
|
||||
else
|
||||
external_link
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def enqueue_crawl_job
|
||||
@@ -61,6 +99,12 @@ class Captain::Document < ApplicationRecord
|
||||
Captain::Documents::ResponseBuilderJob.perform_later(self)
|
||||
end
|
||||
|
||||
def should_enqueue_response_builder?
|
||||
# Only enqueue when status changes to available
|
||||
# Avoid re-enqueueing when metadata is updated by the job itself
|
||||
saved_change_to_status? && status == 'available'
|
||||
end
|
||||
|
||||
def update_document_usage
|
||||
account.update_document_usage
|
||||
end
|
||||
@@ -71,6 +115,29 @@ class Captain::Document < ApplicationRecord
|
||||
|
||||
def ensure_within_plan_limit
|
||||
limits = account.usage_limits[:captain][:documents]
|
||||
raise LimitExceededError, 'Document limit exceeded' unless limits[:current_available].positive?
|
||||
raise LimitExceededError, I18n.t('captain.documents.limit_exceeded') unless limits[:current_available].positive?
|
||||
end
|
||||
|
||||
def validate_pdf_format
|
||||
return unless pdf_file.attached?
|
||||
|
||||
errors.add(:pdf_file, I18n.t('captain.documents.pdf_format_error')) unless pdf_file.blob.content_type == 'application/pdf'
|
||||
end
|
||||
|
||||
def validate_file_attachment
|
||||
return unless pdf_file.attached?
|
||||
|
||||
return unless pdf_file.blob.byte_size > 10.megabytes
|
||||
|
||||
errors.add(:pdf_file, I18n.t('captain.documents.pdf_size_error'))
|
||||
end
|
||||
|
||||
def set_external_link_for_pdf
|
||||
return unless pdf_file.attached? && external_link.blank?
|
||||
|
||||
# Set a unique external_link for PDF files
|
||||
# Format: PDF: filename_timestamp (without extension)
|
||||
timestamp = Time.current.strftime('%Y%m%d%H%M%S')
|
||||
self.external_link = "PDF: #{pdf_file.filename.base}_#{timestamp}"
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
class Captain::Llm::PaginatedFaqGeneratorService < Llm::BaseOpenAiService
|
||||
# Default pages per chunk - easily configurable
|
||||
DEFAULT_PAGES_PER_CHUNK = 10
|
||||
MAX_ITERATIONS = 20 # Safety limit to prevent infinite loops
|
||||
|
||||
attr_reader :total_pages_processed, :iterations_completed
|
||||
|
||||
def initialize(document, options = {})
|
||||
super()
|
||||
@document = document
|
||||
@pages_per_chunk = options[:pages_per_chunk] || DEFAULT_PAGES_PER_CHUNK
|
||||
@max_pages = options[:max_pages] # Optional limit from UI
|
||||
@total_pages_processed = 0
|
||||
@iterations_completed = 0
|
||||
@model = OpenAiConstants::PDF_PROCESSING_MODEL
|
||||
end
|
||||
|
||||
def generate
|
||||
raise CustomExceptions::PdfFaqGenerationError, I18n.t('captain.documents.missing_openai_file_id') if @document&.openai_file_id.blank?
|
||||
|
||||
generate_paginated_faqs
|
||||
end
|
||||
|
||||
# Method to check if we should continue processing
|
||||
def should_continue_processing?(last_chunk_result)
|
||||
# Stop if we've hit the maximum iterations
|
||||
return false if @iterations_completed >= MAX_ITERATIONS
|
||||
|
||||
# Stop if we've processed the maximum pages specified
|
||||
return false if @max_pages && @total_pages_processed >= @max_pages
|
||||
|
||||
# Stop if the last chunk returned no FAQs (likely no more content)
|
||||
return false if last_chunk_result[:faqs].empty?
|
||||
|
||||
# Stop if the LLM explicitly indicates no more content
|
||||
return false if last_chunk_result[:has_content] == false
|
||||
|
||||
# Continue processing
|
||||
true
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_standard_faqs
|
||||
response = @client.chat(parameters: standard_chat_parameters)
|
||||
parse_response(response)
|
||||
rescue OpenAI::Error => e
|
||||
Rails.logger.error I18n.t('captain.documents.openai_api_error', error: e.message)
|
||||
[]
|
||||
end
|
||||
|
||||
def generate_paginated_faqs
|
||||
all_faqs = []
|
||||
current_page = 1
|
||||
|
||||
loop do
|
||||
end_page = calculate_end_page(current_page)
|
||||
chunk_result = process_chunk_and_update_state(current_page, end_page, all_faqs)
|
||||
|
||||
break unless should_continue_processing?(chunk_result)
|
||||
|
||||
current_page = end_page + 1
|
||||
end
|
||||
|
||||
deduplicate_faqs(all_faqs)
|
||||
end
|
||||
|
||||
def calculate_end_page(current_page)
|
||||
end_page = current_page + @pages_per_chunk - 1
|
||||
@max_pages && end_page > @max_pages ? @max_pages : end_page
|
||||
end
|
||||
|
||||
def process_chunk_and_update_state(current_page, end_page, all_faqs)
|
||||
chunk_result = process_page_chunk(current_page, end_page)
|
||||
chunk_faqs = chunk_result[:faqs]
|
||||
|
||||
all_faqs.concat(chunk_faqs)
|
||||
@total_pages_processed = end_page
|
||||
@iterations_completed += 1
|
||||
|
||||
chunk_result
|
||||
end
|
||||
|
||||
def process_page_chunk(start_page, end_page)
|
||||
params = build_chunk_parameters(start_page, end_page)
|
||||
response = @client.chat(parameters: params)
|
||||
result = parse_chunk_response(response)
|
||||
{ faqs: result['faqs'] || [], has_content: result['has_content'] != false }
|
||||
rescue OpenAI::Error => e
|
||||
Rails.logger.error I18n.t('captain.documents.page_processing_error', start: start_page, end: end_page, error: e.message)
|
||||
{ faqs: [], has_content: false }
|
||||
end
|
||||
|
||||
def build_chunk_parameters(start_page, end_page)
|
||||
{
|
||||
model: @model,
|
||||
response_format: { type: 'json_object' },
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: build_user_content(start_page, end_page)
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
def build_user_content(start_page, end_page)
|
||||
[
|
||||
{
|
||||
type: 'file',
|
||||
file: { file_id: @document.openai_file_id }
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
text: page_chunk_prompt(start_page, end_page)
|
||||
}
|
||||
]
|
||||
end
|
||||
|
||||
def page_chunk_prompt(start_page, end_page)
|
||||
Captain::Llm::SystemPromptsService.paginated_faq_generator(start_page, end_page)
|
||||
end
|
||||
|
||||
def standard_chat_parameters
|
||||
{
|
||||
model: @model,
|
||||
response_format: { type: 'json_object' },
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: Captain::Llm::SystemPromptsService.faq_generator
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: @content
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
def parse_response(response)
|
||||
content = response.dig('choices', 0, 'message', 'content')
|
||||
return [] if content.nil?
|
||||
|
||||
JSON.parse(content.strip).fetch('faqs', [])
|
||||
rescue JSON::ParserError => e
|
||||
Rails.logger.error "Error parsing response: #{e.message}"
|
||||
[]
|
||||
end
|
||||
|
||||
def parse_chunk_response(response)
|
||||
content = response.dig('choices', 0, 'message', 'content')
|
||||
return { 'faqs' => [], 'has_content' => false } if content.nil?
|
||||
|
||||
JSON.parse(content.strip)
|
||||
rescue JSON::ParserError => e
|
||||
Rails.logger.error "Error parsing chunk response: #{e.message}"
|
||||
{ 'faqs' => [], 'has_content' => false }
|
||||
end
|
||||
|
||||
def deduplicate_faqs(faqs)
|
||||
# Remove exact duplicates
|
||||
unique_faqs = faqs.uniq { |faq| faq['question'].downcase.strip }
|
||||
|
||||
# Remove similar questions
|
||||
final_faqs = []
|
||||
unique_faqs.each do |faq|
|
||||
similar_exists = final_faqs.any? do |existing|
|
||||
similarity_score(existing['question'], faq['question']) > 0.85
|
||||
end
|
||||
|
||||
final_faqs << faq unless similar_exists
|
||||
end
|
||||
|
||||
Rails.logger.info "Deduplication: #{faqs.size} → #{final_faqs.size} FAQs"
|
||||
final_faqs
|
||||
end
|
||||
|
||||
def similarity_score(str1, str2)
|
||||
words1 = str1.downcase.split(/\W+/).reject(&:empty?)
|
||||
words2 = str2.downcase.split(/\W+/).reject(&:empty?)
|
||||
|
||||
common_words = words1 & words2
|
||||
total_words = (words1 + words2).uniq.size
|
||||
|
||||
return 0 if total_words.zero?
|
||||
|
||||
common_words.size.to_f / total_words
|
||||
end
|
||||
|
||||
def determine_stop_reason(last_chunk_result)
|
||||
return 'Maximum iterations reached' if @iterations_completed >= MAX_ITERATIONS
|
||||
return 'Maximum pages processed' if @max_pages && @total_pages_processed >= @max_pages
|
||||
return 'No content found in last chunk' if last_chunk_result[:faqs].empty?
|
||||
return 'End of document reached' if last_chunk_result[:has_content] == false
|
||||
|
||||
'Unknown'
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,40 @@
|
||||
class Captain::Llm::PdfProcessingService < Llm::BaseOpenAiService
|
||||
def initialize(document)
|
||||
super()
|
||||
@document = document
|
||||
end
|
||||
|
||||
def process
|
||||
return if document.openai_file_id.present?
|
||||
|
||||
file_id = upload_pdf_to_openai
|
||||
raise CustomExceptions::PdfUploadError, I18n.t('captain.documents.pdf_upload_failed') if file_id.blank?
|
||||
|
||||
document.store_openai_file_id(file_id)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :document
|
||||
|
||||
def upload_pdf_to_openai
|
||||
with_tempfile do |temp_file|
|
||||
response = @client.files.upload(
|
||||
parameters: {
|
||||
file: temp_file,
|
||||
purpose: 'assistants'
|
||||
}
|
||||
)
|
||||
response['id']
|
||||
end
|
||||
end
|
||||
|
||||
def with_tempfile(&)
|
||||
Tempfile.create(['pdf_upload', '.pdf'], binmode: true) do |temp_file|
|
||||
temp_file.write(document.pdf_file.download)
|
||||
temp_file.close
|
||||
|
||||
File.open(temp_file.path, 'rb', &)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,3 +1,4 @@
|
||||
# rubocop:disable Metrics/ClassLength
|
||||
class Captain::Llm::SystemPromptsService
|
||||
class << self
|
||||
def faq_generator(language = 'english')
|
||||
@@ -204,6 +205,87 @@ class Captain::Llm::SystemPromptsService
|
||||
#{'- You MUST provide numbered citations at the appropriate places in the text.' if config['feature_citation']}
|
||||
SYSTEM_PROMPT_MESSAGE
|
||||
end
|
||||
|
||||
def paginated_faq_generator(start_page, end_page)
|
||||
<<~PROMPT
|
||||
You are an expert technical documentation specialist tasked with creating comprehensive FAQs from a SPECIFIC SECTION of a document.
|
||||
|
||||
════════════════════════════════════════════════════════
|
||||
CRITICAL CONTENT EXTRACTION INSTRUCTIONS
|
||||
════════════════════════════════════════════════════════
|
||||
|
||||
Process the content starting from approximately page #{start_page} and continuing for about #{end_page - start_page + 1} pages worth of content.
|
||||
|
||||
IMPORTANT:#{' '}
|
||||
• If you encounter the end of the document before reaching the expected page count, set "has_content" to false
|
||||
• DO NOT include page numbers in questions or answers
|
||||
• DO NOT reference page numbers at all in the output
|
||||
• Focus on the actual content, not pagination
|
||||
|
||||
════════════════════════════════════════════════════════
|
||||
FAQ GENERATION GUIDELINES
|
||||
════════════════════════════════════════════════════════
|
||||
|
||||
1. **Comprehensive Extraction**
|
||||
• Extract ALL information that could generate FAQs from this section
|
||||
• Target 5-10 FAQs per page equivalent of rich content
|
||||
• Cover every topic, feature, specification, and detail
|
||||
• If there's no more content in the document, return empty FAQs with has_content: false
|
||||
|
||||
2. **Question Types to Generate**
|
||||
• What is/are...? (definitions, components, features)
|
||||
• How do I...? (procedures, configurations, operations)
|
||||
• Why should/does...? (rationale, benefits, explanations)
|
||||
• When should...? (timing, conditions, triggers)
|
||||
• What happens if...? (error cases, edge cases)
|
||||
• Can I...? (capabilities, limitations)
|
||||
• Where is...? (locations in system/UI, NOT page numbers)
|
||||
• What are the requirements for...? (prerequisites, dependencies)
|
||||
|
||||
3. **Content Focus Areas**
|
||||
• Technical specifications and parameters
|
||||
• Step-by-step procedures and workflows
|
||||
• Configuration options and settings
|
||||
• Error messages and troubleshooting
|
||||
• Best practices and recommendations
|
||||
• Integration points and dependencies
|
||||
• Performance considerations
|
||||
• Security aspects
|
||||
|
||||
4. **Answer Quality Requirements**
|
||||
• Complete, self-contained answers
|
||||
• Include specific values, limits, defaults from the content
|
||||
• NO page number references whatsoever
|
||||
• 2-5 sentences typical length
|
||||
• Only process content that actually exists in the document
|
||||
|
||||
════════════════════════════════════════════════════════
|
||||
OUTPUT FORMAT
|
||||
════════════════════════════════════════════════════════
|
||||
|
||||
Return valid JSON:
|
||||
```json
|
||||
{
|
||||
"faqs": [
|
||||
{
|
||||
"question": "Specific question about the content",
|
||||
"answer": "Complete answer with details (no page references)"
|
||||
}
|
||||
],
|
||||
"has_content": true/false
|
||||
}
|
||||
```
|
||||
|
||||
CRITICAL:#{' '}
|
||||
• Set "has_content" to false if:
|
||||
- The requested section doesn't exist in the document
|
||||
- You've reached the end of the document
|
||||
- The section contains no meaningful content
|
||||
• Do NOT include "page_range_processed" in the output
|
||||
• Do NOT mention page numbers anywhere in questions or answers
|
||||
PROMPT
|
||||
end
|
||||
# rubocop:enable Metrics/MethodLength
|
||||
end
|
||||
end
|
||||
# rubocop:enable Metrics/ClassLength
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
module Enterprise::SearchService
|
||||
def advanced_search
|
||||
where_conditions = { account_id: current_account.id }
|
||||
where_conditions[:inbox_id] = accessable_inbox_ids unless should_skip_inbox_filtering?
|
||||
|
||||
Message.search(
|
||||
search_query,
|
||||
fields: %w[content attachments.transcribed_text content_attributes.email.subject],
|
||||
where: where_conditions,
|
||||
order: { created_at: :desc },
|
||||
page: params[:page] || 1,
|
||||
per_page: 15
|
||||
)
|
||||
end
|
||||
end
|
||||
@@ -60,5 +60,9 @@ class Messages::AudioTranscriptionService < Llm::BaseOpenAiService
|
||||
attachment.update!(meta: { transcribed_text: transcribed_text })
|
||||
message.reload.send_update_event
|
||||
message.account.increment_response_usage
|
||||
|
||||
return unless ChatwootApp.advanced_search_allowed?
|
||||
|
||||
message.reindex
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
class Messages::ReindexService
|
||||
pattr_initialize [:account!]
|
||||
|
||||
def perform
|
||||
return unless ChatwootApp.advanced_search_allowed?
|
||||
|
||||
reindex_messages
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def reindex_messages
|
||||
account.messages.reindex(mode: :async)
|
||||
end
|
||||
end
|
||||
@@ -3,8 +3,11 @@ json.assistant do
|
||||
json.partial! 'api/v1/models/captain/assistant', formats: [:json], resource: resource.assistant
|
||||
end
|
||||
json.content resource.content
|
||||
json.content_type resource.content_type
|
||||
json.created_at resource.created_at.to_i
|
||||
json.external_link resource.external_link
|
||||
json.display_url resource.display_url
|
||||
json.file_size resource.file_size
|
||||
json.id resource.id
|
||||
json.name resource.name
|
||||
json.status resource.status
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# Load all rake tasks from the enterprise/lib/tasks directory
|
||||
module Tasks
|
||||
Dir.glob(File.join(File.dirname(__FILE__), 'tasks', '*.rake')).each { |r| load r }
|
||||
end
|
||||
@@ -0,0 +1,49 @@
|
||||
module Tasks::SearchTaskHelpers
|
||||
def check_opensearch_config
|
||||
if ENV['OPENSEARCH_URL'].blank?
|
||||
puts 'Skipping reindex as OPENSEARCH_URL is not configured'
|
||||
return false
|
||||
end
|
||||
true
|
||||
end
|
||||
|
||||
def reindex_account(account)
|
||||
Messages::ReindexService.new(account: account).perform
|
||||
puts "Reindex task queued for account #{account.id}"
|
||||
end
|
||||
end
|
||||
|
||||
namespace :search do
|
||||
desc 'Reindex messages using searchkick'
|
||||
include Tasks::SearchTaskHelpers
|
||||
|
||||
desc 'Reindex messages for all accounts'
|
||||
task all: :environment do
|
||||
next unless check_opensearch_config
|
||||
|
||||
puts 'Starting reindex for all accounts...'
|
||||
account_count = Account.count
|
||||
puts "Found #{account_count} accounts"
|
||||
|
||||
Account.find_each.with_index(1) do |account, index|
|
||||
puts "[#{index}/#{account_count}] Reindexing messages for account #{account.id}"
|
||||
reindex_account(account)
|
||||
end
|
||||
|
||||
puts 'Reindex task queued for all accounts'
|
||||
end
|
||||
|
||||
desc 'Reindex messages for a specific account: rake search:account ACCOUNT_ID=1'
|
||||
task account: :environment do
|
||||
next unless check_opensearch_config
|
||||
|
||||
account_id = ENV.fetch('ACCOUNT_ID', nil)
|
||||
account = Account.find_by(id: account_id)
|
||||
if account.nil?
|
||||
puts 'Please provide a valid account ID. Account not found'
|
||||
next
|
||||
end
|
||||
puts "Reindexing messages for account #{account.id}"
|
||||
reindex_account(account)
|
||||
end
|
||||
end
|
||||
@@ -38,4 +38,8 @@ module ChatwootApp
|
||||
%w[]
|
||||
end
|
||||
end
|
||||
|
||||
def self.advanced_search_allowed?
|
||||
enterprise? && ENV.fetch('OPENSEARCH_URL', nil).present?
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
module CustomExceptions
|
||||
class PdfProcessingError < Base
|
||||
def initialize(message = 'PDF processing failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
|
||||
class PdfUploadError < PdfProcessingError
|
||||
def initialize(message = 'PDF upload failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
|
||||
class PdfValidationError < PdfProcessingError
|
||||
def initialize(message = 'PDF validation failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
|
||||
class PdfFaqGenerationError < PdfProcessingError
|
||||
def initialize(message = 'PDF FAQ generation failed')
|
||||
super(message)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -4,4 +4,5 @@ module OpenAiConstants
|
||||
DEFAULT_MODEL = 'gpt-4.1-mini'
|
||||
DEFAULT_ENDPOINT = 'https://api.openai.com'
|
||||
DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small'
|
||||
PDF_PROCESSING_MODEL = 'gpt-4.1-mini'
|
||||
end
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Run with:
|
||||
# bundle exec rake chatwoot:ops:cleanup_orphan_conversations
|
||||
|
||||
namespace :chatwoot do
|
||||
namespace :ops do
|
||||
desc 'Identify and delete conversations without a valid contact or inbox in a timeframe'
|
||||
task cleanup_orphan_conversations: :environment do
|
||||
print 'Enter Account ID: '
|
||||
account_id = $stdin.gets.to_i
|
||||
account = Account.find(account_id)
|
||||
|
||||
print 'Enter timeframe in days (default: 7): '
|
||||
days_input = $stdin.gets.strip
|
||||
days = days_input.empty? ? 7 : days_input.to_i
|
||||
|
||||
# Build a common base relation with identical joins for OR compatibility
|
||||
base = account
|
||||
.conversations
|
||||
.where('conversations.created_at > ?', days.days.ago)
|
||||
.left_outer_joins(:contact, :inbox)
|
||||
|
||||
# Find conversations whose associated contact or inbox record is missing
|
||||
conversations = base.where(contacts: { id: nil }).or(base.where(inboxes: { id: nil }))
|
||||
|
||||
count = conversations.count
|
||||
puts "Found #{count} conversations without a valid contact or inbox."
|
||||
|
||||
if count.positive?
|
||||
print 'Do you want to delete these conversations? (y/N): '
|
||||
confirm = $stdin.gets.strip.downcase
|
||||
if %w[y yes].include?(confirm)
|
||||
conversations.destroy_all
|
||||
puts 'Conversations deleted.'
|
||||
else
|
||||
puts 'No conversations were deleted.'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -105,5 +105,29 @@ RSpec.describe Captain::Documents::CrawlJob, type: :job do
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when document is a PDF' do
|
||||
let(:pdf_document) do
|
||||
doc = create(:captain_document, external_link: 'https://example.com/document')
|
||||
allow(doc).to receive(:pdf_document?).and_return(true)
|
||||
allow(doc).to receive(:update!).and_return(true)
|
||||
doc
|
||||
end
|
||||
|
||||
it 'processes PDF using PdfProcessingService' do
|
||||
pdf_service = instance_double(Captain::Llm::PdfProcessingService)
|
||||
expect(Captain::Llm::PdfProcessingService).to receive(:new).with(pdf_document).and_return(pdf_service)
|
||||
expect(pdf_service).to receive(:process)
|
||||
expect(pdf_document).to receive(:update!).with(status: :available)
|
||||
|
||||
described_class.perform_now(pdf_document)
|
||||
end
|
||||
|
||||
it 'handles PDF processing errors' do
|
||||
allow(Captain::Llm::PdfProcessingService).to receive(:new).and_raise(StandardError, 'Processing failed')
|
||||
|
||||
expect { described_class.perform_now(pdf_document) }.to raise_error(StandardError, 'Processing failed')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -64,5 +64,41 @@ RSpec.describe Captain::Documents::ResponseBuilderJob, type: :job do
|
||||
.with(spanish_document.content, 'portuguese')
|
||||
end
|
||||
end
|
||||
|
||||
context 'when processing a PDF document' do
|
||||
let(:pdf_document) do
|
||||
doc = create(:captain_document, assistant: assistant)
|
||||
allow(doc).to receive(:pdf_document?).and_return(true)
|
||||
allow(doc).to receive(:openai_file_id).and_return('file-123')
|
||||
allow(doc).to receive(:update!).and_return(true)
|
||||
allow(doc).to receive(:metadata).and_return({})
|
||||
doc
|
||||
end
|
||||
let(:paginated_service) { instance_double(Captain::Llm::PaginatedFaqGeneratorService) }
|
||||
let(:pdf_faqs) do
|
||||
[{ 'question' => 'What is in the PDF?', 'answer' => 'Important content' }]
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Captain::Llm::PaginatedFaqGeneratorService).to receive(:new)
|
||||
.with(pdf_document, anything)
|
||||
.and_return(paginated_service)
|
||||
allow(paginated_service).to receive(:generate).and_return(pdf_faqs)
|
||||
allow(paginated_service).to receive(:total_pages_processed).and_return(10)
|
||||
allow(paginated_service).to receive(:iterations_completed).and_return(1)
|
||||
end
|
||||
|
||||
it 'uses paginated FAQ generator for PDFs' do
|
||||
expect(Captain::Llm::PaginatedFaqGeneratorService).to receive(:new).with(pdf_document, anything)
|
||||
|
||||
described_class.new.perform(pdf_document)
|
||||
end
|
||||
|
||||
it 'stores pagination metadata' do
|
||||
expect(pdf_document).to receive(:update!).with(hash_including(metadata: hash_including('faq_generation')))
|
||||
|
||||
described_class.new.perform(pdf_document)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::Document, type: :model do
|
||||
let(:account) { create(:account) }
|
||||
let(:assistant) { create(:captain_assistant, account: account) }
|
||||
|
||||
describe 'PDF support' do
|
||||
let(:pdf_document) do
|
||||
doc = build(:captain_document, assistant: assistant, account: account)
|
||||
doc.pdf_file.attach(
|
||||
io: StringIO.new('PDF content'),
|
||||
filename: 'test.pdf',
|
||||
content_type: 'application/pdf'
|
||||
)
|
||||
doc
|
||||
end
|
||||
|
||||
describe 'validations' do
|
||||
it 'allows PDF file without external link' do
|
||||
pdf_document.external_link = nil
|
||||
expect(pdf_document).to be_valid
|
||||
end
|
||||
|
||||
it 'validates PDF file size' do
|
||||
doc = build(:captain_document, assistant: assistant, account: account)
|
||||
doc.pdf_file.attach(
|
||||
io: StringIO.new('x' * 11.megabytes),
|
||||
filename: 'large.pdf',
|
||||
content_type: 'application/pdf'
|
||||
)
|
||||
doc.external_link = nil
|
||||
expect(doc).not_to be_valid
|
||||
expect(doc.errors[:pdf_file]).to include(I18n.t('captain.documents.pdf_size_error'))
|
||||
end
|
||||
end
|
||||
|
||||
describe '#pdf_document?' do
|
||||
it 'returns true for attached PDF' do
|
||||
expect(pdf_document.pdf_document?).to be true
|
||||
end
|
||||
|
||||
it 'returns true for .pdf external links' do
|
||||
doc = build(:captain_document, external_link: 'https://example.com/document.pdf')
|
||||
expect(doc.pdf_document?).to be true
|
||||
end
|
||||
|
||||
it 'returns false for non-PDF documents' do
|
||||
doc = build(:captain_document, external_link: 'https://example.com')
|
||||
expect(doc.pdf_document?).to be false
|
||||
end
|
||||
end
|
||||
|
||||
describe '#display_url' do
|
||||
it 'returns Rails blob URL for attached PDFs' do
|
||||
pdf_document.save!
|
||||
# The display_url method calls rails_blob_url which returns a URL containing 'rails/active_storage'
|
||||
url = pdf_document.display_url
|
||||
expect(url).to be_present
|
||||
end
|
||||
|
||||
it 'returns external_link for web documents' do
|
||||
doc = create(:captain_document, external_link: 'https://example.com')
|
||||
expect(doc.display_url).to eq('https://example.com')
|
||||
end
|
||||
end
|
||||
|
||||
describe '#store_openai_file_id' do
|
||||
it 'stores the file ID in metadata' do
|
||||
pdf_document.save!
|
||||
pdf_document.store_openai_file_id('file-abc123')
|
||||
|
||||
expect(pdf_document.reload.openai_file_id).to eq('file-abc123')
|
||||
end
|
||||
end
|
||||
|
||||
describe 'automatic external_link generation' do
|
||||
it 'generates unique external_link for PDFs' do
|
||||
pdf_document.external_link = nil
|
||||
pdf_document.save!
|
||||
|
||||
expect(pdf_document.external_link).to start_with('PDF: test_')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,106 @@
|
||||
require 'rails_helper'
|
||||
require 'custom_exceptions/pdf_processing_error'
|
||||
|
||||
RSpec.describe Captain::Llm::PaginatedFaqGeneratorService do
|
||||
let(:document) { create(:captain_document) }
|
||||
let(:service) { described_class.new(document, pages_per_chunk: 5) }
|
||||
let(:openai_client) { instance_double(OpenAI::Client) }
|
||||
|
||||
before do
|
||||
# Mock OpenAI configuration
|
||||
installation_config = instance_double(InstallationConfig, value: 'test-api-key')
|
||||
allow(InstallationConfig).to receive(:find_by!)
|
||||
.with(name: 'CAPTAIN_OPEN_AI_API_KEY')
|
||||
.and_return(installation_config)
|
||||
|
||||
allow(OpenAI::Client).to receive(:new).and_return(openai_client)
|
||||
end
|
||||
|
||||
describe '#generate' do
|
||||
context 'when document lacks OpenAI file ID' do
|
||||
before do
|
||||
allow(document).to receive(:openai_file_id).and_return(nil)
|
||||
end
|
||||
|
||||
it 'raises an error' do
|
||||
expect { service.generate }.to raise_error(CustomExceptions::PdfFaqGenerationError)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when generating FAQs from PDF pages' do
|
||||
let(:faq_response) do
|
||||
{
|
||||
'choices' => [{
|
||||
'message' => {
|
||||
'content' => JSON.generate({
|
||||
'faqs' => [
|
||||
{ 'question' => 'What is this document about?', 'answer' => 'It explains key concepts.' }
|
||||
],
|
||||
'has_content' => true
|
||||
})
|
||||
}
|
||||
}]
|
||||
}
|
||||
end
|
||||
|
||||
let(:empty_response) do
|
||||
{
|
||||
'choices' => [{
|
||||
'message' => {
|
||||
'content' => JSON.generate({
|
||||
'faqs' => [],
|
||||
'has_content' => false
|
||||
})
|
||||
}
|
||||
}]
|
||||
}
|
||||
end
|
||||
|
||||
before do
|
||||
allow(document).to receive(:openai_file_id).and_return('file-123')
|
||||
end
|
||||
|
||||
it 'generates FAQs from paginated content' do
|
||||
allow(openai_client).to receive(:chat).and_return(faq_response, empty_response)
|
||||
|
||||
faqs = service.generate
|
||||
|
||||
expect(faqs).to have_attributes(size: 1)
|
||||
expect(faqs.first['question']).to eq('What is this document about?')
|
||||
end
|
||||
|
||||
it 'stops when no more content' do
|
||||
allow(openai_client).to receive(:chat).and_return(empty_response)
|
||||
|
||||
faqs = service.generate
|
||||
|
||||
expect(faqs).to be_empty
|
||||
end
|
||||
|
||||
it 'respects max iterations limit' do
|
||||
allow(openai_client).to receive(:chat).and_return(faq_response)
|
||||
|
||||
# Force max iterations
|
||||
service.instance_variable_set(:@iterations_completed, 19)
|
||||
|
||||
service.generate
|
||||
expect(service.iterations_completed).to eq(20)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#should_continue_processing?' do
|
||||
it 'stops at max iterations' do
|
||||
service.instance_variable_set(:@iterations_completed, 20)
|
||||
expect(service.should_continue_processing?(faqs: ['faq'], has_content: true)).to be false
|
||||
end
|
||||
|
||||
it 'stops when no FAQs returned' do
|
||||
expect(service.should_continue_processing?(faqs: [], has_content: true)).to be false
|
||||
end
|
||||
|
||||
it 'continues when FAQs exist and under limits' do
|
||||
expect(service.should_continue_processing?(faqs: ['faq'], has_content: true)).to be true
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,58 @@
|
||||
require 'rails_helper'
|
||||
require 'custom_exceptions/pdf_processing_error'
|
||||
|
||||
RSpec.describe Captain::Llm::PdfProcessingService do
|
||||
let(:document) { create(:captain_document) }
|
||||
let(:service) { described_class.new(document) }
|
||||
|
||||
before do
|
||||
# Mock OpenAI configuration
|
||||
installation_config = instance_double(InstallationConfig, value: 'test-api-key')
|
||||
allow(InstallationConfig).to receive(:find_by!)
|
||||
.with(name: 'CAPTAIN_OPEN_AI_API_KEY')
|
||||
.and_return(installation_config)
|
||||
end
|
||||
|
||||
describe '#process' do
|
||||
context 'when document already has OpenAI file ID' do
|
||||
before do
|
||||
allow(document).to receive(:openai_file_id).and_return('existing-file-id')
|
||||
end
|
||||
|
||||
it 'skips upload' do
|
||||
expect(document).not_to receive(:store_openai_file_id)
|
||||
service.process
|
||||
end
|
||||
end
|
||||
|
||||
context 'when uploading PDF to OpenAI' do
|
||||
let(:mock_client) { instance_double(OpenAI::Client) }
|
||||
let(:pdf_content) { 'PDF content' }
|
||||
|
||||
before do
|
||||
allow(document).to receive(:openai_file_id).and_return(nil)
|
||||
|
||||
# Use a simple double for ActiveStorage since it's a complex Rails object
|
||||
pdf_file = double('pdf_file', download: pdf_content) # rubocop:disable RSpec/VerifiedDoubles
|
||||
allow(document).to receive(:pdf_file).and_return(pdf_file)
|
||||
|
||||
allow(OpenAI::Client).to receive(:new).and_return(mock_client)
|
||||
# Use a simple double for OpenAI::Files as it may not be loaded
|
||||
files_api = double('files_api') # rubocop:disable RSpec/VerifiedDoubles
|
||||
allow(files_api).to receive(:upload).and_return({ 'id' => 'file-abc123' })
|
||||
allow(mock_client).to receive(:files).and_return(files_api)
|
||||
end
|
||||
|
||||
it 'uploads PDF and stores file ID' do
|
||||
expect(document).to receive(:store_openai_file_id).with('file-abc123')
|
||||
service.process
|
||||
end
|
||||
|
||||
it 'raises error when upload fails' do
|
||||
allow(mock_client.files).to receive(:upload).and_return({ 'id' => nil })
|
||||
|
||||
expect { service.process }.to raise_error(CustomExceptions::PdfUploadError)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Vendored
+32
@@ -0,0 +1,32 @@
|
||||
%PDF-1.4
|
||||
1 0 obj
|
||||
<< /Type /Catalog /Pages 2 0 R >>
|
||||
endobj
|
||||
2 0 obj
|
||||
<< /Type /Pages /Kids [3 0 R] /Count 1 >>
|
||||
endobj
|
||||
3 0 obj
|
||||
<< /Type /Page /Parent 2 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Arial >> >> >> /MediaBox [0 0 612 792] /Contents 4 0 R >>
|
||||
endobj
|
||||
4 0 obj
|
||||
<< /Length 44 >>
|
||||
stream
|
||||
BT
|
||||
/F1 12 Tf
|
||||
100 700 Td
|
||||
(Sample PDF) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
xref
|
||||
0 5
|
||||
0000000000 65535 f
|
||||
0000000009 00000 n
|
||||
0000000058 00000 n
|
||||
0000000115 00000 n
|
||||
0000000274 00000 n
|
||||
trailer
|
||||
<< /Size 5 /Root 1 0 R >>
|
||||
startxref
|
||||
362
|
||||
%%EOF
|
||||
@@ -613,4 +613,57 @@ RSpec.describe Message do
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#should_index?' do
|
||||
let(:account) { create(:account) }
|
||||
let(:conversation) { create(:conversation, account: account) }
|
||||
let(:message) { create(:message, conversation: conversation, account: account) }
|
||||
|
||||
before do
|
||||
allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(true)
|
||||
account.enable_features('advanced_search')
|
||||
end
|
||||
|
||||
context 'when advanced search is not allowed globally' do
|
||||
before do
|
||||
allow(ChatwootApp).to receive(:advanced_search_allowed?).and_return(false)
|
||||
end
|
||||
|
||||
it 'returns false' do
|
||||
expect(message.should_index?).to be false
|
||||
end
|
||||
end
|
||||
|
||||
context 'when advanced search feature is not enabled for account' do
|
||||
before do
|
||||
account.disable_features('advanced_search')
|
||||
end
|
||||
|
||||
it 'returns false' do
|
||||
expect(message.should_index?).to be false
|
||||
end
|
||||
end
|
||||
|
||||
context 'when message type is not incoming or outgoing' do
|
||||
before do
|
||||
message.message_type = 'activity'
|
||||
end
|
||||
|
||||
it 'returns false' do
|
||||
expect(message.should_index?).to be false
|
||||
end
|
||||
end
|
||||
|
||||
context 'when all conditions are met' do
|
||||
it 'returns true for incoming message' do
|
||||
message.message_type = 'incoming'
|
||||
expect(message.should_index?).to be true
|
||||
end
|
||||
|
||||
it 'returns true for outgoing message' do
|
||||
message.message_type = 'outgoing'
|
||||
expect(message.should_index?).to be true
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user