update the flow to use openai pdf
This commit is contained in:
Tanmay Deep Sharma
@@ -174,8 +174,6 @@ gem 'neighbor'
|
||||
gem 'pgvector'
|
||||
# Convert Website HTML to Markdown
|
||||
gem 'reverse_markdown'
|
||||
# PDF text extraction
|
||||
gem 'pdf-reader', '~> 2.0'
|
||||
|
||||
gem 'iso-639'
|
||||
gem 'ruby-openai'
|
||||
|
||||
@@ -25,7 +25,6 @@ GIT
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
Ascii85 (2.0.1)
|
||||
actioncable (7.1.5.1)
|
||||
actionpack (= 7.1.5.1)
|
||||
activesupport (= 7.1.5.1)
|
||||
@@ -127,7 +126,6 @@ GEM
|
||||
jbuilder (~> 2)
|
||||
rails (>= 4.2, < 7.2)
|
||||
selectize-rails (~> 0.6)
|
||||
afm (0.2.2)
|
||||
annotate (3.2.0)
|
||||
activerecord (>= 3.2, < 8.0)
|
||||
rake (>= 10.4, < 14.0)
|
||||
@@ -367,7 +365,6 @@ GEM
|
||||
hana (1.3.7)
|
||||
hash_diff (1.1.1)
|
||||
hashdiff (1.1.0)
|
||||
hashery (2.1.2)
|
||||
hashie (5.0.0)
|
||||
html2text (0.4.0)
|
||||
nokogiri (>= 1.0, < 2.0)
|
||||
@@ -557,12 +554,6 @@ GEM
|
||||
parser (3.3.8.0)
|
||||
ast (~> 2.4.1)
|
||||
racc
|
||||
pdf-reader (2.14.1)
|
||||
Ascii85 (>= 1.0, < 3.0, != 2.0.0)
|
||||
afm (~> 0.2.1)
|
||||
hashery (~> 2.0)
|
||||
ruby-rc4
|
||||
ttfunk
|
||||
pg (1.5.3)
|
||||
pg_search (2.3.6)
|
||||
activerecord (>= 5.2)
|
||||
@@ -720,7 +711,6 @@ GEM
|
||||
faraday (>= 1)
|
||||
faraday-multipart (>= 1)
|
||||
ruby-progressbar (1.13.0)
|
||||
ruby-rc4 (0.1.5)
|
||||
ruby-vips (2.1.4)
|
||||
ffi (~> 1.12)
|
||||
ruby2_keywords (0.0.5)
|
||||
@@ -829,8 +819,6 @@ GEM
|
||||
i18n
|
||||
timeout (0.4.3)
|
||||
trailblazer-option (0.1.2)
|
||||
ttfunk (1.8.0)
|
||||
bigdecimal (~> 3.1)
|
||||
twilio-ruby (5.77.0)
|
||||
faraday (>= 0.9, < 3.0)
|
||||
jwt (>= 1.5, < 3.0)
|
||||
@@ -975,7 +963,6 @@ DEPENDENCIES
|
||||
omniauth-google-oauth2 (>= 1.1.3)
|
||||
omniauth-oauth2
|
||||
omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2)
|
||||
pdf-reader (~> 2.0)
|
||||
pg
|
||||
pg_search
|
||||
pgvector
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
class AddPdfSupportToCaptainDocuments < ActiveRecord::Migration[7.1]
|
||||
def change
|
||||
add_column :captain_documents, :source_type, :string, default: 'url'
|
||||
|
||||
add_index :captain_documents, :source_type
|
||||
end
|
||||
end
|
||||
@@ -34,8 +34,7 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC
|
||||
|
||||
@document = @assistant.documents.build(
|
||||
name: pdf_params[:pdf_document].original_filename,
|
||||
external_link: "pdf_upload_#{SecureRandom.hex(8)}.pdf",
|
||||
source_type: 'pdf_upload'
|
||||
external_link: "pdf_upload_#{SecureRandom.hex(8)}.pdf"
|
||||
)
|
||||
@document.file.attach(pdf_params[:pdf_document])
|
||||
@document.save!
|
||||
|
||||
@@ -14,7 +14,7 @@ class Captain::Documents::CrawlJob < ApplicationJob
|
||||
def pdf_document?(document)
|
||||
return false if document.nil?
|
||||
|
||||
document.source_type == 'pdf_upload'
|
||||
document.pdf_document?
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
@@ -4,12 +4,18 @@ class Captain::Documents::PdfExtractionJob < ApplicationJob
|
||||
|
||||
queue_as :low
|
||||
|
||||
# This job runs in parallel with S3 upload
|
||||
# ActiveStorage handles S3 upload asynchronously
|
||||
# We process the PDF independently
|
||||
|
||||
def perform(document)
|
||||
return unless document.pdf_document?
|
||||
|
||||
initialize_document_processing(document)
|
||||
result = extract_pdf_content(document)
|
||||
process_extraction_result(document, result)
|
||||
rescue StandardError => e
|
||||
handle_extraction_failure(document, e)
|
||||
end
|
||||
|
||||
private
|
||||
@@ -19,57 +25,58 @@ class Captain::Documents::PdfExtractionJob < ApplicationJob
|
||||
end
|
||||
|
||||
def extract_pdf_content(document)
|
||||
pdf_source = document.file.attached? ? document.file : document.external_link
|
||||
pdf_extraction_service = Captain::Tools::PdfExtractionService.new(pdf_source)
|
||||
pdf_extraction_service.perform
|
||||
pdf_source = document.file
|
||||
|
||||
# Always use OpenAI direct PDF processing
|
||||
pdf_service = Captain::Tools::PdfOpenaiService.new(pdf_source)
|
||||
pdf_service.perform
|
||||
end
|
||||
|
||||
def process_extraction_result(document, result)
|
||||
return unless result[:success] && result[:content].present?
|
||||
|
||||
process_pdf_content_chunks(document, result[:content])
|
||||
if result[:success] && result[:content].present?
|
||||
process_pdf_content(document, result[:content])
|
||||
else
|
||||
handle_pdf_extraction_failure(document, result)
|
||||
end
|
||||
end
|
||||
|
||||
def process_pdf_content_chunks(document, content_chunks)
|
||||
Rails.logger.info "PDF extraction successful for document #{document.id}: #{content_chunks.length} chunks will be processed"
|
||||
def process_pdf_content(document, content)
|
||||
# Content is an array with single item for direct PDF processing
|
||||
pdf_data = content.first || {}
|
||||
metadata = pdf_data[:metadata]
|
||||
|
||||
# Update main document with first page content only
|
||||
first_page_content = content_chunks.first&.dig(:content) || ''
|
||||
if metadata && metadata[:processing_type] == 'direct_pdf'
|
||||
process_direct_pdf_content(document, metadata)
|
||||
else
|
||||
document.update!(
|
||||
content: pdf_data[:content] || '',
|
||||
status: 'available'
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def process_direct_pdf_content(document, metadata)
|
||||
# Store the OpenAI file_id in the document
|
||||
document.update!(
|
||||
content: first_page_content,
|
||||
content: metadata[:openai_file_id],
|
||||
status: 'available'
|
||||
)
|
||||
|
||||
# Reset previous responses once at the beginning to avoid race conditions
|
||||
# Process the PDF directly
|
||||
document.responses.destroy_all
|
||||
|
||||
# Process each chunk separately but link FAQs to main document
|
||||
content_chunks.each_with_index do |content_chunk, index|
|
||||
log_chunk_queueing(document, content_chunk, index, content_chunks.length)
|
||||
queue_pdf_chunk_job(document, content_chunk)
|
||||
end
|
||||
|
||||
Rails.logger.info "All #{content_chunks.length} chunks queued for processing for document #{document.id}"
|
||||
end
|
||||
|
||||
def log_chunk_queueing(document, content_chunk, index, total_chunks)
|
||||
page_num = content_chunk[:page_number]
|
||||
content_length = content_chunk[:content].length
|
||||
Rails.logger.info "Queueing chunk #{index + 1}/#{total_chunks} for document #{document.id} " \
|
||||
"(Page #{page_num}, #{content_length} chars)"
|
||||
end
|
||||
|
||||
def queue_pdf_chunk_job(document, content_chunk)
|
||||
Captain::Tools::PdfExtractionParserJob.perform_later(
|
||||
assistant_id: document.assistant_id,
|
||||
pdf_content: content_chunk,
|
||||
document_id: document.id
|
||||
Captain::Documents::ResponseBuilderJob.perform_later(
|
||||
document,
|
||||
metadata[:openai_file_id],
|
||||
skip_reset: false,
|
||||
metadata: metadata
|
||||
)
|
||||
end
|
||||
|
||||
def handle_pdf_extraction_failure(document, result)
|
||||
error_message = result[:errors]&.join(', ') || 'Failed to extract text from PDF'
|
||||
Rails.logger.error "PDF extraction failed for document #{document.id}: #{error_message}"
|
||||
def handle_pdf_extraction_failure(document, _result)
|
||||
document.update(status: 'available')
|
||||
end
|
||||
|
||||
def handle_extraction_failure(document, _error)
|
||||
document.update(status: 'available')
|
||||
end
|
||||
end
|
||||
@@ -1,17 +1,28 @@
|
||||
class Captain::Documents::ResponseBuilderJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(document, full_content = nil, skip_reset: false)
|
||||
def perform(document, full_content = nil, skip_reset: false, metadata: {})
|
||||
# Use full content for FAQ generation if provided (for PDFs), otherwise use document.content
|
||||
content_for_faqs = full_content || document.content
|
||||
|
||||
# Skip processing if no content available
|
||||
return if content_for_faqs.blank?
|
||||
|
||||
# Only reset responses if not explicitly skipped (for PDF chunks processing)
|
||||
# Only reset responses if not explicitly skipped
|
||||
reset_previous_responses(document) unless skip_reset
|
||||
|
||||
faqs = Captain::Llm::FaqGeneratorService.new(content_for_faqs).generate
|
||||
# Check if this is a direct PDF processing
|
||||
faqs = if metadata[:processing_type] == 'direct_pdf' && metadata[:openai_file_id]
|
||||
Captain::Llm::PdfFaqGeneratorService.new(
|
||||
content_for_faqs,
|
||||
is_pdf_file: true,
|
||||
metadata: metadata
|
||||
).generate
|
||||
# NOTE: Not deleting the file_id - keeping it for future use
|
||||
else
|
||||
Captain::Llm::FaqGeneratorService.new(content_for_faqs).generate
|
||||
end
|
||||
|
||||
faqs.each do |faq|
|
||||
create_response(faq, document)
|
||||
end
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
class Captain::Tools::PdfExtractionParserJob < ApplicationJob
|
||||
queue_as :low
|
||||
|
||||
def perform(assistant_id:, pdf_content:, document_id: nil)
|
||||
assistant = Captain::Assistant.find(assistant_id)
|
||||
main_document = find_main_document(document_id)
|
||||
|
||||
return unless can_process_content?(pdf_content[:content], assistant.account, main_document)
|
||||
|
||||
enqueue_response_builder_job_for_chunk(main_document, pdf_content)
|
||||
rescue ActiveRecord::RecordNotFound => e
|
||||
Rails.logger.error "PDF parser job failed - Document not found: #{e.message}"
|
||||
rescue Captain::Document::LimitExceededError => e
|
||||
Rails.logger.info "PDF parser job stopped - Document limit exceeded: #{e.message}"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def find_main_document(document_id)
|
||||
return unless document_id
|
||||
|
||||
Captain::Document.find(document_id)
|
||||
end
|
||||
|
||||
def can_process_content?(content, account, main_document)
|
||||
main_document && should_process_content?(content, account)
|
||||
end
|
||||
|
||||
def enqueue_response_builder_job_for_chunk(main_document, pdf_content)
|
||||
# Create a context string for better AI processing that includes page info
|
||||
page_info = pdf_content[:page_number] ? " (Page #{pdf_content[:page_number]})" : ''
|
||||
chunk_info = pdf_content[:chunk_index] ? " Part #{pdf_content[:chunk_index]}" : ''
|
||||
|
||||
# Combine main document content with chunk content for comprehensive FAQ generation
|
||||
context_content = "#{main_document.content}\n\n--- Additional Content#{page_info}#{chunk_info} ---\n#{pdf_content[:content]}"
|
||||
|
||||
# Use the ResponseBuilderJob with full_content parameter to generate FAQs
|
||||
# This will link all FAQs to the main document while using the chunk content for AI processing
|
||||
# Skip reset since it's already done in the main PDF extraction job
|
||||
Captain::Documents::ResponseBuilderJob.perform_later(main_document, context_content, skip_reset: true)
|
||||
end
|
||||
|
||||
def should_process_content?(content, account)
|
||||
content.present? && !limit_exceeded?(account)
|
||||
end
|
||||
|
||||
def limit_exceeded?(account)
|
||||
limits = account.usage_limits.dig(:captain, :documents)
|
||||
limits && limits[:current_available].to_i <= 0
|
||||
end
|
||||
end
|
||||
@@ -6,7 +6,6 @@
|
||||
# content :text
|
||||
# external_link :string not null
|
||||
# name :string
|
||||
# source_type :string default("url")
|
||||
# status :integer default("in_progress"), not null
|
||||
# created_at :datetime not null
|
||||
# updated_at :datetime not null
|
||||
@@ -18,7 +17,6 @@
|
||||
# index_captain_documents_on_account_id (account_id)
|
||||
# index_captain_documents_on_assistant_id (assistant_id)
|
||||
# index_captain_documents_on_assistant_id_and_external_link (assistant_id,external_link) UNIQUE
|
||||
# index_captain_documents_on_source_type (source_type)
|
||||
# index_captain_documents_on_status (status)
|
||||
#
|
||||
class Captain::Document < ApplicationRecord
|
||||
@@ -33,9 +31,7 @@ class Captain::Document < ApplicationRecord
|
||||
validates :external_link, presence: true
|
||||
validates :external_link, uniqueness: { scope: :assistant_id }
|
||||
validates :content, length: { maximum: 400_000 }
|
||||
validates :source_type, inclusion: { in: %w[url pdf_upload] }, allow_blank: true
|
||||
before_validation :ensure_account_id
|
||||
before_validation :set_default_source_type
|
||||
|
||||
enum status: {
|
||||
in_progress: 0,
|
||||
@@ -53,7 +49,7 @@ class Captain::Document < ApplicationRecord
|
||||
scope :for_assistant, ->(assistant_id) { where(assistant_id: assistant_id) }
|
||||
|
||||
def pdf_document?
|
||||
source_type == 'pdf_upload' || file.attached?
|
||||
file.attached? || external_link&.match?(/\.pdf$/i)
|
||||
end
|
||||
|
||||
private
|
||||
@@ -84,14 +80,4 @@ class Captain::Document < ApplicationRecord
|
||||
limits = account.usage_limits[:captain][:documents]
|
||||
raise LimitExceededError, 'Document limit exceeded' unless limits[:current_available].positive?
|
||||
end
|
||||
|
||||
def set_default_source_type
|
||||
return if source_type.present?
|
||||
|
||||
self.source_type = if file.attached?
|
||||
'pdf_upload'
|
||||
else
|
||||
'url'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -22,8 +22,4 @@ class Captain::AssistantPolicy < ApplicationPolicy
|
||||
def playground?
|
||||
true
|
||||
end
|
||||
|
||||
def upload_pdf?
|
||||
@account_user.administrator?
|
||||
end
|
||||
end
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
module Captain::Llm::Concerns::PdfResponsesApi
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
RESPONSES_API_URL = 'https://api.openai.com'.freeze
|
||||
RESPONSES_API_ENDPOINT = '/v1/responses'.freeze
|
||||
API_KEY_CONFIG_NAME = 'CAPTAIN_OPEN_AI_API_KEY'.freeze
|
||||
|
||||
private
|
||||
|
||||
def process_pdf_with_responses_api
|
||||
request_body = build_pdf_request_body
|
||||
response = call_responses_api(request_body)
|
||||
handle_api_response(response)
|
||||
end
|
||||
|
||||
def build_pdf_request_body
|
||||
{
|
||||
model: @model,
|
||||
input: [
|
||||
build_system_input,
|
||||
build_user_input_for_pdf
|
||||
],
|
||||
text: {
|
||||
format: { type: 'json_object' }
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def build_system_input
|
||||
{
|
||||
role: 'system',
|
||||
content: Captain::Llm::SystemPromptsService.faq_generator
|
||||
}
|
||||
end
|
||||
|
||||
def build_user_input_for_pdf
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'input_file', file_id: metadata[:openai_file_id] },
|
||||
{ type: 'input_text', text: build_analysis_prompt }
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
def build_analysis_prompt
|
||||
if metadata[:processing_instruction]
|
||||
"#{pdf_analysis_prompt} #{metadata[:processing_instruction]}"
|
||||
else
|
||||
pdf_analysis_prompt
|
||||
end
|
||||
end
|
||||
|
||||
def pdf_analysis_prompt
|
||||
'Please analyze this PDF document and generate comprehensive FAQs based on its content.'
|
||||
end
|
||||
|
||||
def call_responses_api(request_body)
|
||||
responses_api_connection.post(RESPONSES_API_ENDPOINT) do |req|
|
||||
req.headers['Authorization'] = "Bearer #{api_key}"
|
||||
req.headers['Content-Type'] = 'application/json'
|
||||
req.body = request_body
|
||||
end
|
||||
end
|
||||
|
||||
def responses_api_connection
|
||||
@responses_api_connection ||= Faraday.new(url: RESPONSES_API_URL) do |faraday|
|
||||
faraday.request :json
|
||||
faraday.response :json
|
||||
faraday.adapter Faraday.default_adapter
|
||||
end
|
||||
end
|
||||
|
||||
def api_key
|
||||
@api_key ||= InstallationConfig.find_by!(name: API_KEY_CONFIG_NAME).value
|
||||
rescue ActiveRecord::RecordNotFound
|
||||
raise OpenAI::Error, "API key configuration not found: #{API_KEY_CONFIG_NAME}"
|
||||
end
|
||||
|
||||
def handle_api_response(response)
|
||||
return response.body if response.status == 200
|
||||
|
||||
error_message = extract_error_message(response)
|
||||
Rails.logger.error "OpenAI Responses API error: #{response.status} - #{error_message}"
|
||||
raise OpenAI::Error, "Responses API error: #{response.status} - #{error_message}"
|
||||
end
|
||||
|
||||
def extract_error_message(response)
|
||||
body = response.body
|
||||
return body unless body.is_a?(Hash)
|
||||
|
||||
body.dig('error', 'message') || body['error'] || body.to_s
|
||||
rescue StandardError
|
||||
response.body.to_s
|
||||
end
|
||||
|
||||
def extract_content_from_responses_api(response)
|
||||
output_message = Array(response['output']).first
|
||||
return nil unless output_message
|
||||
|
||||
content_item = output_message['content']&.find { |c| c['type'] == 'output_text' }
|
||||
content_item&.dig('text')
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,34 @@
|
||||
class Captain::Llm::PdfFaqGeneratorService < Captain::Llm::FaqGeneratorService
|
||||
include Captain::Llm::Concerns::PdfResponsesApi
|
||||
|
||||
def initialize(content_or_file_id, is_pdf_file: false, metadata: {})
|
||||
@is_pdf_file = is_pdf_file
|
||||
@metadata = metadata
|
||||
super(content_or_file_id)
|
||||
end
|
||||
|
||||
def generate
|
||||
return super unless pdf_processing?
|
||||
|
||||
validate_inputs!
|
||||
response = process_pdf_with_responses_api
|
||||
parse_response(response)
|
||||
rescue ArgumentError, OpenAI::Error
|
||||
[]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :is_pdf_file, :metadata
|
||||
|
||||
def pdf_processing?
|
||||
is_pdf_file && metadata[:openai_file_id]
|
||||
end
|
||||
|
||||
def validate_inputs!
|
||||
return unless is_pdf_file
|
||||
|
||||
raise ArgumentError, 'Missing file_id for PDF processing' if metadata[:openai_file_id].blank?
|
||||
raise ArgumentError, 'Invalid metadata format' unless metadata.is_a?(Hash)
|
||||
end
|
||||
end
|
||||
@@ -1,131 +0,0 @@
|
||||
module Captain::Tools::PdfContentChunkingConcern
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
private
|
||||
|
||||
def chunk_content(page_contents, max_chunk_size: self.class::MAX_CHUNK_SIZE)
|
||||
return [] if page_contents.blank?
|
||||
|
||||
all_chunks = []
|
||||
global_chunk_index = 0
|
||||
|
||||
page_contents.each do |page_content|
|
||||
page_chunks = split_content_into_chunks(page_content[:content], max_chunk_size)
|
||||
page_total_chunks = page_chunks.length
|
||||
|
||||
page_chunks.each_with_index do |chunk_content, page_chunk_index|
|
||||
global_chunk_index += 1
|
||||
all_chunks << build_chunk(chunk_content, page_content[:page_number], page_chunk_index + 1, page_total_chunks)
|
||||
end
|
||||
end
|
||||
|
||||
all_chunks
|
||||
end
|
||||
|
||||
def split_content_into_chunks(content, max_size)
|
||||
return [content] if content.length <= max_size
|
||||
|
||||
paragraphs = content.split(/\n\s*\n/)
|
||||
process_paragraphs_into_chunks(paragraphs, max_size, content)
|
||||
end
|
||||
|
||||
def process_paragraphs_into_chunks(paragraphs, max_size, original_content)
|
||||
chunks = []
|
||||
current_chunk = ''
|
||||
|
||||
paragraphs.each do |paragraph|
|
||||
chunks, current_chunk = process_single_paragraph(chunks, current_chunk, paragraph, max_size)
|
||||
end
|
||||
|
||||
finalize_chunks(chunks, current_chunk, original_content)
|
||||
end
|
||||
|
||||
def process_single_paragraph(chunks, current_chunk, paragraph, max_size)
|
||||
return handle_oversized_paragraph(chunks, current_chunk, paragraph, max_size) if paragraph.length > max_size
|
||||
return handle_paragraph_overflow(chunks, current_chunk, paragraph) if paragraph_causes_overflow?(current_chunk, paragraph, max_size)
|
||||
|
||||
combined = combine_paragraph_content(current_chunk, paragraph)
|
||||
[chunks, combined]
|
||||
end
|
||||
|
||||
def finalize_chunks(chunks, current_chunk, original_content)
|
||||
add_chunk_if_present(chunks, current_chunk)
|
||||
chunks.presence || [original_content]
|
||||
end
|
||||
|
||||
def split_paragraph_into_chunks(paragraph, max_size)
|
||||
sentences = paragraph.split(/(?<=[.!?])\s+/)
|
||||
process_sentences_into_chunks(sentences, max_size)
|
||||
end
|
||||
|
||||
def process_sentences_into_chunks(sentences, max_size)
|
||||
chunks = []
|
||||
current_chunk = ''
|
||||
|
||||
sentences.each do |sentence|
|
||||
chunks, current_chunk = process_single_sentence(chunks, current_chunk, sentence, max_size)
|
||||
end
|
||||
|
||||
add_chunk_if_present(chunks, current_chunk)
|
||||
chunks
|
||||
end
|
||||
|
||||
def process_single_sentence(chunks, current_chunk, sentence, max_size)
|
||||
return handle_oversized_sentence(chunks, current_chunk, sentence, max_size) if sentence.length > max_size
|
||||
return handle_sentence_overflow(chunks, current_chunk, sentence) if sentence_causes_overflow?(current_chunk, sentence, max_size)
|
||||
|
||||
combined = combine_sentence_content(current_chunk, sentence)
|
||||
[chunks, combined]
|
||||
end
|
||||
|
||||
def handle_oversized_paragraph(chunks, current_chunk, paragraph, max_size)
|
||||
add_chunk_if_present(chunks, current_chunk)
|
||||
chunks.concat(split_paragraph_into_chunks(paragraph, max_size))
|
||||
[chunks, '']
|
||||
end
|
||||
|
||||
def handle_paragraph_overflow(chunks, current_chunk, paragraph)
|
||||
add_chunk_if_present(chunks, current_chunk)
|
||||
[chunks, paragraph]
|
||||
end
|
||||
|
||||
def paragraph_causes_overflow?(current_chunk, paragraph, max_size)
|
||||
("#{current_chunk}\n\n#{paragraph}").length > max_size
|
||||
end
|
||||
|
||||
def combine_paragraph_content(current_chunk, paragraph)
|
||||
current_chunk.blank? ? paragraph : "#{current_chunk}\n\n#{paragraph}"
|
||||
end
|
||||
|
||||
def handle_oversized_sentence(chunks, current_chunk, sentence, max_size)
|
||||
add_chunk_if_present(chunks, current_chunk)
|
||||
chunks << sentence[0, max_size]
|
||||
[chunks, '']
|
||||
end
|
||||
|
||||
def handle_sentence_overflow(chunks, current_chunk, sentence)
|
||||
add_chunk_if_present(chunks, current_chunk)
|
||||
[chunks, sentence]
|
||||
end
|
||||
|
||||
def sentence_causes_overflow?(current_chunk, sentence, max_size)
|
||||
("#{current_chunk} #{sentence}").length > max_size
|
||||
end
|
||||
|
||||
def combine_sentence_content(current_chunk, sentence)
|
||||
current_chunk.blank? ? sentence : "#{current_chunk} #{sentence}"
|
||||
end
|
||||
|
||||
def add_chunk_if_present(chunks, chunk)
|
||||
chunks << chunk.strip if chunk.present?
|
||||
end
|
||||
|
||||
def build_chunk(content, page_number, chunk_index, total_chunks)
|
||||
{
|
||||
content: content,
|
||||
page_number: page_number,
|
||||
chunk_index: chunk_index,
|
||||
total_chunks: total_chunks
|
||||
}
|
||||
end
|
||||
end
|
||||
@@ -1,64 +0,0 @@
|
||||
require 'pdf-reader'
|
||||
|
||||
class Captain::Tools::PdfExtractionService
|
||||
include ActiveModel::Validations
|
||||
include Captain::Tools::PdfValidationConcern
|
||||
include Captain::Tools::PdfContentChunkingConcern
|
||||
include Captain::Tools::PdfExtractionService::SourceHandler
|
||||
include Captain::Tools::PdfExtractionService::BlobHandler
|
||||
include Captain::Tools::PdfExtractionService::TextProcessor
|
||||
|
||||
class ExtractionError < StandardError; end
|
||||
|
||||
attr_reader :pdf_source, :errors
|
||||
|
||||
# Fixed PDF processing limits
|
||||
MAX_PDF_SIZE = 25.megabytes
|
||||
MAX_CHUNK_SIZE = 10_000
|
||||
DOWNLOAD_TIMEOUT = 60 # Increased for larger file downloads
|
||||
|
||||
def initialize(pdf_source)
|
||||
@pdf_source = pdf_source
|
||||
@errors = []
|
||||
end
|
||||
|
||||
def perform
|
||||
return failure_response(['Invalid PDF source']) if pdf_source.blank?
|
||||
|
||||
validation_result = validate_pdf_source
|
||||
return validation_result unless validation_result[:success]
|
||||
|
||||
process_pdf_extraction
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def process_pdf_extraction
|
||||
content = extract_text
|
||||
return failure_response(['No text content found in PDF']) if content.blank?
|
||||
|
||||
chunked_content = chunk_content(content)
|
||||
log_extraction_success(chunked_content)
|
||||
|
||||
success_response(chunked_content)
|
||||
end
|
||||
|
||||
def log_extraction_success(chunked_content)
|
||||
total_chars = chunked_content.sum { |chunk| chunk[:content].length }
|
||||
Rails.logger.info "PDF extraction completed: #{chunked_content.length} chunks, #{total_chars} characters"
|
||||
end
|
||||
|
||||
def success_response(content)
|
||||
{
|
||||
success: true,
|
||||
content: content
|
||||
}
|
||||
end
|
||||
|
||||
def failure_response(errors)
|
||||
{
|
||||
success: false,
|
||||
errors: errors
|
||||
}
|
||||
end
|
||||
end
|
||||
@@ -1,49 +0,0 @@
|
||||
module Captain::Tools::PdfExtractionService::BlobHandler
|
||||
private
|
||||
|
||||
def active_storage_blob_url?
|
||||
pdf_source.include?('/rails/active_storage/blobs/') ||
|
||||
(pdf_source.include?('blob') && pdf_source.include?('pdf'))
|
||||
end
|
||||
|
||||
def extract_from_active_storage_blob
|
||||
blob = find_blob_from_url
|
||||
return extract_blob_content(blob) if blob
|
||||
|
||||
Rails.logger.warn 'ActiveStorage blob not found, falling back to URL download'
|
||||
extract_from_url
|
||||
end
|
||||
|
||||
def extract_blob_content(blob)
|
||||
blob.open { |file| extract_from_file(file.path) }
|
||||
end
|
||||
|
||||
def find_blob_from_url
|
||||
blob_key = extract_blob_key_from_url
|
||||
return nil if blob_key.blank?
|
||||
|
||||
ActiveStorage::Blob.find_by(key: blob_key)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Error finding blob with key '#{blob_key}': #{e.message}"
|
||||
nil
|
||||
end
|
||||
|
||||
def extract_blob_key_from_url
|
||||
blob_key = if pdf_source.include?('/rails/active_storage/blobs/')
|
||||
extract_key_from_rails_path
|
||||
elsif pdf_source.include?('/blobs/')
|
||||
extract_key_from_blob_path
|
||||
end
|
||||
|
||||
blob_key&.split('?')&.first # Remove query parameters
|
||||
end
|
||||
|
||||
def extract_key_from_rails_path
|
||||
parts = pdf_source.split('/rails/active_storage/blobs/').last.split('/')
|
||||
parts.length > 1 && parts[0] != 'redirect' ? parts[0] : parts[1]
|
||||
end
|
||||
|
||||
def extract_key_from_blob_path
|
||||
pdf_source.split('/blobs/').last.split('/').first
|
||||
end
|
||||
end
|
||||
@@ -1,67 +0,0 @@
|
||||
module Captain::Tools::PdfExtractionService::SourceHandler
|
||||
private
|
||||
|
||||
def determine_source_type
|
||||
return :url if pdf_source.is_a?(String) && pdf_source.start_with?('http')
|
||||
return :uploaded_file if pdf_source.respond_to?(:tempfile)
|
||||
return :active_storage_attachment if pdf_source.is_a?(ActiveStorage::Attached::One)
|
||||
|
||||
:file_path
|
||||
end
|
||||
|
||||
def extract_text
|
||||
case determine_source_type
|
||||
when :url
|
||||
extract_from_url
|
||||
when :uploaded_file
|
||||
extract_from_uploaded_file
|
||||
when :active_storage_attachment
|
||||
extract_from_attachment
|
||||
else
|
||||
extract_from_file_path
|
||||
end
|
||||
end
|
||||
|
||||
def extract_from_url
|
||||
return extract_from_active_storage_blob if active_storage_blob_url?
|
||||
|
||||
temp_file = Down.download(
|
||||
pdf_source,
|
||||
max_size: self.class::MAX_PDF_SIZE,
|
||||
open_timeout: self.class::DOWNLOAD_TIMEOUT,
|
||||
read_timeout: self.class::DOWNLOAD_TIMEOUT
|
||||
)
|
||||
|
||||
begin
|
||||
result = extract_from_file(temp_file.path)
|
||||
ensure
|
||||
temp_file.close
|
||||
temp_file.unlink
|
||||
end
|
||||
|
||||
result
|
||||
end
|
||||
|
||||
def extract_from_uploaded_file
|
||||
extract_from_file(pdf_source.tempfile.path)
|
||||
end
|
||||
|
||||
def extract_from_file_path
|
||||
extract_from_file(pdf_source)
|
||||
end
|
||||
|
||||
def extract_from_attachment
|
||||
return failure_response(['No file attached']) unless pdf_source.attached?
|
||||
|
||||
pdf_source.open { |file| extract_from_file(file.path) }
|
||||
end
|
||||
|
||||
def pdf_source_type
|
||||
case pdf_source
|
||||
when String
|
||||
pdf_source.start_with?('http') ? 'URL' : 'file_path'
|
||||
else
|
||||
'uploaded_file'
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,52 +0,0 @@
|
||||
module Captain::Tools::PdfExtractionService::TextProcessor
|
||||
private
|
||||
|
||||
def extract_from_file(file_path)
|
||||
text_content = []
|
||||
|
||||
PDF::Reader.open(file_path) do |reader|
|
||||
reader.pages.each_with_index do |page, index|
|
||||
page_content = extract_page_content(page, index)
|
||||
text_content << page_content if page_content
|
||||
end
|
||||
end
|
||||
|
||||
text_content
|
||||
rescue PDF::Reader::MalformedPDFError => e
|
||||
Rails.logger.error "Malformed PDF error: #{e.message}"
|
||||
raise StandardError, 'Invalid or corrupted PDF file'
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "PDF extraction error: #{e.message}"
|
||||
raise StandardError, "Failed to extract text from PDF: #{e.message}"
|
||||
end
|
||||
|
||||
def extract_page_content(page, index)
|
||||
page_text = page.text
|
||||
return nil if page_text.blank?
|
||||
|
||||
cleaned_text = clean_text(page_text)
|
||||
return nil if cleaned_text.blank?
|
||||
|
||||
{ page_number: index + 1, content: cleaned_text }
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn "Failed to extract text from page #{index + 1}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
|
||||
def clean_text(text)
|
||||
# Remove form feeds and normalize whitespace
|
||||
cleaned = text.tr("\f", "\n")
|
||||
.gsub("\r\n", "\n")
|
||||
.tr("\r", "\n")
|
||||
.gsub(/\s+/, ' ')
|
||||
.gsub(/\n\s*\n\s*\n+/, "\n\n")
|
||||
.strip
|
||||
|
||||
# Remove common PDF artifacts
|
||||
cleaned = cleaned.gsub(/^\d+\s*$/, '') # Remove standalone page numbers
|
||||
.gsub(/^[\s\-_=]+$/, '') # Remove separator lines
|
||||
.strip
|
||||
|
||||
cleaned.presence
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,117 @@
|
||||
class Captain::Tools::PdfOpenaiService
|
||||
include ActiveModel::Validations
|
||||
|
||||
class ProcessingError < StandardError; end
|
||||
|
||||
attr_reader :pdf_source
|
||||
|
||||
MAX_PDF_SIZE = 25.megabytes
|
||||
OPENAI_PURPOSE = 'assistants'.freeze
|
||||
API_KEY_CONFIG = 'CAPTAIN_OPEN_AI_API_KEY'.freeze
|
||||
|
||||
def initialize(pdf_source)
|
||||
@pdf_source = pdf_source
|
||||
end
|
||||
|
||||
def perform
|
||||
return failure_response(['Invalid PDF source']) if pdf_source.blank?
|
||||
|
||||
file_response = upload_pdf_to_openai
|
||||
return file_response unless file_response[:success]
|
||||
|
||||
build_success_response(file_response[:file_id])
|
||||
rescue ProcessingError => e
|
||||
failure_response([e.message])
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "PDF processing error: #{e.message}"
|
||||
failure_response(['An unexpected error occurred while processing the PDF'])
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def openai_client
|
||||
@openai_client ||= OpenAI::Client.new(
|
||||
access_token: api_key,
|
||||
log_errors: Rails.env.development?
|
||||
)
|
||||
end
|
||||
|
||||
def api_key
|
||||
@api_key ||= InstallationConfig.find_by!(name: API_KEY_CONFIG).value
|
||||
rescue ActiveRecord::RecordNotFound
|
||||
raise ProcessingError, 'OpenAI API key not configured'
|
||||
end
|
||||
|
||||
def upload_pdf_to_openai
|
||||
file_content = prepare_pdf_content
|
||||
return failure_response(['Could not retrieve PDF content']) unless file_content
|
||||
|
||||
validate_pdf_size!(file_content)
|
||||
|
||||
response = openai_client.files.upload(
|
||||
parameters: {
|
||||
file: file_content,
|
||||
purpose: OPENAI_PURPOSE
|
||||
}
|
||||
)
|
||||
|
||||
{ success: true, file_id: response['id'] }
|
||||
rescue OpenAI::Error => e
|
||||
failure_response(["OpenAI upload failed: #{e.message}"])
|
||||
end
|
||||
|
||||
def prepare_pdf_content
|
||||
attachment = extract_attachment
|
||||
return nil unless attachment&.blob
|
||||
|
||||
blob = attachment.blob
|
||||
create_file_io(blob.download, blob.filename.to_s, blob.content_type)
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "Failed to prepare PDF content: #{e.message}"
|
||||
nil
|
||||
end
|
||||
|
||||
def extract_attachment
|
||||
return pdf_source unless pdf_source.is_a?(ActiveStorage::Attached::One)
|
||||
|
||||
pdf_source.attachment
|
||||
end
|
||||
|
||||
def validate_pdf_size!(file_content)
|
||||
size = file_content.size
|
||||
return if size <= MAX_PDF_SIZE
|
||||
|
||||
raise ProcessingError, "PDF size (#{(size / 1.megabyte).round(2)}MB) exceeds maximum allowed size (#{MAX_PDF_SIZE / 1.megabyte}MB)"
|
||||
end
|
||||
|
||||
def create_file_io(content, filename, content_type)
|
||||
StringIO.new(content).tap do |io|
|
||||
io.define_singleton_method(:path) { filename }
|
||||
io.define_singleton_method(:content_type) { content_type }
|
||||
end
|
||||
end
|
||||
|
||||
def build_success_response(file_id)
|
||||
success_response([{
|
||||
content: file_id,
|
||||
metadata: {
|
||||
openai_file_id: file_id,
|
||||
processing_type: 'direct_pdf'
|
||||
}
|
||||
}])
|
||||
end
|
||||
|
||||
def success_response(content)
|
||||
{
|
||||
success: true,
|
||||
content: content
|
||||
}
|
||||
end
|
||||
|
||||
def failure_response(errors)
|
||||
{
|
||||
success: false,
|
||||
errors: errors
|
||||
}
|
||||
end
|
||||
end
|
||||
@@ -1,35 +0,0 @@
|
||||
module Captain::Tools::PdfValidationConcern
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
private
|
||||
|
||||
def validate_pdf_source
|
||||
case determine_source_type
|
||||
when :url
|
||||
validate_url_format
|
||||
when :uploaded_file
|
||||
validate_file_type_and_size
|
||||
when :active_storage_attachment
|
||||
validate_attachment
|
||||
end
|
||||
|
||||
{ success: true }
|
||||
rescue StandardError => e
|
||||
Rails.logger.error "PDF validation failed: #{e.message}"
|
||||
{ success: false, errors: [e.message] }
|
||||
end
|
||||
|
||||
def validate_url_format
|
||||
URI.parse(pdf_source)
|
||||
end
|
||||
|
||||
def validate_file_type_and_size
|
||||
raise StandardError, 'Invalid file type' unless pdf_source.content_type == 'application/pdf'
|
||||
raise StandardError, 'File too large' if pdf_source.size > 25.megabytes
|
||||
end
|
||||
|
||||
def validate_attachment
|
||||
raise StandardError, 'No file attached' unless pdf_source.attached?
|
||||
raise StandardError, 'Invalid file type' unless pdf_source.content_type == 'application/pdf'
|
||||
end
|
||||
end
|
||||
@@ -1,167 +0,0 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::Documents::PdfExtractionJob, type: :job do
|
||||
let(:document) { create(:captain_document, external_link: 'https://example.com/document.pdf', source_type: 'pdf_upload') }
|
||||
let(:pdf_service) { instance_double(Captain::Tools::PdfExtractionService) }
|
||||
let(:pdf_content) do
|
||||
[
|
||||
{ content: 'PDF page 1 content', page_number: 1, chunk_index: 1, total_chunks: 2 },
|
||||
{ content: 'PDF page 2 content', page_number: 2, chunk_index: 2, total_chunks: 2 }
|
||||
]
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Captain::Tools::PdfExtractionService)
|
||||
.to receive(:new)
|
||||
.and_return(pdf_service)
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
context 'when document is not a PDF' do
|
||||
let(:web_document) { create(:captain_document, external_link: 'https://example.com/page.html') }
|
||||
|
||||
it 'returns early without processing' do
|
||||
expect(pdf_service).not_to receive(:perform)
|
||||
described_class.perform_now(web_document)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when document is a PDF' do
|
||||
it 'updates document status to in_progress' do
|
||||
allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content })
|
||||
allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later)
|
||||
|
||||
expect(document).to receive(:update).with(status: 'in_progress').once
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
it 'uses correct PDF source for extraction' do
|
||||
allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content })
|
||||
allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later)
|
||||
|
||||
expect(Captain::Tools::PdfExtractionService)
|
||||
.to receive(:new)
|
||||
.with(document.external_link)
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
context 'when document has attached file' do
|
||||
let(:attached_file) { instance_double(ActiveStorage::Attached::One, attached?: true) }
|
||||
|
||||
before do
|
||||
allow(document).to receive(:file).and_return(attached_file)
|
||||
end
|
||||
|
||||
it 'uses attached file for extraction' do
|
||||
allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content })
|
||||
allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later)
|
||||
|
||||
expect(Captain::Tools::PdfExtractionService)
|
||||
.to receive(:new)
|
||||
.with(document.file)
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when extraction succeeds' do
|
||||
before do
|
||||
allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content })
|
||||
end
|
||||
|
||||
it 'updates document with first page content only' do
|
||||
expect(document).to receive(:update!).with(
|
||||
content: 'PDF page 1 content',
|
||||
status: 'available'
|
||||
)
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
it 'resets previous responses once at the beginning' do
|
||||
allow(document).to receive(:update!)
|
||||
allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later)
|
||||
|
||||
expect(document.responses).to receive(:destroy_all).once
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
it 'enqueues PdfExtractionParserJob for each content chunk' do
|
||||
allow(document).to receive(:update!)
|
||||
|
||||
pdf_content.each do |chunk|
|
||||
expect(Captain::Tools::PdfExtractionParserJob)
|
||||
.to receive(:perform_later)
|
||||
.with(
|
||||
assistant_id: document.assistant_id,
|
||||
pdf_content: chunk,
|
||||
document_id: document.id
|
||||
)
|
||||
end
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
it 'updates document status to in_progress initially' do
|
||||
allow(document).to receive(:update!)
|
||||
allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later)
|
||||
|
||||
expect(document).to receive(:update).with(status: 'in_progress').once
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
it 'logs successful extraction' do
|
||||
allow(document).to receive(:update!)
|
||||
allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later)
|
||||
|
||||
expect(Rails.logger).to receive(:info).with(/PDF extraction successful/).at_least(:once)
|
||||
expect(Rails.logger).to receive(:info).with(/chunks queued/).at_least(:once)
|
||||
allow(Rails.logger).to receive(:info) # Allow other logging calls
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when extraction fails' do
|
||||
before do
|
||||
allow(pdf_service).to receive(:perform).and_return({
|
||||
success: false,
|
||||
errors: ['Invalid PDF format', 'File corrupted']
|
||||
})
|
||||
end
|
||||
|
||||
it 'updates document status to available' do
|
||||
expect(document).to receive(:update).with(status: 'in_progress')
|
||||
expect(document).not_to receive(:update).with(status: 'available')
|
||||
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
|
||||
it 'logs the error' do
|
||||
expect(Rails.logger).not_to receive(:error)
|
||||
described_class.perform_now(document)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when extraction raises an exception' do
|
||||
before do
|
||||
allow(pdf_service).to receive(:perform).and_raise(Captain::Tools::PdfExtractionService::ExtractionError, 'Network error')
|
||||
end
|
||||
|
||||
it 'updates document status to available' do
|
||||
expect(document).to receive(:update).with(status: 'in_progress')
|
||||
expect(document).not_to receive(:update).with(status: 'available')
|
||||
|
||||
expect { described_class.perform_now(document) }.to raise_error(Captain::Tools::PdfExtractionService::ExtractionError)
|
||||
end
|
||||
|
||||
it 'logs the exception' do
|
||||
allow(Rails.logger).to receive(:error)
|
||||
expect { described_class.perform_now(document) }.to raise_error(Captain::Tools::PdfExtractionService::ExtractionError)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,167 +0,0 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::Tools::PdfExtractionParserJob, type: :job do
|
||||
let(:account) { create(:account, custom_attributes: { plan_name: 'startups' }) }
|
||||
let(:assistant) { create(:captain_assistant, account: account) }
|
||||
let!(:main_document) { create(:captain_document, assistant: assistant, account: account, content: 'Main document content') }
|
||||
let(:pdf_content) do
|
||||
{
|
||||
content: 'This is sample PDF content extracted from a document. It contains useful information for FAQ generation.',
|
||||
page_number: 2,
|
||||
chunk_index: 1,
|
||||
total_chunks: 1
|
||||
}
|
||||
end
|
||||
|
||||
before do
|
||||
# Mock usage limits
|
||||
allow(account).to receive(:usage_limits).and_return(
|
||||
captain: {
|
||||
documents: {
|
||||
current_available: 3
|
||||
}
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
context 'when document_id is provided and document exists' do
|
||||
it 'does not create new documents' do
|
||||
expect do
|
||||
described_class.new.perform(
|
||||
assistant_id: assistant.id,
|
||||
pdf_content: pdf_content,
|
||||
document_id: main_document.id
|
||||
)
|
||||
end.not_to change(Captain::Document, :count)
|
||||
end
|
||||
|
||||
it 'enqueues ResponseBuilderJob with combined content for the main document' do
|
||||
expected_content = "Main document content\n\n--- Additional Content (Page 2) Part 1 ---\n" \
|
||||
'This is sample PDF content extracted from a document. It contains useful information for FAQ generation.'
|
||||
|
||||
expect(Captain::Documents::ResponseBuilderJob)
|
||||
.to receive(:perform_later)
|
||||
.with(main_document, expected_content, skip_reset: true)
|
||||
|
||||
described_class.new.perform(
|
||||
assistant_id: assistant.id,
|
||||
pdf_content: pdf_content,
|
||||
document_id: main_document.id
|
||||
)
|
||||
end
|
||||
|
||||
it 'combines main document content with chunk content' do
|
||||
chunk_content = pdf_content.merge(page_number: 3, chunk_index: 2)
|
||||
expected_content = "Main document content\n\n--- Additional Content (Page 3) Part 2 ---\n" \
|
||||
'This is sample PDF content extracted from a document. It contains useful information for FAQ generation.'
|
||||
|
||||
expect(Captain::Documents::ResponseBuilderJob)
|
||||
.to receive(:perform_later)
|
||||
.with(main_document, expected_content, skip_reset: true)
|
||||
|
||||
described_class.new.perform(
|
||||
assistant_id: assistant.id,
|
||||
pdf_content: chunk_content,
|
||||
document_id: main_document.id
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when document_id is nil' do
|
||||
it 'returns early without processing' do
|
||||
expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later)
|
||||
|
||||
described_class.new.perform(
|
||||
assistant_id: assistant.id,
|
||||
pdf_content: pdf_content,
|
||||
document_id: nil
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when document_id is provided but document does not exist' do
|
||||
it 'logs error and returns without processing' do
|
||||
expect(Rails.logger).to receive(:error).with(/Document not found/)
|
||||
expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later)
|
||||
|
||||
described_class.new.perform(
|
||||
assistant_id: assistant.id,
|
||||
pdf_content: pdf_content,
|
||||
document_id: 99_999
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when assistant is not found' do
|
||||
it 'logs error and returns without processing' do
|
||||
expect(Rails.logger).to receive(:error).with(/Document not found/)
|
||||
expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later)
|
||||
|
||||
described_class.new.perform(
|
||||
assistant_id: 99_999,
|
||||
pdf_content: pdf_content,
|
||||
document_id: main_document.id
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
context 'when limits are exceeded' do
|
||||
before do
|
||||
# Set up account limits configuration to exceed limits
|
||||
captain_limits = {
|
||||
'startups' => {
|
||||
'captain_documents' => 1,
|
||||
'captain_responses' => 100
|
||||
}
|
||||
}
|
||||
|
||||
# First delete any existing config to avoid conflicts
|
||||
InstallationConfig.where(name: 'CAPTAIN_CLOUD_PLAN_LIMITS').destroy_all
|
||||
create(:installation_config, name: 'CAPTAIN_CLOUD_PLAN_LIMITS', value: captain_limits.to_json)
|
||||
|
||||
# Create more documents than the limit to exceed it
|
||||
create_list(:captain_document, 5, assistant: assistant, account: account, status: :available)
|
||||
account.update_document_usage
|
||||
account.reload
|
||||
end
|
||||
|
||||
it 'does not process content when limit exceeded' do
|
||||
expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later)
|
||||
|
||||
described_class.new.perform(
|
||||
assistant_id: assistant.id,
|
||||
pdf_content: pdf_content,
|
||||
document_id: main_document.id
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'private methods' do
|
||||
let(:job) { described_class.new }
|
||||
|
||||
describe '#limit_exceeded?' do
|
||||
it 'returns true when limit is zero' do
|
||||
allow(account).to receive(:usage_limits).and_return(
|
||||
captain: { documents: { current_available: 0 } }
|
||||
)
|
||||
expect(job.send(:limit_exceeded?, account)).to be true
|
||||
end
|
||||
|
||||
it 'returns true when limit is negative' do
|
||||
allow(account).to receive(:usage_limits).and_return(
|
||||
captain: { documents: { current_available: -1 } }
|
||||
)
|
||||
expect(job.send(:limit_exceeded?, account)).to be true
|
||||
end
|
||||
|
||||
it 'returns false when limit is positive' do
|
||||
allow(account).to receive(:usage_limits).and_return(
|
||||
captain: { documents: { current_available: 5 } }
|
||||
)
|
||||
expect(job.send(:limit_exceeded?, account)).to be false
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,217 +0,0 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe Captain::Tools::PdfExtractionService do
|
||||
let(:service) { described_class.new(pdf_source) }
|
||||
let(:sample_pdf_path) { Rails.root.join('spec/fixtures/files/valid_test.pdf') }
|
||||
|
||||
describe '#initialize' do
|
||||
context 'with valid PDF path' do
|
||||
let(:pdf_source) { sample_pdf_path.to_s }
|
||||
|
||||
it 'initializes with the PDF source' do
|
||||
expect(service.pdf_source).to eq(pdf_source)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with URL' do
|
||||
let(:pdf_source) { 'https://example.com/sample.pdf' }
|
||||
|
||||
it 'initializes with the PDF URL' do
|
||||
expect(service.pdf_source).to eq(pdf_source)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#perform' do
|
||||
context 'with blank PDF source' do
|
||||
let(:pdf_source) { nil }
|
||||
|
||||
it 'returns error for blank source' do
|
||||
result = service.perform
|
||||
expect(result[:success]).to be false
|
||||
expect(result[:errors]).to include('Invalid PDF source')
|
||||
end
|
||||
end
|
||||
|
||||
context 'with valid PDF file path' do
|
||||
let(:pdf_source) { sample_pdf_path.to_s }
|
||||
|
||||
before do
|
||||
# Ensure the sample PDF exists
|
||||
skip 'Sample PDF file not available for testing' unless File.exist?(sample_pdf_path)
|
||||
end
|
||||
|
||||
it 'handles PDF extraction gracefully and returns error for invalid files' do
|
||||
expect { service.perform }.to raise_error(StandardError, 'Invalid or corrupted PDF file')
|
||||
end
|
||||
end
|
||||
|
||||
context 'with malformed PDF' do
|
||||
let(:pdf_source) { sample_pdf_path.to_s }
|
||||
|
||||
it 'handles malformed PDF gracefully' do
|
||||
# Mock PDF::Reader to raise a MalformedPDFError
|
||||
allow(PDF::Reader).to receive(:open).and_raise(PDF::Reader::MalformedPDFError, 'PDF does not contain EOF marker')
|
||||
|
||||
expect { service.perform }.to raise_error(StandardError, 'Invalid or corrupted PDF file')
|
||||
end
|
||||
end
|
||||
|
||||
context 'with HTTP URL' do
|
||||
let(:pdf_source) { 'https://example.com/sample.pdf' }
|
||||
|
||||
it 'attempts to download and process PDF from URL' do
|
||||
temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s, close: nil, unlink: nil)
|
||||
allow(Down).to receive(:download).and_return(temp_file)
|
||||
|
||||
# Mock PDF reader to return sample content
|
||||
mock_page = instance_double(PDF::Reader::Page, text: 'Sample PDF content')
|
||||
mock_pages = [mock_page]
|
||||
mock_reader = instance_double(PDF::Reader, pages: mock_pages)
|
||||
allow(PDF::Reader).to receive(:open).and_yield(mock_reader)
|
||||
|
||||
result = service.perform
|
||||
|
||||
# The test should handle both success and controlled failures gracefully
|
||||
expect(result).to have_key(:success)
|
||||
if result[:success]
|
||||
expect(result[:content]).to be_an(Array)
|
||||
else
|
||||
expect(result[:errors]).to be_an(Array)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#extract_text (private method)' do
|
||||
let(:pdf_source) { sample_pdf_path.to_s }
|
||||
|
||||
before do
|
||||
skip 'Sample PDF file not available for testing' unless File.exist?(sample_pdf_path)
|
||||
end
|
||||
|
||||
it 'handles text extraction gracefully through perform method' do
|
||||
expect { service.perform }.to raise_error(StandardError, 'Invalid or corrupted PDF file')
|
||||
end
|
||||
end
|
||||
|
||||
describe 'private methods' do
|
||||
let(:pdf_source) { sample_pdf_path.to_s }
|
||||
|
||||
describe '#clean_text' do
|
||||
it 'cleans text content properly' do
|
||||
dirty_text = " \f Some text with \r\n line breaks \n\n\n and extra spaces "
|
||||
cleaned = service.send(:clean_text, dirty_text)
|
||||
|
||||
expect(cleaned).not_to include("\f")
|
||||
expect(cleaned).not_to include("\r")
|
||||
expect(cleaned).to include('Some text with')
|
||||
expect(cleaned.strip).to eq(cleaned)
|
||||
end
|
||||
|
||||
it 'returns nil for blank text' do
|
||||
cleaned = service.send(:clean_text, " \n\n ")
|
||||
expect(cleaned).to be_nil
|
||||
end
|
||||
end
|
||||
|
||||
describe '#chunk_content' do
|
||||
let(:page_contents) do
|
||||
[
|
||||
{ page_number: 1, content: 'Short content' },
|
||||
{ page_number: 2, content: 'A' * 3000 } # Long content that needs chunking
|
||||
]
|
||||
end
|
||||
|
||||
it 'chunks content appropriately' do
|
||||
chunks = service.send(:chunk_content, page_contents, max_chunk_size: 1000)
|
||||
|
||||
expect(chunks.length).to be >= 2 # Should have at least 2 chunks
|
||||
|
||||
# Check first chunk (short content)
|
||||
first_chunk = chunks.first
|
||||
expect(first_chunk[:page_number]).to eq(1)
|
||||
expect(first_chunk[:total_chunks]).to eq(1)
|
||||
|
||||
# Check that long content was processed
|
||||
long_content_chunks = chunks.select { |c| c[:page_number] == 2 }
|
||||
expect(long_content_chunks.length).to be >= 1
|
||||
|
||||
# Verify long content was split appropriately
|
||||
total_long_content_length = long_content_chunks.sum { |c| c[:content].length }
|
||||
expect(total_long_content_length).to be > 0
|
||||
end
|
||||
end
|
||||
|
||||
describe '#split_content_into_chunks' do
|
||||
it 'splits content by paragraphs first' do
|
||||
content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph."
|
||||
chunks = service.send(:split_content_into_chunks, content, 50)
|
||||
|
||||
expect(chunks.length).to be >= 2
|
||||
expect(chunks.join(' ')).to include('First paragraph')
|
||||
expect(chunks.join(' ')).to include('Second paragraph')
|
||||
end
|
||||
|
||||
it 'splits by sentences when paragraphs are too large' do
|
||||
long_paragraph = "#{('A' * 100)}. #{('B' * 100)}. #{('C' * 100)}."
|
||||
chunks = service.send(:split_content_into_chunks, long_paragraph, 150)
|
||||
|
||||
expect(chunks.length).to be > 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'file validation' do
|
||||
context 'with uploaded file object' do
|
||||
let(:uploaded_file) do
|
||||
temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s)
|
||||
instance_double(
|
||||
ActionDispatch::Http::UploadedFile,
|
||||
tempfile: temp_file,
|
||||
content_type: 'application/pdf',
|
||||
size: 1024
|
||||
)
|
||||
end
|
||||
let(:pdf_source) { uploaded_file }
|
||||
|
||||
it 'validates uploaded file properties' do
|
||||
expect { service.send(:validate_file_type_and_size) }.not_to raise_error
|
||||
end
|
||||
end
|
||||
|
||||
context 'with oversized file' do
|
||||
let(:uploaded_file) do
|
||||
temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s)
|
||||
instance_double(
|
||||
ActionDispatch::Http::UploadedFile,
|
||||
tempfile: temp_file,
|
||||
content_type: 'application/pdf',
|
||||
size: 30.megabytes
|
||||
)
|
||||
end
|
||||
let(:pdf_source) { uploaded_file }
|
||||
|
||||
it 'raises error for oversized file' do
|
||||
expect { service.send(:validate_file_type_and_size) }.to raise_error(/File too large/)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with invalid content type' do
|
||||
let(:uploaded_file) do
|
||||
temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s)
|
||||
instance_double(
|
||||
ActionDispatch::Http::UploadedFile,
|
||||
tempfile: temp_file,
|
||||
content_type: 'text/plain',
|
||||
size: 1024
|
||||
)
|
||||
end
|
||||
let(:pdf_source) { uploaded_file }
|
||||
|
||||
it 'raises error for invalid content type' do
|
||||
expect { service.send(:validate_file_type_and_size) }.to raise_error('Invalid file type')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user