diff --git a/Gemfile b/Gemfile index 96b011799..b4752c745 100644 --- a/Gemfile +++ b/Gemfile @@ -174,8 +174,6 @@ gem 'neighbor' gem 'pgvector' # Convert Website HTML to Markdown gem 'reverse_markdown' -# PDF text extraction -gem 'pdf-reader', '~> 2.0' gem 'iso-639' gem 'ruby-openai' diff --git a/Gemfile.lock b/Gemfile.lock index ca4871a87..147d7c606 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,7 +25,6 @@ GIT GEM remote: https://rubygems.org/ specs: - Ascii85 (2.0.1) actioncable (7.1.5.1) actionpack (= 7.1.5.1) activesupport (= 7.1.5.1) @@ -127,7 +126,6 @@ GEM jbuilder (~> 2) rails (>= 4.2, < 7.2) selectize-rails (~> 0.6) - afm (0.2.2) annotate (3.2.0) activerecord (>= 3.2, < 8.0) rake (>= 10.4, < 14.0) @@ -367,7 +365,6 @@ GEM hana (1.3.7) hash_diff (1.1.1) hashdiff (1.1.0) - hashery (2.1.2) hashie (5.0.0) html2text (0.4.0) nokogiri (>= 1.0, < 2.0) @@ -557,12 +554,6 @@ GEM parser (3.3.8.0) ast (~> 2.4.1) racc - pdf-reader (2.14.1) - Ascii85 (>= 1.0, < 3.0, != 2.0.0) - afm (~> 0.2.1) - hashery (~> 2.0) - ruby-rc4 - ttfunk pg (1.5.3) pg_search (2.3.6) activerecord (>= 5.2) @@ -720,7 +711,6 @@ GEM faraday (>= 1) faraday-multipart (>= 1) ruby-progressbar (1.13.0) - ruby-rc4 (0.1.5) ruby-vips (2.1.4) ffi (~> 1.12) ruby2_keywords (0.0.5) @@ -829,8 +819,6 @@ GEM i18n timeout (0.4.3) trailblazer-option (0.1.2) - ttfunk (1.8.0) - bigdecimal (~> 3.1) twilio-ruby (5.77.0) faraday (>= 0.9, < 3.0) jwt (>= 1.5, < 3.0) @@ -975,7 +963,6 @@ DEPENDENCIES omniauth-google-oauth2 (>= 1.1.3) omniauth-oauth2 omniauth-rails_csrf_protection (~> 1.0, >= 1.0.2) - pdf-reader (~> 2.0) pg pg_search pgvector diff --git a/db/migrate/20250702075600_add_pdf_support_to_captain_documents.rb b/db/migrate/20250702075600_add_pdf_support_to_captain_documents.rb deleted file mode 100644 index bc2b21c0e..000000000 --- a/db/migrate/20250702075600_add_pdf_support_to_captain_documents.rb +++ /dev/null @@ -1,7 +0,0 @@ -class AddPdfSupportToCaptainDocuments < ActiveRecord::Migration[7.1] - def change - add_column :captain_documents, :source_type, :string, default: 'url' - - add_index :captain_documents, :source_type - end -end \ No newline at end of file diff --git a/enterprise/app/controllers/api/v1/accounts/captain/documents_controller.rb b/enterprise/app/controllers/api/v1/accounts/captain/documents_controller.rb index 8897e1cdc..f071a92f1 100644 --- a/enterprise/app/controllers/api/v1/accounts/captain/documents_controller.rb +++ b/enterprise/app/controllers/api/v1/accounts/captain/documents_controller.rb @@ -34,8 +34,7 @@ class Api::V1::Accounts::Captain::DocumentsController < Api::V1::Accounts::BaseC @document = @assistant.documents.build( name: pdf_params[:pdf_document].original_filename, - external_link: "pdf_upload_#{SecureRandom.hex(8)}.pdf", - source_type: 'pdf_upload' + external_link: "pdf_upload_#{SecureRandom.hex(8)}.pdf" ) @document.file.attach(pdf_params[:pdf_document]) @document.save! diff --git a/enterprise/app/jobs/captain/documents/crawl_job.rb b/enterprise/app/jobs/captain/documents/crawl_job.rb index bf15b8845..e8cfbba07 100644 --- a/enterprise/app/jobs/captain/documents/crawl_job.rb +++ b/enterprise/app/jobs/captain/documents/crawl_job.rb @@ -14,7 +14,7 @@ class Captain::Documents::CrawlJob < ApplicationJob def pdf_document?(document) return false if document.nil? - document.source_type == 'pdf_upload' + document.pdf_document? end private diff --git a/enterprise/app/jobs/captain/documents/pdf_extraction_job.rb b/enterprise/app/jobs/captain/documents/pdf_extraction_job.rb index 97deb4fc1..d502c0174 100644 --- a/enterprise/app/jobs/captain/documents/pdf_extraction_job.rb +++ b/enterprise/app/jobs/captain/documents/pdf_extraction_job.rb @@ -4,12 +4,18 @@ class Captain::Documents::PdfExtractionJob < ApplicationJob queue_as :low + # This job runs in parallel with S3 upload + # ActiveStorage handles S3 upload asynchronously + # We process the PDF independently + def perform(document) return unless document.pdf_document? initialize_document_processing(document) result = extract_pdf_content(document) process_extraction_result(document, result) + rescue StandardError => e + handle_extraction_failure(document, e) end private @@ -19,57 +25,58 @@ class Captain::Documents::PdfExtractionJob < ApplicationJob end def extract_pdf_content(document) - pdf_source = document.file.attached? ? document.file : document.external_link - pdf_extraction_service = Captain::Tools::PdfExtractionService.new(pdf_source) - pdf_extraction_service.perform + pdf_source = document.file + + # Always use OpenAI direct PDF processing + pdf_service = Captain::Tools::PdfOpenaiService.new(pdf_source) + pdf_service.perform end def process_extraction_result(document, result) - return unless result[:success] && result[:content].present? - - process_pdf_content_chunks(document, result[:content]) + if result[:success] && result[:content].present? + process_pdf_content(document, result[:content]) + else + handle_pdf_extraction_failure(document, result) + end end - def process_pdf_content_chunks(document, content_chunks) - Rails.logger.info "PDF extraction successful for document #{document.id}: #{content_chunks.length} chunks will be processed" + def process_pdf_content(document, content) + # Content is an array with single item for direct PDF processing + pdf_data = content.first || {} + metadata = pdf_data[:metadata] - # Update main document with first page content only - first_page_content = content_chunks.first&.dig(:content) || '' + if metadata && metadata[:processing_type] == 'direct_pdf' + process_direct_pdf_content(document, metadata) + else + document.update!( + content: pdf_data[:content] || '', + status: 'available' + ) + end + end + + def process_direct_pdf_content(document, metadata) + # Store the OpenAI file_id in the document document.update!( - content: first_page_content, + content: metadata[:openai_file_id], status: 'available' ) - # Reset previous responses once at the beginning to avoid race conditions + # Process the PDF directly document.responses.destroy_all - - # Process each chunk separately but link FAQs to main document - content_chunks.each_with_index do |content_chunk, index| - log_chunk_queueing(document, content_chunk, index, content_chunks.length) - queue_pdf_chunk_job(document, content_chunk) - end - - Rails.logger.info "All #{content_chunks.length} chunks queued for processing for document #{document.id}" - end - - def log_chunk_queueing(document, content_chunk, index, total_chunks) - page_num = content_chunk[:page_number] - content_length = content_chunk[:content].length - Rails.logger.info "Queueing chunk #{index + 1}/#{total_chunks} for document #{document.id} " \ - "(Page #{page_num}, #{content_length} chars)" - end - - def queue_pdf_chunk_job(document, content_chunk) - Captain::Tools::PdfExtractionParserJob.perform_later( - assistant_id: document.assistant_id, - pdf_content: content_chunk, - document_id: document.id + Captain::Documents::ResponseBuilderJob.perform_later( + document, + metadata[:openai_file_id], + skip_reset: false, + metadata: metadata ) end - def handle_pdf_extraction_failure(document, result) - error_message = result[:errors]&.join(', ') || 'Failed to extract text from PDF' - Rails.logger.error "PDF extraction failed for document #{document.id}: #{error_message}" + def handle_pdf_extraction_failure(document, _result) + document.update(status: 'available') + end + + def handle_extraction_failure(document, _error) document.update(status: 'available') end end \ No newline at end of file diff --git a/enterprise/app/jobs/captain/documents/response_builder_job.rb b/enterprise/app/jobs/captain/documents/response_builder_job.rb index 1a854671f..f62bf0c5b 100644 --- a/enterprise/app/jobs/captain/documents/response_builder_job.rb +++ b/enterprise/app/jobs/captain/documents/response_builder_job.rb @@ -1,17 +1,28 @@ class Captain::Documents::ResponseBuilderJob < ApplicationJob queue_as :low - def perform(document, full_content = nil, skip_reset: false) + def perform(document, full_content = nil, skip_reset: false, metadata: {}) # Use full content for FAQ generation if provided (for PDFs), otherwise use document.content content_for_faqs = full_content || document.content # Skip processing if no content available return if content_for_faqs.blank? - # Only reset responses if not explicitly skipped (for PDF chunks processing) + # Only reset responses if not explicitly skipped reset_previous_responses(document) unless skip_reset - faqs = Captain::Llm::FaqGeneratorService.new(content_for_faqs).generate + # Check if this is a direct PDF processing + faqs = if metadata[:processing_type] == 'direct_pdf' && metadata[:openai_file_id] + Captain::Llm::PdfFaqGeneratorService.new( + content_for_faqs, + is_pdf_file: true, + metadata: metadata + ).generate + # NOTE: Not deleting the file_id - keeping it for future use + else + Captain::Llm::FaqGeneratorService.new(content_for_faqs).generate + end + faqs.each do |faq| create_response(faq, document) end diff --git a/enterprise/app/jobs/captain/tools/pdf_extraction_parser_job.rb b/enterprise/app/jobs/captain/tools/pdf_extraction_parser_job.rb deleted file mode 100644 index c9ebfeb83..000000000 --- a/enterprise/app/jobs/captain/tools/pdf_extraction_parser_job.rb +++ /dev/null @@ -1,51 +0,0 @@ -class Captain::Tools::PdfExtractionParserJob < ApplicationJob - queue_as :low - - def perform(assistant_id:, pdf_content:, document_id: nil) - assistant = Captain::Assistant.find(assistant_id) - main_document = find_main_document(document_id) - - return unless can_process_content?(pdf_content[:content], assistant.account, main_document) - - enqueue_response_builder_job_for_chunk(main_document, pdf_content) - rescue ActiveRecord::RecordNotFound => e - Rails.logger.error "PDF parser job failed - Document not found: #{e.message}" - rescue Captain::Document::LimitExceededError => e - Rails.logger.info "PDF parser job stopped - Document limit exceeded: #{e.message}" - end - - private - - def find_main_document(document_id) - return unless document_id - - Captain::Document.find(document_id) - end - - def can_process_content?(content, account, main_document) - main_document && should_process_content?(content, account) - end - - def enqueue_response_builder_job_for_chunk(main_document, pdf_content) - # Create a context string for better AI processing that includes page info - page_info = pdf_content[:page_number] ? " (Page #{pdf_content[:page_number]})" : '' - chunk_info = pdf_content[:chunk_index] ? " Part #{pdf_content[:chunk_index]}" : '' - - # Combine main document content with chunk content for comprehensive FAQ generation - context_content = "#{main_document.content}\n\n--- Additional Content#{page_info}#{chunk_info} ---\n#{pdf_content[:content]}" - - # Use the ResponseBuilderJob with full_content parameter to generate FAQs - # This will link all FAQs to the main document while using the chunk content for AI processing - # Skip reset since it's already done in the main PDF extraction job - Captain::Documents::ResponseBuilderJob.perform_later(main_document, context_content, skip_reset: true) - end - - def should_process_content?(content, account) - content.present? && !limit_exceeded?(account) - end - - def limit_exceeded?(account) - limits = account.usage_limits.dig(:captain, :documents) - limits && limits[:current_available].to_i <= 0 - end -end \ No newline at end of file diff --git a/enterprise/app/models/captain/document.rb b/enterprise/app/models/captain/document.rb index 81819637f..03cf3c42d 100644 --- a/enterprise/app/models/captain/document.rb +++ b/enterprise/app/models/captain/document.rb @@ -6,7 +6,6 @@ # content :text # external_link :string not null # name :string -# source_type :string default("url") # status :integer default("in_progress"), not null # created_at :datetime not null # updated_at :datetime not null @@ -18,7 +17,6 @@ # index_captain_documents_on_account_id (account_id) # index_captain_documents_on_assistant_id (assistant_id) # index_captain_documents_on_assistant_id_and_external_link (assistant_id,external_link) UNIQUE -# index_captain_documents_on_source_type (source_type) # index_captain_documents_on_status (status) # class Captain::Document < ApplicationRecord @@ -33,9 +31,7 @@ class Captain::Document < ApplicationRecord validates :external_link, presence: true validates :external_link, uniqueness: { scope: :assistant_id } validates :content, length: { maximum: 400_000 } - validates :source_type, inclusion: { in: %w[url pdf_upload] }, allow_blank: true before_validation :ensure_account_id - before_validation :set_default_source_type enum status: { in_progress: 0, @@ -53,7 +49,7 @@ class Captain::Document < ApplicationRecord scope :for_assistant, ->(assistant_id) { where(assistant_id: assistant_id) } def pdf_document? - source_type == 'pdf_upload' || file.attached? + file.attached? || external_link&.match?(/\.pdf$/i) end private @@ -84,14 +80,4 @@ class Captain::Document < ApplicationRecord limits = account.usage_limits[:captain][:documents] raise LimitExceededError, 'Document limit exceeded' unless limits[:current_available].positive? end - - def set_default_source_type - return if source_type.present? - - self.source_type = if file.attached? - 'pdf_upload' - else - 'url' - end - end end diff --git a/enterprise/app/policies/captain/assistant_policy.rb b/enterprise/app/policies/captain/assistant_policy.rb index f9ba16edc..2a03fc0ea 100644 --- a/enterprise/app/policies/captain/assistant_policy.rb +++ b/enterprise/app/policies/captain/assistant_policy.rb @@ -22,8 +22,4 @@ class Captain::AssistantPolicy < ApplicationPolicy def playground? true end - - def upload_pdf? - @account_user.administrator? - end end diff --git a/enterprise/app/services/captain/llm/concerns/pdf_responses_api.rb b/enterprise/app/services/captain/llm/concerns/pdf_responses_api.rb new file mode 100644 index 000000000..9c996d292 --- /dev/null +++ b/enterprise/app/services/captain/llm/concerns/pdf_responses_api.rb @@ -0,0 +1,104 @@ +module Captain::Llm::Concerns::PdfResponsesApi + extend ActiveSupport::Concern + + RESPONSES_API_URL = 'https://api.openai.com'.freeze + RESPONSES_API_ENDPOINT = '/v1/responses'.freeze + API_KEY_CONFIG_NAME = 'CAPTAIN_OPEN_AI_API_KEY'.freeze + + private + + def process_pdf_with_responses_api + request_body = build_pdf_request_body + response = call_responses_api(request_body) + handle_api_response(response) + end + + def build_pdf_request_body + { + model: @model, + input: [ + build_system_input, + build_user_input_for_pdf + ], + text: { + format: { type: 'json_object' } + } + } + end + + def build_system_input + { + role: 'system', + content: Captain::Llm::SystemPromptsService.faq_generator + } + end + + def build_user_input_for_pdf + { + role: 'user', + content: [ + { type: 'input_file', file_id: metadata[:openai_file_id] }, + { type: 'input_text', text: build_analysis_prompt } + ] + } + end + + def build_analysis_prompt + if metadata[:processing_instruction] + "#{pdf_analysis_prompt} #{metadata[:processing_instruction]}" + else + pdf_analysis_prompt + end + end + + def pdf_analysis_prompt + 'Please analyze this PDF document and generate comprehensive FAQs based on its content.' + end + + def call_responses_api(request_body) + responses_api_connection.post(RESPONSES_API_ENDPOINT) do |req| + req.headers['Authorization'] = "Bearer #{api_key}" + req.headers['Content-Type'] = 'application/json' + req.body = request_body + end + end + + def responses_api_connection + @responses_api_connection ||= Faraday.new(url: RESPONSES_API_URL) do |faraday| + faraday.request :json + faraday.response :json + faraday.adapter Faraday.default_adapter + end + end + + def api_key + @api_key ||= InstallationConfig.find_by!(name: API_KEY_CONFIG_NAME).value + rescue ActiveRecord::RecordNotFound + raise OpenAI::Error, "API key configuration not found: #{API_KEY_CONFIG_NAME}" + end + + def handle_api_response(response) + return response.body if response.status == 200 + + error_message = extract_error_message(response) + Rails.logger.error "OpenAI Responses API error: #{response.status} - #{error_message}" + raise OpenAI::Error, "Responses API error: #{response.status} - #{error_message}" + end + + def extract_error_message(response) + body = response.body + return body unless body.is_a?(Hash) + + body.dig('error', 'message') || body['error'] || body.to_s + rescue StandardError + response.body.to_s + end + + def extract_content_from_responses_api(response) + output_message = Array(response['output']).first + return nil unless output_message + + content_item = output_message['content']&.find { |c| c['type'] == 'output_text' } + content_item&.dig('text') + end +end \ No newline at end of file diff --git a/enterprise/app/services/captain/llm/pdf_faq_generator_service.rb b/enterprise/app/services/captain/llm/pdf_faq_generator_service.rb new file mode 100644 index 000000000..0e2043cc5 --- /dev/null +++ b/enterprise/app/services/captain/llm/pdf_faq_generator_service.rb @@ -0,0 +1,34 @@ +class Captain::Llm::PdfFaqGeneratorService < Captain::Llm::FaqGeneratorService + include Captain::Llm::Concerns::PdfResponsesApi + + def initialize(content_or_file_id, is_pdf_file: false, metadata: {}) + @is_pdf_file = is_pdf_file + @metadata = metadata + super(content_or_file_id) + end + + def generate + return super unless pdf_processing? + + validate_inputs! + response = process_pdf_with_responses_api + parse_response(response) + rescue ArgumentError, OpenAI::Error + [] + end + + private + + attr_reader :is_pdf_file, :metadata + + def pdf_processing? + is_pdf_file && metadata[:openai_file_id] + end + + def validate_inputs! + return unless is_pdf_file + + raise ArgumentError, 'Missing file_id for PDF processing' if metadata[:openai_file_id].blank? + raise ArgumentError, 'Invalid metadata format' unless metadata.is_a?(Hash) + end +end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_content_chunking_concern.rb b/enterprise/app/services/captain/tools/pdf_content_chunking_concern.rb deleted file mode 100644 index 163d5a4dc..000000000 --- a/enterprise/app/services/captain/tools/pdf_content_chunking_concern.rb +++ /dev/null @@ -1,131 +0,0 @@ -module Captain::Tools::PdfContentChunkingConcern - extend ActiveSupport::Concern - - private - - def chunk_content(page_contents, max_chunk_size: self.class::MAX_CHUNK_SIZE) - return [] if page_contents.blank? - - all_chunks = [] - global_chunk_index = 0 - - page_contents.each do |page_content| - page_chunks = split_content_into_chunks(page_content[:content], max_chunk_size) - page_total_chunks = page_chunks.length - - page_chunks.each_with_index do |chunk_content, page_chunk_index| - global_chunk_index += 1 - all_chunks << build_chunk(chunk_content, page_content[:page_number], page_chunk_index + 1, page_total_chunks) - end - end - - all_chunks - end - - def split_content_into_chunks(content, max_size) - return [content] if content.length <= max_size - - paragraphs = content.split(/\n\s*\n/) - process_paragraphs_into_chunks(paragraphs, max_size, content) - end - - def process_paragraphs_into_chunks(paragraphs, max_size, original_content) - chunks = [] - current_chunk = '' - - paragraphs.each do |paragraph| - chunks, current_chunk = process_single_paragraph(chunks, current_chunk, paragraph, max_size) - end - - finalize_chunks(chunks, current_chunk, original_content) - end - - def process_single_paragraph(chunks, current_chunk, paragraph, max_size) - return handle_oversized_paragraph(chunks, current_chunk, paragraph, max_size) if paragraph.length > max_size - return handle_paragraph_overflow(chunks, current_chunk, paragraph) if paragraph_causes_overflow?(current_chunk, paragraph, max_size) - - combined = combine_paragraph_content(current_chunk, paragraph) - [chunks, combined] - end - - def finalize_chunks(chunks, current_chunk, original_content) - add_chunk_if_present(chunks, current_chunk) - chunks.presence || [original_content] - end - - def split_paragraph_into_chunks(paragraph, max_size) - sentences = paragraph.split(/(?<=[.!?])\s+/) - process_sentences_into_chunks(sentences, max_size) - end - - def process_sentences_into_chunks(sentences, max_size) - chunks = [] - current_chunk = '' - - sentences.each do |sentence| - chunks, current_chunk = process_single_sentence(chunks, current_chunk, sentence, max_size) - end - - add_chunk_if_present(chunks, current_chunk) - chunks - end - - def process_single_sentence(chunks, current_chunk, sentence, max_size) - return handle_oversized_sentence(chunks, current_chunk, sentence, max_size) if sentence.length > max_size - return handle_sentence_overflow(chunks, current_chunk, sentence) if sentence_causes_overflow?(current_chunk, sentence, max_size) - - combined = combine_sentence_content(current_chunk, sentence) - [chunks, combined] - end - - def handle_oversized_paragraph(chunks, current_chunk, paragraph, max_size) - add_chunk_if_present(chunks, current_chunk) - chunks.concat(split_paragraph_into_chunks(paragraph, max_size)) - [chunks, ''] - end - - def handle_paragraph_overflow(chunks, current_chunk, paragraph) - add_chunk_if_present(chunks, current_chunk) - [chunks, paragraph] - end - - def paragraph_causes_overflow?(current_chunk, paragraph, max_size) - ("#{current_chunk}\n\n#{paragraph}").length > max_size - end - - def combine_paragraph_content(current_chunk, paragraph) - current_chunk.blank? ? paragraph : "#{current_chunk}\n\n#{paragraph}" - end - - def handle_oversized_sentence(chunks, current_chunk, sentence, max_size) - add_chunk_if_present(chunks, current_chunk) - chunks << sentence[0, max_size] - [chunks, ''] - end - - def handle_sentence_overflow(chunks, current_chunk, sentence) - add_chunk_if_present(chunks, current_chunk) - [chunks, sentence] - end - - def sentence_causes_overflow?(current_chunk, sentence, max_size) - ("#{current_chunk} #{sentence}").length > max_size - end - - def combine_sentence_content(current_chunk, sentence) - current_chunk.blank? ? sentence : "#{current_chunk} #{sentence}" - end - - def add_chunk_if_present(chunks, chunk) - chunks << chunk.strip if chunk.present? - end - - def build_chunk(content, page_number, chunk_index, total_chunks) - { - content: content, - page_number: page_number, - chunk_index: chunk_index, - total_chunks: total_chunks - } - end -end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_extraction_service.rb b/enterprise/app/services/captain/tools/pdf_extraction_service.rb deleted file mode 100644 index e6ccd3e4f..000000000 --- a/enterprise/app/services/captain/tools/pdf_extraction_service.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'pdf-reader' - -class Captain::Tools::PdfExtractionService - include ActiveModel::Validations - include Captain::Tools::PdfValidationConcern - include Captain::Tools::PdfContentChunkingConcern - include Captain::Tools::PdfExtractionService::SourceHandler - include Captain::Tools::PdfExtractionService::BlobHandler - include Captain::Tools::PdfExtractionService::TextProcessor - - class ExtractionError < StandardError; end - - attr_reader :pdf_source, :errors - - # Fixed PDF processing limits - MAX_PDF_SIZE = 25.megabytes - MAX_CHUNK_SIZE = 10_000 - DOWNLOAD_TIMEOUT = 60 # Increased for larger file downloads - - def initialize(pdf_source) - @pdf_source = pdf_source - @errors = [] - end - - def perform - return failure_response(['Invalid PDF source']) if pdf_source.blank? - - validation_result = validate_pdf_source - return validation_result unless validation_result[:success] - - process_pdf_extraction - end - - private - - def process_pdf_extraction - content = extract_text - return failure_response(['No text content found in PDF']) if content.blank? - - chunked_content = chunk_content(content) - log_extraction_success(chunked_content) - - success_response(chunked_content) - end - - def log_extraction_success(chunked_content) - total_chars = chunked_content.sum { |chunk| chunk[:content].length } - Rails.logger.info "PDF extraction completed: #{chunked_content.length} chunks, #{total_chars} characters" - end - - def success_response(content) - { - success: true, - content: content - } - end - - def failure_response(errors) - { - success: false, - errors: errors - } - end -end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_extraction_service/blob_handler.rb b/enterprise/app/services/captain/tools/pdf_extraction_service/blob_handler.rb deleted file mode 100644 index fc7f7d54e..000000000 --- a/enterprise/app/services/captain/tools/pdf_extraction_service/blob_handler.rb +++ /dev/null @@ -1,49 +0,0 @@ -module Captain::Tools::PdfExtractionService::BlobHandler - private - - def active_storage_blob_url? - pdf_source.include?('/rails/active_storage/blobs/') || - (pdf_source.include?('blob') && pdf_source.include?('pdf')) - end - - def extract_from_active_storage_blob - blob = find_blob_from_url - return extract_blob_content(blob) if blob - - Rails.logger.warn 'ActiveStorage blob not found, falling back to URL download' - extract_from_url - end - - def extract_blob_content(blob) - blob.open { |file| extract_from_file(file.path) } - end - - def find_blob_from_url - blob_key = extract_blob_key_from_url - return nil if blob_key.blank? - - ActiveStorage::Blob.find_by(key: blob_key) - rescue StandardError => e - Rails.logger.error "Error finding blob with key '#{blob_key}': #{e.message}" - nil - end - - def extract_blob_key_from_url - blob_key = if pdf_source.include?('/rails/active_storage/blobs/') - extract_key_from_rails_path - elsif pdf_source.include?('/blobs/') - extract_key_from_blob_path - end - - blob_key&.split('?')&.first # Remove query parameters - end - - def extract_key_from_rails_path - parts = pdf_source.split('/rails/active_storage/blobs/').last.split('/') - parts.length > 1 && parts[0] != 'redirect' ? parts[0] : parts[1] - end - - def extract_key_from_blob_path - pdf_source.split('/blobs/').last.split('/').first - end -end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_extraction_service/source_handler.rb b/enterprise/app/services/captain/tools/pdf_extraction_service/source_handler.rb deleted file mode 100644 index 39f2ebd4d..000000000 --- a/enterprise/app/services/captain/tools/pdf_extraction_service/source_handler.rb +++ /dev/null @@ -1,67 +0,0 @@ -module Captain::Tools::PdfExtractionService::SourceHandler - private - - def determine_source_type - return :url if pdf_source.is_a?(String) && pdf_source.start_with?('http') - return :uploaded_file if pdf_source.respond_to?(:tempfile) - return :active_storage_attachment if pdf_source.is_a?(ActiveStorage::Attached::One) - - :file_path - end - - def extract_text - case determine_source_type - when :url - extract_from_url - when :uploaded_file - extract_from_uploaded_file - when :active_storage_attachment - extract_from_attachment - else - extract_from_file_path - end - end - - def extract_from_url - return extract_from_active_storage_blob if active_storage_blob_url? - - temp_file = Down.download( - pdf_source, - max_size: self.class::MAX_PDF_SIZE, - open_timeout: self.class::DOWNLOAD_TIMEOUT, - read_timeout: self.class::DOWNLOAD_TIMEOUT - ) - - begin - result = extract_from_file(temp_file.path) - ensure - temp_file.close - temp_file.unlink - end - - result - end - - def extract_from_uploaded_file - extract_from_file(pdf_source.tempfile.path) - end - - def extract_from_file_path - extract_from_file(pdf_source) - end - - def extract_from_attachment - return failure_response(['No file attached']) unless pdf_source.attached? - - pdf_source.open { |file| extract_from_file(file.path) } - end - - def pdf_source_type - case pdf_source - when String - pdf_source.start_with?('http') ? 'URL' : 'file_path' - else - 'uploaded_file' - end - end -end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_extraction_service/text_processor.rb b/enterprise/app/services/captain/tools/pdf_extraction_service/text_processor.rb deleted file mode 100644 index 709a1977b..000000000 --- a/enterprise/app/services/captain/tools/pdf_extraction_service/text_processor.rb +++ /dev/null @@ -1,52 +0,0 @@ -module Captain::Tools::PdfExtractionService::TextProcessor - private - - def extract_from_file(file_path) - text_content = [] - - PDF::Reader.open(file_path) do |reader| - reader.pages.each_with_index do |page, index| - page_content = extract_page_content(page, index) - text_content << page_content if page_content - end - end - - text_content - rescue PDF::Reader::MalformedPDFError => e - Rails.logger.error "Malformed PDF error: #{e.message}" - raise StandardError, 'Invalid or corrupted PDF file' - rescue StandardError => e - Rails.logger.error "PDF extraction error: #{e.message}" - raise StandardError, "Failed to extract text from PDF: #{e.message}" - end - - def extract_page_content(page, index) - page_text = page.text - return nil if page_text.blank? - - cleaned_text = clean_text(page_text) - return nil if cleaned_text.blank? - - { page_number: index + 1, content: cleaned_text } - rescue StandardError => e - Rails.logger.warn "Failed to extract text from page #{index + 1}: #{e.message}" - nil - end - - def clean_text(text) - # Remove form feeds and normalize whitespace - cleaned = text.tr("\f", "\n") - .gsub("\r\n", "\n") - .tr("\r", "\n") - .gsub(/\s+/, ' ') - .gsub(/\n\s*\n\s*\n+/, "\n\n") - .strip - - # Remove common PDF artifacts - cleaned = cleaned.gsub(/^\d+\s*$/, '') # Remove standalone page numbers - .gsub(/^[\s\-_=]+$/, '') # Remove separator lines - .strip - - cleaned.presence - end -end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_openai_service.rb b/enterprise/app/services/captain/tools/pdf_openai_service.rb new file mode 100644 index 000000000..c53f5f57e --- /dev/null +++ b/enterprise/app/services/captain/tools/pdf_openai_service.rb @@ -0,0 +1,117 @@ +class Captain::Tools::PdfOpenaiService + include ActiveModel::Validations + + class ProcessingError < StandardError; end + + attr_reader :pdf_source + + MAX_PDF_SIZE = 25.megabytes + OPENAI_PURPOSE = 'assistants'.freeze + API_KEY_CONFIG = 'CAPTAIN_OPEN_AI_API_KEY'.freeze + + def initialize(pdf_source) + @pdf_source = pdf_source + end + + def perform + return failure_response(['Invalid PDF source']) if pdf_source.blank? + + file_response = upload_pdf_to_openai + return file_response unless file_response[:success] + + build_success_response(file_response[:file_id]) + rescue ProcessingError => e + failure_response([e.message]) + rescue StandardError => e + Rails.logger.error "PDF processing error: #{e.message}" + failure_response(['An unexpected error occurred while processing the PDF']) + end + + private + + def openai_client + @openai_client ||= OpenAI::Client.new( + access_token: api_key, + log_errors: Rails.env.development? + ) + end + + def api_key + @api_key ||= InstallationConfig.find_by!(name: API_KEY_CONFIG).value + rescue ActiveRecord::RecordNotFound + raise ProcessingError, 'OpenAI API key not configured' + end + + def upload_pdf_to_openai + file_content = prepare_pdf_content + return failure_response(['Could not retrieve PDF content']) unless file_content + + validate_pdf_size!(file_content) + + response = openai_client.files.upload( + parameters: { + file: file_content, + purpose: OPENAI_PURPOSE + } + ) + + { success: true, file_id: response['id'] } + rescue OpenAI::Error => e + failure_response(["OpenAI upload failed: #{e.message}"]) + end + + def prepare_pdf_content + attachment = extract_attachment + return nil unless attachment&.blob + + blob = attachment.blob + create_file_io(blob.download, blob.filename.to_s, blob.content_type) + rescue StandardError => e + Rails.logger.error "Failed to prepare PDF content: #{e.message}" + nil + end + + def extract_attachment + return pdf_source unless pdf_source.is_a?(ActiveStorage::Attached::One) + + pdf_source.attachment + end + + def validate_pdf_size!(file_content) + size = file_content.size + return if size <= MAX_PDF_SIZE + + raise ProcessingError, "PDF size (#{(size / 1.megabyte).round(2)}MB) exceeds maximum allowed size (#{MAX_PDF_SIZE / 1.megabyte}MB)" + end + + def create_file_io(content, filename, content_type) + StringIO.new(content).tap do |io| + io.define_singleton_method(:path) { filename } + io.define_singleton_method(:content_type) { content_type } + end + end + + def build_success_response(file_id) + success_response([{ + content: file_id, + metadata: { + openai_file_id: file_id, + processing_type: 'direct_pdf' + } + }]) + end + + def success_response(content) + { + success: true, + content: content + } + end + + def failure_response(errors) + { + success: false, + errors: errors + } + end +end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/pdf_validation_concern.rb b/enterprise/app/services/captain/tools/pdf_validation_concern.rb deleted file mode 100644 index 5c53d2444..000000000 --- a/enterprise/app/services/captain/tools/pdf_validation_concern.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Captain::Tools::PdfValidationConcern - extend ActiveSupport::Concern - - private - - def validate_pdf_source - case determine_source_type - when :url - validate_url_format - when :uploaded_file - validate_file_type_and_size - when :active_storage_attachment - validate_attachment - end - - { success: true } - rescue StandardError => e - Rails.logger.error "PDF validation failed: #{e.message}" - { success: false, errors: [e.message] } - end - - def validate_url_format - URI.parse(pdf_source) - end - - def validate_file_type_and_size - raise StandardError, 'Invalid file type' unless pdf_source.content_type == 'application/pdf' - raise StandardError, 'File too large' if pdf_source.size > 25.megabytes - end - - def validate_attachment - raise StandardError, 'No file attached' unless pdf_source.attached? - raise StandardError, 'Invalid file type' unless pdf_source.content_type == 'application/pdf' - end -end \ No newline at end of file diff --git a/spec/enterprise/jobs/captain/documents/pdf_extraction_job_spec.rb b/spec/enterprise/jobs/captain/documents/pdf_extraction_job_spec.rb deleted file mode 100644 index a011620af..000000000 --- a/spec/enterprise/jobs/captain/documents/pdf_extraction_job_spec.rb +++ /dev/null @@ -1,167 +0,0 @@ -require 'rails_helper' - -RSpec.describe Captain::Documents::PdfExtractionJob, type: :job do - let(:document) { create(:captain_document, external_link: 'https://example.com/document.pdf', source_type: 'pdf_upload') } - let(:pdf_service) { instance_double(Captain::Tools::PdfExtractionService) } - let(:pdf_content) do - [ - { content: 'PDF page 1 content', page_number: 1, chunk_index: 1, total_chunks: 2 }, - { content: 'PDF page 2 content', page_number: 2, chunk_index: 2, total_chunks: 2 } - ] - end - - before do - allow(Captain::Tools::PdfExtractionService) - .to receive(:new) - .and_return(pdf_service) - end - - describe '#perform' do - context 'when document is not a PDF' do - let(:web_document) { create(:captain_document, external_link: 'https://example.com/page.html') } - - it 'returns early without processing' do - expect(pdf_service).not_to receive(:perform) - described_class.perform_now(web_document) - end - end - - context 'when document is a PDF' do - it 'updates document status to in_progress' do - allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content }) - allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later) - - expect(document).to receive(:update).with(status: 'in_progress').once - described_class.perform_now(document) - end - - it 'uses correct PDF source for extraction' do - allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content }) - allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later) - - expect(Captain::Tools::PdfExtractionService) - .to receive(:new) - .with(document.external_link) - - described_class.perform_now(document) - end - - context 'when document has attached file' do - let(:attached_file) { instance_double(ActiveStorage::Attached::One, attached?: true) } - - before do - allow(document).to receive(:file).and_return(attached_file) - end - - it 'uses attached file for extraction' do - allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content }) - allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later) - - expect(Captain::Tools::PdfExtractionService) - .to receive(:new) - .with(document.file) - - described_class.perform_now(document) - end - end - - context 'when extraction succeeds' do - before do - allow(pdf_service).to receive(:perform).and_return({ success: true, content: pdf_content }) - end - - it 'updates document with first page content only' do - expect(document).to receive(:update!).with( - content: 'PDF page 1 content', - status: 'available' - ) - - described_class.perform_now(document) - end - - it 'resets previous responses once at the beginning' do - allow(document).to receive(:update!) - allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later) - - expect(document.responses).to receive(:destroy_all).once - - described_class.perform_now(document) - end - - it 'enqueues PdfExtractionParserJob for each content chunk' do - allow(document).to receive(:update!) - - pdf_content.each do |chunk| - expect(Captain::Tools::PdfExtractionParserJob) - .to receive(:perform_later) - .with( - assistant_id: document.assistant_id, - pdf_content: chunk, - document_id: document.id - ) - end - - described_class.perform_now(document) - end - - it 'updates document status to in_progress initially' do - allow(document).to receive(:update!) - allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later) - - expect(document).to receive(:update).with(status: 'in_progress').once - described_class.perform_now(document) - end - - it 'logs successful extraction' do - allow(document).to receive(:update!) - allow(Captain::Tools::PdfExtractionParserJob).to receive(:perform_later) - - expect(Rails.logger).to receive(:info).with(/PDF extraction successful/).at_least(:once) - expect(Rails.logger).to receive(:info).with(/chunks queued/).at_least(:once) - allow(Rails.logger).to receive(:info) # Allow other logging calls - - described_class.perform_now(document) - end - end - - context 'when extraction fails' do - before do - allow(pdf_service).to receive(:perform).and_return({ - success: false, - errors: ['Invalid PDF format', 'File corrupted'] - }) - end - - it 'updates document status to available' do - expect(document).to receive(:update).with(status: 'in_progress') - expect(document).not_to receive(:update).with(status: 'available') - - described_class.perform_now(document) - end - - it 'logs the error' do - expect(Rails.logger).not_to receive(:error) - described_class.perform_now(document) - end - end - - context 'when extraction raises an exception' do - before do - allow(pdf_service).to receive(:perform).and_raise(Captain::Tools::PdfExtractionService::ExtractionError, 'Network error') - end - - it 'updates document status to available' do - expect(document).to receive(:update).with(status: 'in_progress') - expect(document).not_to receive(:update).with(status: 'available') - - expect { described_class.perform_now(document) }.to raise_error(Captain::Tools::PdfExtractionService::ExtractionError) - end - - it 'logs the exception' do - allow(Rails.logger).to receive(:error) - expect { described_class.perform_now(document) }.to raise_error(Captain::Tools::PdfExtractionService::ExtractionError) - end - end - end - end -end \ No newline at end of file diff --git a/spec/enterprise/jobs/captain/tools/pdf_extraction_parser_job_spec.rb b/spec/enterprise/jobs/captain/tools/pdf_extraction_parser_job_spec.rb deleted file mode 100644 index d36b247c6..000000000 --- a/spec/enterprise/jobs/captain/tools/pdf_extraction_parser_job_spec.rb +++ /dev/null @@ -1,167 +0,0 @@ -require 'rails_helper' - -RSpec.describe Captain::Tools::PdfExtractionParserJob, type: :job do - let(:account) { create(:account, custom_attributes: { plan_name: 'startups' }) } - let(:assistant) { create(:captain_assistant, account: account) } - let!(:main_document) { create(:captain_document, assistant: assistant, account: account, content: 'Main document content') } - let(:pdf_content) do - { - content: 'This is sample PDF content extracted from a document. It contains useful information for FAQ generation.', - page_number: 2, - chunk_index: 1, - total_chunks: 1 - } - end - - before do - # Mock usage limits - allow(account).to receive(:usage_limits).and_return( - captain: { - documents: { - current_available: 3 - } - } - ) - end - - describe '#perform' do - context 'when document_id is provided and document exists' do - it 'does not create new documents' do - expect do - described_class.new.perform( - assistant_id: assistant.id, - pdf_content: pdf_content, - document_id: main_document.id - ) - end.not_to change(Captain::Document, :count) - end - - it 'enqueues ResponseBuilderJob with combined content for the main document' do - expected_content = "Main document content\n\n--- Additional Content (Page 2) Part 1 ---\n" \ - 'This is sample PDF content extracted from a document. It contains useful information for FAQ generation.' - - expect(Captain::Documents::ResponseBuilderJob) - .to receive(:perform_later) - .with(main_document, expected_content, skip_reset: true) - - described_class.new.perform( - assistant_id: assistant.id, - pdf_content: pdf_content, - document_id: main_document.id - ) - end - - it 'combines main document content with chunk content' do - chunk_content = pdf_content.merge(page_number: 3, chunk_index: 2) - expected_content = "Main document content\n\n--- Additional Content (Page 3) Part 2 ---\n" \ - 'This is sample PDF content extracted from a document. It contains useful information for FAQ generation.' - - expect(Captain::Documents::ResponseBuilderJob) - .to receive(:perform_later) - .with(main_document, expected_content, skip_reset: true) - - described_class.new.perform( - assistant_id: assistant.id, - pdf_content: chunk_content, - document_id: main_document.id - ) - end - end - - context 'when document_id is nil' do - it 'returns early without processing' do - expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later) - - described_class.new.perform( - assistant_id: assistant.id, - pdf_content: pdf_content, - document_id: nil - ) - end - end - - context 'when document_id is provided but document does not exist' do - it 'logs error and returns without processing' do - expect(Rails.logger).to receive(:error).with(/Document not found/) - expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later) - - described_class.new.perform( - assistant_id: assistant.id, - pdf_content: pdf_content, - document_id: 99_999 - ) - end - end - - context 'when assistant is not found' do - it 'logs error and returns without processing' do - expect(Rails.logger).to receive(:error).with(/Document not found/) - expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later) - - described_class.new.perform( - assistant_id: 99_999, - pdf_content: pdf_content, - document_id: main_document.id - ) - end - end - - context 'when limits are exceeded' do - before do - # Set up account limits configuration to exceed limits - captain_limits = { - 'startups' => { - 'captain_documents' => 1, - 'captain_responses' => 100 - } - } - - # First delete any existing config to avoid conflicts - InstallationConfig.where(name: 'CAPTAIN_CLOUD_PLAN_LIMITS').destroy_all - create(:installation_config, name: 'CAPTAIN_CLOUD_PLAN_LIMITS', value: captain_limits.to_json) - - # Create more documents than the limit to exceed it - create_list(:captain_document, 5, assistant: assistant, account: account, status: :available) - account.update_document_usage - account.reload - end - - it 'does not process content when limit exceeded' do - expect(Captain::Documents::ResponseBuilderJob).not_to receive(:perform_later) - - described_class.new.perform( - assistant_id: assistant.id, - pdf_content: pdf_content, - document_id: main_document.id - ) - end - end - end - - describe 'private methods' do - let(:job) { described_class.new } - - describe '#limit_exceeded?' do - it 'returns true when limit is zero' do - allow(account).to receive(:usage_limits).and_return( - captain: { documents: { current_available: 0 } } - ) - expect(job.send(:limit_exceeded?, account)).to be true - end - - it 'returns true when limit is negative' do - allow(account).to receive(:usage_limits).and_return( - captain: { documents: { current_available: -1 } } - ) - expect(job.send(:limit_exceeded?, account)).to be true - end - - it 'returns false when limit is positive' do - allow(account).to receive(:usage_limits).and_return( - captain: { documents: { current_available: 5 } } - ) - expect(job.send(:limit_exceeded?, account)).to be false - end - end - end -end \ No newline at end of file diff --git a/spec/enterprise/services/captain/tools/pdf_extraction_service_spec.rb b/spec/enterprise/services/captain/tools/pdf_extraction_service_spec.rb deleted file mode 100644 index 6b4110453..000000000 --- a/spec/enterprise/services/captain/tools/pdf_extraction_service_spec.rb +++ /dev/null @@ -1,217 +0,0 @@ -require 'rails_helper' - -RSpec.describe Captain::Tools::PdfExtractionService do - let(:service) { described_class.new(pdf_source) } - let(:sample_pdf_path) { Rails.root.join('spec/fixtures/files/valid_test.pdf') } - - describe '#initialize' do - context 'with valid PDF path' do - let(:pdf_source) { sample_pdf_path.to_s } - - it 'initializes with the PDF source' do - expect(service.pdf_source).to eq(pdf_source) - end - end - - context 'with URL' do - let(:pdf_source) { 'https://example.com/sample.pdf' } - - it 'initializes with the PDF URL' do - expect(service.pdf_source).to eq(pdf_source) - end - end - end - - describe '#perform' do - context 'with blank PDF source' do - let(:pdf_source) { nil } - - it 'returns error for blank source' do - result = service.perform - expect(result[:success]).to be false - expect(result[:errors]).to include('Invalid PDF source') - end - end - - context 'with valid PDF file path' do - let(:pdf_source) { sample_pdf_path.to_s } - - before do - # Ensure the sample PDF exists - skip 'Sample PDF file not available for testing' unless File.exist?(sample_pdf_path) - end - - it 'handles PDF extraction gracefully and returns error for invalid files' do - expect { service.perform }.to raise_error(StandardError, 'Invalid or corrupted PDF file') - end - end - - context 'with malformed PDF' do - let(:pdf_source) { sample_pdf_path.to_s } - - it 'handles malformed PDF gracefully' do - # Mock PDF::Reader to raise a MalformedPDFError - allow(PDF::Reader).to receive(:open).and_raise(PDF::Reader::MalformedPDFError, 'PDF does not contain EOF marker') - - expect { service.perform }.to raise_error(StandardError, 'Invalid or corrupted PDF file') - end - end - - context 'with HTTP URL' do - let(:pdf_source) { 'https://example.com/sample.pdf' } - - it 'attempts to download and process PDF from URL' do - temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s, close: nil, unlink: nil) - allow(Down).to receive(:download).and_return(temp_file) - - # Mock PDF reader to return sample content - mock_page = instance_double(PDF::Reader::Page, text: 'Sample PDF content') - mock_pages = [mock_page] - mock_reader = instance_double(PDF::Reader, pages: mock_pages) - allow(PDF::Reader).to receive(:open).and_yield(mock_reader) - - result = service.perform - - # The test should handle both success and controlled failures gracefully - expect(result).to have_key(:success) - if result[:success] - expect(result[:content]).to be_an(Array) - else - expect(result[:errors]).to be_an(Array) - end - end - end - end - - describe '#extract_text (private method)' do - let(:pdf_source) { sample_pdf_path.to_s } - - before do - skip 'Sample PDF file not available for testing' unless File.exist?(sample_pdf_path) - end - - it 'handles text extraction gracefully through perform method' do - expect { service.perform }.to raise_error(StandardError, 'Invalid or corrupted PDF file') - end - end - - describe 'private methods' do - let(:pdf_source) { sample_pdf_path.to_s } - - describe '#clean_text' do - it 'cleans text content properly' do - dirty_text = " \f Some text with \r\n line breaks \n\n\n and extra spaces " - cleaned = service.send(:clean_text, dirty_text) - - expect(cleaned).not_to include("\f") - expect(cleaned).not_to include("\r") - expect(cleaned).to include('Some text with') - expect(cleaned.strip).to eq(cleaned) - end - - it 'returns nil for blank text' do - cleaned = service.send(:clean_text, " \n\n ") - expect(cleaned).to be_nil - end - end - - describe '#chunk_content' do - let(:page_contents) do - [ - { page_number: 1, content: 'Short content' }, - { page_number: 2, content: 'A' * 3000 } # Long content that needs chunking - ] - end - - it 'chunks content appropriately' do - chunks = service.send(:chunk_content, page_contents, max_chunk_size: 1000) - - expect(chunks.length).to be >= 2 # Should have at least 2 chunks - - # Check first chunk (short content) - first_chunk = chunks.first - expect(first_chunk[:page_number]).to eq(1) - expect(first_chunk[:total_chunks]).to eq(1) - - # Check that long content was processed - long_content_chunks = chunks.select { |c| c[:page_number] == 2 } - expect(long_content_chunks.length).to be >= 1 - - # Verify long content was split appropriately - total_long_content_length = long_content_chunks.sum { |c| c[:content].length } - expect(total_long_content_length).to be > 0 - end - end - - describe '#split_content_into_chunks' do - it 'splits content by paragraphs first' do - content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph." - chunks = service.send(:split_content_into_chunks, content, 50) - - expect(chunks.length).to be >= 2 - expect(chunks.join(' ')).to include('First paragraph') - expect(chunks.join(' ')).to include('Second paragraph') - end - - it 'splits by sentences when paragraphs are too large' do - long_paragraph = "#{('A' * 100)}. #{('B' * 100)}. #{('C' * 100)}." - chunks = service.send(:split_content_into_chunks, long_paragraph, 150) - - expect(chunks.length).to be > 1 - end - end - end - - describe 'file validation' do - context 'with uploaded file object' do - let(:uploaded_file) do - temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s) - instance_double( - ActionDispatch::Http::UploadedFile, - tempfile: temp_file, - content_type: 'application/pdf', - size: 1024 - ) - end - let(:pdf_source) { uploaded_file } - - it 'validates uploaded file properties' do - expect { service.send(:validate_file_type_and_size) }.not_to raise_error - end - end - - context 'with oversized file' do - let(:uploaded_file) do - temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s) - instance_double( - ActionDispatch::Http::UploadedFile, - tempfile: temp_file, - content_type: 'application/pdf', - size: 30.megabytes - ) - end - let(:pdf_source) { uploaded_file } - - it 'raises error for oversized file' do - expect { service.send(:validate_file_type_and_size) }.to raise_error(/File too large/) - end - end - - context 'with invalid content type' do - let(:uploaded_file) do - temp_file = instance_double(Tempfile, path: sample_pdf_path.to_s) - instance_double( - ActionDispatch::Http::UploadedFile, - tempfile: temp_file, - content_type: 'text/plain', - size: 1024 - ) - end - let(:pdf_source) { uploaded_file } - - it 'raises error for invalid content type' do - expect { service.send(:validate_file_type_and_size) }.to raise_error('Invalid file type') - end - end - end -end \ No newline at end of file