diff --git a/enterprise/app/jobs/captain/documents/chunk_builder_job.rb b/enterprise/app/jobs/captain/documents/chunk_builder_job.rb new file mode 100644 index 000000000..e606a00e7 --- /dev/null +++ b/enterprise/app/jobs/captain/documents/chunk_builder_job.rb @@ -0,0 +1,7 @@ +class Captain::Documents::ChunkBuilderJob < ApplicationJob + queue_as :low + + def perform(document) + Captain::Documents::ChunkBuilderService.new(document).process + end +end diff --git a/enterprise/app/services/captain/documents/chunk_builder_service.rb b/enterprise/app/services/captain/documents/chunk_builder_service.rb new file mode 100644 index 000000000..d8ebdafcd --- /dev/null +++ b/enterprise/app/services/captain/documents/chunk_builder_service.rb @@ -0,0 +1,56 @@ +class Captain::Documents::ChunkBuilderService + def initialize(document) + @document = document + @embedding_service = Captain::Documents::ChunkEmbeddingService.new(account_id: document.account_id) + end + + def process + raise ArgumentError, 'Document content is required for chunk building' if @document.content.blank? + + chunks = Captain::Documents::ChunkingService.new(@document.content).chunk + + @document.update!( + chunking_status: :chunking, + last_chunk_error: nil, + chunks_generated_at: nil, + expected_chunk_count: chunks.count, + indexed_chunk_count: 0 + ) + + indexed_count = rebuild_chunks(chunks) + + @document.update!( + chunking_status: :ready, + indexed_chunk_count: indexed_count, + chunks_generated_at: Time.current + ) + rescue StandardError => e + @document.update!(chunking_status: :failed, last_chunk_error: e.message) + raise + end + + private + + def rebuild_chunks(chunks) + @document.update!(chunking_status: :indexing) + + @document.chunks.delete_all + + chunks.each_with_index do |chunk, index| + context = generate_context(chunk[:content]) + attributes = @embedding_service.build_record_attributes(document: @document, chunk: chunk, context: context) + Captain::DocumentChunk.create!(attributes) + @document.update_column(:indexed_chunk_count, index + 1) # rubocop:disable Rails/SkipsModelValidations + end + + chunks.count + end + + def generate_context(chunk_content) + Captain::Documents::ContextGenerationService.new( + document_content: @document.content, + chunk_content: chunk_content, + account_id: @document.account_id + ).generate + end +end diff --git a/spec/enterprise/jobs/captain/documents/chunk_builder_job_spec.rb b/spec/enterprise/jobs/captain/documents/chunk_builder_job_spec.rb new file mode 100644 index 000000000..99f0ab67b --- /dev/null +++ b/spec/enterprise/jobs/captain/documents/chunk_builder_job_spec.rb @@ -0,0 +1,25 @@ +require 'rails_helper' + +RSpec.describe Captain::Documents::ChunkBuilderJob, type: :job do + let(:document) { create(:captain_document, status: :available) } + let(:builder_service) { instance_double(Captain::Documents::ChunkBuilderService) } + + describe '#perform' do + it 'runs the chunk builder service for the document' do + allow(Captain::Documents::ChunkBuilderService).to receive(:new).with(document).and_return(builder_service) + allow(builder_service).to receive(:process) + + described_class.new.perform(document) + + expect(Captain::Documents::ChunkBuilderService).to have_received(:new).with(document) + expect(builder_service).to have_received(:process) + end + + it 'bubbles service errors for retry handling' do + allow(Captain::Documents::ChunkBuilderService).to receive(:new).with(document).and_return(builder_service) + allow(builder_service).to receive(:process).and_raise(StandardError, 'transient failure') + + expect { described_class.new.perform(document) }.to raise_error(StandardError, 'transient failure') + end + end +end diff --git a/spec/enterprise/services/captain/documents/chunk_builder_service_spec.rb b/spec/enterprise/services/captain/documents/chunk_builder_service_spec.rb new file mode 100644 index 000000000..a8dba0e11 --- /dev/null +++ b/spec/enterprise/services/captain/documents/chunk_builder_service_spec.rb @@ -0,0 +1,80 @@ +require 'rails_helper' + +RSpec.describe Captain::Documents::ChunkBuilderService do + let(:document) { create(:captain_document, status: :available, content: 'Pricing and limits content') } + let(:chunking_service) { instance_double(Captain::Documents::ChunkingService) } + let(:embedding_service) { instance_double(Captain::Documents::ChunkEmbeddingService) } + let(:context_service) { instance_double(Captain::Documents::ContextGenerationService) } + let(:chunks) do + [ + { position: 0, content: 'Chunk 1', token_count: 10 }, + { position: 1, content: 'Chunk 2', token_count: 12 } + ] + end + let(:embedding_vector) { Array.new(1536, 0.1) } + + before do + allow(Captain::Documents::ChunkingService).to receive(:new).and_return(chunking_service) + allow(chunking_service).to receive(:chunk).and_return(chunks) + + allow(Captain::Documents::ChunkEmbeddingService).to receive(:new).and_return(embedding_service) + allow(embedding_service).to receive(:build_record_attributes) do |args| + chunk = args.fetch(:chunk) + { + document_id: document.id, + assistant_id: document.assistant_id, + account_id: document.account_id, + position: chunk.fetch(:position), + content: chunk.fetch(:content), + token_count: chunk[:token_count], + context: args[:context], + embedding: embedding_vector, + created_at: Time.current, + updated_at: Time.current + } + end + + allow(Captain::Documents::ContextGenerationService).to receive(:new).and_return(context_service) + allow(context_service).to receive(:generate).and_return('Chunk context') + end + + describe '#process' do + it 'builds chunks and marks the document ready with progress counters' do + described_class.new(document).process + + document.reload + expect(document.chunking_status).to eq('ready') + expect(document.expected_chunk_count).to eq(2) + expect(document.indexed_chunk_count).to eq(2) + expect(document.chunks_generated_at).to be_present + expect(document.last_chunk_error).to be_nil + expect(document.chunks.count).to eq(2) + end + + it 'replaces existing chunks when reprocessed (idempotent rebuild)' do + create( + :captain_document_chunk, + document: document, + assistant: document.assistant, + account: document.account, + position: 99, + embedding: nil + ) + + described_class.new(document).process + + positions = document.chunks.reload.order(:position).pluck(:position) + expect(positions).to eq([0, 1]) + end + + it 'marks the document failed and stores the error when processing fails' do + allow(chunking_service).to receive(:chunk).and_raise(StandardError, 'chunking failed') + + expect { described_class.new(document).process }.to raise_error(StandardError, 'chunking failed') + + document.reload + expect(document.chunking_status).to eq('failed') + expect(document.last_chunk_error).to eq('chunking failed') + end + end +end diff --git a/spec/factories/captain/document_chunk.rb b/spec/factories/captain/document_chunk.rb new file mode 100644 index 000000000..3b3a5ac72 --- /dev/null +++ b/spec/factories/captain/document_chunk.rb @@ -0,0 +1,12 @@ +FactoryBot.define do + factory :captain_document_chunk, class: 'Captain::DocumentChunk' do + association :document, factory: :captain_document + assistant { document.assistant } + account { document.account } + content { 'Chunk content' } + add_attribute(:context) { 'Chunk context' } + embedding { Array.new(1536, 0.1) } + position { 0 } + token_count { 12 } + end +end