feat(captain): add chunk builder job and orchestration

enterprise/app/jobs/captain/documents/chunk_builder_job.rb

@@ -0,0 +1,7 @@
+class Captain::Documents::ChunkBuilderJob < ApplicationJob
+  queue_as :low
+
+  def perform(document)
+    Captain::Documents::ChunkBuilderService.new(document).process
+  end
+end

enterprise/app/services/captain/documents/chunk_builder_service.rb

@@ -0,0 +1,56 @@
+class Captain::Documents::ChunkBuilderService
+  def initialize(document)
+    @document = document
+    @embedding_service = Captain::Documents::ChunkEmbeddingService.new(account_id: document.account_id)
+  end
+
+  def process
+    raise ArgumentError, 'Document content is required for chunk building' if @document.content.blank?
+
+    chunks = Captain::Documents::ChunkingService.new(@document.content).chunk
+
+    @document.update!(
+      chunking_status: :chunking,
+      last_chunk_error: nil,
+      chunks_generated_at: nil,
+      expected_chunk_count: chunks.count,
+      indexed_chunk_count: 0
+    )
+
+    indexed_count = rebuild_chunks(chunks)
+
+    @document.update!(
+      chunking_status: :ready,
+      indexed_chunk_count: indexed_count,
+      chunks_generated_at: Time.current
+    )
+  rescue StandardError => e
+    @document.update!(chunking_status: :failed, last_chunk_error: e.message)
+    raise
+  end
+
+  private
+
+  def rebuild_chunks(chunks)
+    @document.update!(chunking_status: :indexing)
+
+    @document.chunks.delete_all
+
+    chunks.each_with_index do |chunk, index|
+      context = generate_context(chunk[:content])
+      attributes = @embedding_service.build_record_attributes(document: @document, chunk: chunk, context: context)
+      Captain::DocumentChunk.create!(attributes)
+      @document.update_column(:indexed_chunk_count, index + 1) # rubocop:disable Rails/SkipsModelValidations
+    end
+
+    chunks.count
+  end
+
+  def generate_context(chunk_content)
+    Captain::Documents::ContextGenerationService.new(
+      document_content: @document.content,
+      chunk_content: chunk_content,
+      account_id: @document.account_id
+    ).generate
+  end
+end

spec/enterprise/jobs/captain/documents/chunk_builder_job_spec.rb

@@ -0,0 +1,25 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Documents::ChunkBuilderJob, type: :job do
+  let(:document) { create(:captain_document, status: :available) }
+  let(:builder_service) { instance_double(Captain::Documents::ChunkBuilderService) }
+
+  describe '#perform' do
+    it 'runs the chunk builder service for the document' do
+      allow(Captain::Documents::ChunkBuilderService).to receive(:new).with(document).and_return(builder_service)
+      allow(builder_service).to receive(:process)
+
+      described_class.new.perform(document)
+
+      expect(Captain::Documents::ChunkBuilderService).to have_received(:new).with(document)
+      expect(builder_service).to have_received(:process)
+    end
+
+    it 'bubbles service errors for retry handling' do
+      allow(Captain::Documents::ChunkBuilderService).to receive(:new).with(document).and_return(builder_service)
+      allow(builder_service).to receive(:process).and_raise(StandardError, 'transient failure')
+
+      expect { described_class.new.perform(document) }.to raise_error(StandardError, 'transient failure')
+    end
+  end
+end

spec/enterprise/services/captain/documents/chunk_builder_service_spec.rb

@@ -0,0 +1,80 @@
+require 'rails_helper'
+
+RSpec.describe Captain::Documents::ChunkBuilderService do
+  let(:document) { create(:captain_document, status: :available, content: 'Pricing and limits content') }
+  let(:chunking_service) { instance_double(Captain::Documents::ChunkingService) }
+  let(:embedding_service) { instance_double(Captain::Documents::ChunkEmbeddingService) }
+  let(:context_service) { instance_double(Captain::Documents::ContextGenerationService) }
+  let(:chunks) do
+    [
+      { position: 0, content: 'Chunk 1', token_count: 10 },
+      { position: 1, content: 'Chunk 2', token_count: 12 }
+    ]
+  end
+  let(:embedding_vector) { Array.new(1536, 0.1) }
+
+  before do
+    allow(Captain::Documents::ChunkingService).to receive(:new).and_return(chunking_service)
+    allow(chunking_service).to receive(:chunk).and_return(chunks)
+
+    allow(Captain::Documents::ChunkEmbeddingService).to receive(:new).and_return(embedding_service)
+    allow(embedding_service).to receive(:build_record_attributes) do |args|
+      chunk = args.fetch(:chunk)
+      {
+        document_id: document.id,
+        assistant_id: document.assistant_id,
+        account_id: document.account_id,
+        position: chunk.fetch(:position),
+        content: chunk.fetch(:content),
+        token_count: chunk[:token_count],
+        context: args[:context],
+        embedding: embedding_vector,
+        created_at: Time.current,
+        updated_at: Time.current
+      }
+    end
+
+    allow(Captain::Documents::ContextGenerationService).to receive(:new).and_return(context_service)
+    allow(context_service).to receive(:generate).and_return('Chunk context')
+  end
+
+  describe '#process' do
+    it 'builds chunks and marks the document ready with progress counters' do
+      described_class.new(document).process
+
+      document.reload
+      expect(document.chunking_status).to eq('ready')
+      expect(document.expected_chunk_count).to eq(2)
+      expect(document.indexed_chunk_count).to eq(2)
+      expect(document.chunks_generated_at).to be_present
+      expect(document.last_chunk_error).to be_nil
+      expect(document.chunks.count).to eq(2)
+    end
+
+    it 'replaces existing chunks when reprocessed (idempotent rebuild)' do
+      create(
+        :captain_document_chunk,
+        document: document,
+        assistant: document.assistant,
+        account: document.account,
+        position: 99,
+        embedding: nil
+      )
+
+      described_class.new(document).process
+
+      positions = document.chunks.reload.order(:position).pluck(:position)
+      expect(positions).to eq([0, 1])
+    end
+
+    it 'marks the document failed and stores the error when processing fails' do
+      allow(chunking_service).to receive(:chunk).and_raise(StandardError, 'chunking failed')
+
+      expect { described_class.new(document).process }.to raise_error(StandardError, 'chunking failed')
+
+      document.reload
+      expect(document.chunking_status).to eq('failed')
+      expect(document.last_chunk_error).to eq('chunking failed')
+    end
+  end
+end

spec/factories/captain/document_chunk.rb

@@ -0,0 +1,12 @@
+FactoryBot.define do
+  factory :captain_document_chunk, class: 'Captain::DocumentChunk' do
+    association :document, factory: :captain_document
+    assistant { document.assistant }
+    account { document.account }
+    content { 'Chunk content' }
+    add_attribute(:context) { 'Chunk context' }
+    embedding { Array.new(1536, 0.1) }
+    position { 0 }
+    token_count { 12 }
+  end
+end