feat(captain): improve contextual chunk retrieval quality

spec/enterprise/services/captain/documents/chunking_service_spec.rb

@@ -53,5 +53,30 @@ RSpec.describe Captain::Documents::ChunkingService do
       expect(result.size).to eq(2)
       expect(result.last[:content]).to include('nine ten')
     end
+
+    it 'removes obvious boilerplate navigation sections before chunking' do
+      content = <<~TEXT
+        # Account deletion
+        You can delete your account from Settings.
+
+        ## Related articles
+        - [How to block someone](https://example.com/block)
+        - [How to report someone](https://example.com/report)
+        - [How to update profile](https://example.com/profile)
+      TEXT
+
+      result = described_class.new(
+        content,
+        target_tokens: 30,
+        min_tokens: 10,
+        max_tokens: 50,
+        overlap_tokens: 0
+      ).chunk
+
+      combined_content = result.map { |chunk| chunk[:content] }.join("\n")
+      expect(combined_content).to include('You can delete your account from Settings')
+      expect(combined_content).not_to include('How to block someone')
+      expect(combined_content).not_to include('Related articles')
+    end
   end
 end

spec/enterprise/services/captain/documents/context_generation_service_spec.rb

@@ -28,6 +28,19 @@ RSpec.describe Captain::Documents::ContextGenerationService do
       expect(result).to eq('Pricing page context.')
     end
 
+    it 'uses retrieval-oriented context instructions' do
+      service = described_class.new(
+        document_content: 'Document text',
+        chunk_content: 'Chunk text',
+        account_id: 1
+      )
+
+      service.generate
+
+      expect(chat).to have_received(:with_instructions).with(include('user intents this chunk can answer'))
+      expect(chat).to have_received(:with_instructions).with(include('make this chunk easier to find'))
+    end
+
     it 'uses explicit model when provided' do
       service = described_class.new(
         document_content: 'Doc text',

spec/enterprise/services/captain/documents/hybrid_chunk_search_service_spec.rb

@@ -54,5 +54,42 @@ RSpec.describe Captain::Documents::HybridChunkSearchService do
       expect(results.first.document_id).to eq(ready_document.id)
       expect(results.first.content).to include('reset password')
     end
+
+    it 'uses BM25 scoring to prioritize stronger lexical matches' do
+      ready_document = create(
+        :captain_document,
+        account: account,
+        assistant: assistant,
+        status: :available,
+        chunking_status: :ready
+      )
+
+      weaker_chunk = create(
+        :captain_document_chunk,
+        document: ready_document,
+        account: account,
+        assistant: assistant,
+        position: 0,
+        content: 'Incognito mode exists for profile visibility.',
+        context: 'Privacy and safety settings.'
+      )
+      stronger_chunk = create(
+        :captain_document_chunk,
+        document: ready_document,
+        account: account,
+        assistant: assistant,
+        position: 1,
+        content: 'Incognito mode allows hidden browsing. Incognito mode keeps your profile hidden.',
+        context: 'Incognito mode details and hidden profile behavior.'
+      )
+
+      embedding_service = instance_double(Captain::Llm::EmbeddingService, get_embedding: [])
+      allow(Captain::Llm::EmbeddingService).to receive(:new).and_return(embedding_service)
+
+      results = service.search('how does incognito mode work')
+
+      expect(results.first.id).to eq(stronger_chunk.id)
+      expect(results.map(&:id)).to include(weaker_chunk.id)
+    end
   end
 end