feat(captain): strip navigation boilerplate from document chunks

enterprise/app/services/captain/documents/chunking_service.rb

@@ -16,6 +16,11 @@ class Captain::Documents::ChunkingService
     /all rights reserved/i,
     /back to top/i
   ].freeze
+  BOILERPLATE_LINE_PATTERNS = [
+    /help center home page/i,
+    /theming_assets/i,
+    %r{hc/change_language/}i
+  ].freeze
 
   def initialize(content, target_tokens: DEFAULT_TARGET_TOKENS, min_tokens: DEFAULT_MIN_TOKENS,
                  max_tokens: DEFAULT_MAX_TOKENS, overlap_tokens: DEFAULT_OVERLAP_TOKENS)
@@ -47,7 +52,7 @@ class Captain::Documents::ChunkingService
   Section = Struct.new(:content, :heading_path, keyword_init: true)
 
   def split_into_sections(content)
-    cleaned_content = remove_boilerplate_sections(content)
+    cleaned_content = remove_boilerplate_sections(remove_boilerplate_lines(content))
     heading_path = []
 
     cleaned_content
@@ -69,6 +74,13 @@ class Captain::Documents::ChunkingService
       .join("\n\n")
   end
 
+  def remove_boilerplate_lines(content)
+    content
+      .lines
+      .reject { |line| boilerplate_line?(line) }
+      .join
+  end
+
   def boilerplate_section?(section)
     normalized = section.downcase.strip
     return true if BOILERPLATE_SECTION_PATTERNS.any? { |pattern| normalized.match?(pattern) }
@@ -76,41 +88,37 @@ class Captain::Documents::ChunkingService
     link_heavy_navigation_section?(section)
   end
 
+  def boilerplate_line?(line)
+    normalized = line.to_s.downcase.strip
+    return false if normalized.blank?
+    return true if BOILERPLATE_LINE_PATTERNS.any? { |pattern| normalized.match?(pattern) }
+
+    markdown_links = normalized.scan(/\[[^\]]+\]\([^)]+\)/).size
+    return true if normalized.include?('change_language/') && markdown_links >= 3
+    return false unless markdown_links >= 6
+
+    non_link_tokens = normalized
+                      .gsub(/\[[^\]]+\]\([^)]+\)/, ' ')
+                      .gsub(/[^a-z0-9\s]/, ' ')
+                      .squeeze(' ')
+                      .strip
+                      .split
+    non_link_tokens.length <= 12
+  end
+
   def link_heavy_navigation_section?(section)
-    lines = non_blank_lines(section)
-    return false unless navigation_candidate?(lines)
+    lines = section.lines.map(&:strip).reject(&:blank?)
+    return false if lines.size < 3
 
-    markdown_links = markdown_link_count(section)
-    linked_lines = linked_line_count(lines)
-    return false unless dense_link_cluster?(markdown_links, linked_lines)
+    markdown_links = section.scan(/\[[^\]]+\]\([^)]+\)/).size
+    linked_lines = lines.count { |line| link_line?(line) }
+    return false unless markdown_links >= 2 && linked_lines >= 3
 
-    short_section?(section)
-  end
-
-  def non_blank_lines(section)
-    section.lines.map(&:strip).reject(&:blank?)
-  end
-
-  def navigation_candidate?(lines)
-    lines.size >= 3
-  end
-
-  def markdown_link_count(section)
-    section.scan(/\[[^\]]+\]\([^)]+\)/).size
-  end
-
-  def linked_line_count(lines)
-    lines.count { |line| line.match?(/\[[^\]]+\]\([^)]+\)/) || line.start_with?('* [', '- [') }
-  end
-
-  def dense_link_cluster?(markdown_links, linked_lines)
-    markdown_links >= 2 && linked_lines >= 3
-  end
-
-  def short_section?(section)
     section.scan(/\b[\w']+\b/).size <= 180
   end
 
+  def link_line?(line) = line.match?(/\[[^\]]+\]\([^)]+\)/) || line.start_with?('* [', '- [')
+
   def build_chunks(sections)
     state = { chunks: [], current_chunk: +'', current_tokens: 0 }
     sections.each { |section| process_section(section, state) }

spec/enterprise/services/captain/documents/chunking_service_spec.rb

@@ -78,5 +78,28 @@ RSpec.describe Captain::Documents::ChunkingService do
       expect(combined_content).not_to include('How to block someone')
       expect(combined_content).not_to include('Related articles')
     end
+
+    it 'removes single-line navigation and language selector blobs before chunking' do
+      content = <<~TEXT
+        [![Bumble Support Help Center home page](https://support.bumble.com/hc/theming_assets/logo.svg)] [Date](https://bumble.com/en-us/date) [Friends](https://bumble.com/en-us/bff) [Bizz](https://bumble.com/en-us/bizz) [Safety](https://bumble.com/en-us/the-buzz/category/safety) [Dansk](https://support.bumble.com/hc/change_language/da?return_to=%2Fhc%2Fen-us) [Deutsch](https://support.bumble.com/hc/change_language/de?return_to=%2Fhc%2Fen-us) [Espanol](https://support.bumble.com/hc/change_language/es?return_to=%2Fhc%2Fen-us)
+
+        # How to delete my account
+        Open Settings and select Delete account.
+      TEXT
+
+      result = described_class.new(
+        content,
+        target_tokens: 30,
+        min_tokens: 10,
+        max_tokens: 50,
+        overlap_tokens: 0
+      ).chunk
+
+      combined_content = result.map { |chunk| chunk[:content] }.join("\n")
+      expect(combined_content).to include('Open Settings and select Delete account')
+      expect(combined_content).not_to include('Help Center home page')
+      expect(combined_content).not_to include('change_language')
+      expect(combined_content).not_to include('[Date](')
+    end
   end
 end