From 25d30fbebf9cf942296f0fe591eba4a3c1284b7f Mon Sep 17 00:00:00 2001 From: aakashb95 Date: Fri, 20 Feb 2026 15:17:52 +0530 Subject: [PATCH] feat(captain): strip navigation boilerplate from document chunks --- .../captain/documents/chunking_service.rb | 68 +++++++++++-------- .../documents/chunking_service_spec.rb | 23 +++++++ 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/enterprise/app/services/captain/documents/chunking_service.rb b/enterprise/app/services/captain/documents/chunking_service.rb index aaee36a20..74bae0797 100644 --- a/enterprise/app/services/captain/documents/chunking_service.rb +++ b/enterprise/app/services/captain/documents/chunking_service.rb @@ -16,6 +16,11 @@ class Captain::Documents::ChunkingService /all rights reserved/i, /back to top/i ].freeze + BOILERPLATE_LINE_PATTERNS = [ + /help center home page/i, + /theming_assets/i, + %r{hc/change_language/}i + ].freeze def initialize(content, target_tokens: DEFAULT_TARGET_TOKENS, min_tokens: DEFAULT_MIN_TOKENS, max_tokens: DEFAULT_MAX_TOKENS, overlap_tokens: DEFAULT_OVERLAP_TOKENS) @@ -47,7 +52,7 @@ class Captain::Documents::ChunkingService Section = Struct.new(:content, :heading_path, keyword_init: true) def split_into_sections(content) - cleaned_content = remove_boilerplate_sections(content) + cleaned_content = remove_boilerplate_sections(remove_boilerplate_lines(content)) heading_path = [] cleaned_content @@ -69,6 +74,13 @@ class Captain::Documents::ChunkingService .join("\n\n") end + def remove_boilerplate_lines(content) + content + .lines + .reject { |line| boilerplate_line?(line) } + .join + end + def boilerplate_section?(section) normalized = section.downcase.strip return true if BOILERPLATE_SECTION_PATTERNS.any? { |pattern| normalized.match?(pattern) } @@ -76,41 +88,37 @@ class Captain::Documents::ChunkingService link_heavy_navigation_section?(section) end + def boilerplate_line?(line) + normalized = line.to_s.downcase.strip + return false if normalized.blank? + return true if BOILERPLATE_LINE_PATTERNS.any? { |pattern| normalized.match?(pattern) } + + markdown_links = normalized.scan(/\[[^\]]+\]\([^)]+\)/).size + return true if normalized.include?('change_language/') && markdown_links >= 3 + return false unless markdown_links >= 6 + + non_link_tokens = normalized + .gsub(/\[[^\]]+\]\([^)]+\)/, ' ') + .gsub(/[^a-z0-9\s]/, ' ') + .squeeze(' ') + .strip + .split + non_link_tokens.length <= 12 + end + def link_heavy_navigation_section?(section) - lines = non_blank_lines(section) - return false unless navigation_candidate?(lines) + lines = section.lines.map(&:strip).reject(&:blank?) + return false if lines.size < 3 - markdown_links = markdown_link_count(section) - linked_lines = linked_line_count(lines) - return false unless dense_link_cluster?(markdown_links, linked_lines) + markdown_links = section.scan(/\[[^\]]+\]\([^)]+\)/).size + linked_lines = lines.count { |line| link_line?(line) } + return false unless markdown_links >= 2 && linked_lines >= 3 - short_section?(section) - end - - def non_blank_lines(section) - section.lines.map(&:strip).reject(&:blank?) - end - - def navigation_candidate?(lines) - lines.size >= 3 - end - - def markdown_link_count(section) - section.scan(/\[[^\]]+\]\([^)]+\)/).size - end - - def linked_line_count(lines) - lines.count { |line| line.match?(/\[[^\]]+\]\([^)]+\)/) || line.start_with?('* [', '- [') } - end - - def dense_link_cluster?(markdown_links, linked_lines) - markdown_links >= 2 && linked_lines >= 3 - end - - def short_section?(section) section.scan(/\b[\w']+\b/).size <= 180 end + def link_line?(line) = line.match?(/\[[^\]]+\]\([^)]+\)/) || line.start_with?('* [', '- [') + def build_chunks(sections) state = { chunks: [], current_chunk: +'', current_tokens: 0 } sections.each { |section| process_section(section, state) } diff --git a/spec/enterprise/services/captain/documents/chunking_service_spec.rb b/spec/enterprise/services/captain/documents/chunking_service_spec.rb index 5ddba777e..5d68f5858 100644 --- a/spec/enterprise/services/captain/documents/chunking_service_spec.rb +++ b/spec/enterprise/services/captain/documents/chunking_service_spec.rb @@ -78,5 +78,28 @@ RSpec.describe Captain::Documents::ChunkingService do expect(combined_content).not_to include('How to block someone') expect(combined_content).not_to include('Related articles') end + + it 'removes single-line navigation and language selector blobs before chunking' do + content = <<~TEXT + [![Bumble Support Help Center home page](https://support.bumble.com/hc/theming_assets/logo.svg)] [Date](https://bumble.com/en-us/date) [Friends](https://bumble.com/en-us/bff) [Bizz](https://bumble.com/en-us/bizz) [Safety](https://bumble.com/en-us/the-buzz/category/safety) [Dansk](https://support.bumble.com/hc/change_language/da?return_to=%2Fhc%2Fen-us) [Deutsch](https://support.bumble.com/hc/change_language/de?return_to=%2Fhc%2Fen-us) [Espanol](https://support.bumble.com/hc/change_language/es?return_to=%2Fhc%2Fen-us) + + # How to delete my account + Open Settings and select Delete account. + TEXT + + result = described_class.new( + content, + target_tokens: 30, + min_tokens: 10, + max_tokens: 50, + overlap_tokens: 0 + ).chunk + + combined_content = result.map { |chunk| chunk[:content] }.join("\n") + expect(combined_content).to include('Open Settings and select Delete account') + expect(combined_content).not_to include('Help Center home page') + expect(combined_content).not_to include('change_language') + expect(combined_content).not_to include('[Date](') + end end end