diff --git a/enterprise/app/jobs/captain/documents/crawl_job.rb b/enterprise/app/jobs/captain/documents/crawl_job.rb index 132671385..94741abab 100644 --- a/enterprise/app/jobs/captain/documents/crawl_job.rb +++ b/enterprise/app/jobs/captain/documents/crawl_job.rb @@ -2,10 +2,15 @@ class Captain::Documents::CrawlJob < ApplicationJob queue_as :low def perform(document) - if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present? - perform_firecrawl_crawl(document) - else - perform_simple_crawl(document) + case document.document_type + when 'notion' + perform_notion_crawl(document) + when 'web' + if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present? + perform_firecrawl_crawl(document) + else + perform_simple_crawl(document) + end end end @@ -13,6 +18,21 @@ class Captain::Documents::CrawlJob < ApplicationJob include Captain::FirecrawlHelper + def perform_notion_crawl(document) + # external_id contains the Notion page ID for notion documents + page_id = document.external_id + crawler = Captain::Tools::NotionCrawlService.new(document.account, page_id) + + page_ids = crawler.page_ids + + page_ids.each do |notion_page_id| + Captain::Tools::NotionCrawlParserJob.perform_later( + assistant_id: document.assistant_id, + page_id: notion_page_id + ) + end + end + def perform_simple_crawl(document) page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links diff --git a/enterprise/app/jobs/captain/tools/notion_crawl_parser_job.rb b/enterprise/app/jobs/captain/tools/notion_crawl_parser_job.rb new file mode 100644 index 000000000..10457eafd --- /dev/null +++ b/enterprise/app/jobs/captain/tools/notion_crawl_parser_job.rb @@ -0,0 +1,47 @@ +class Captain::Tools::NotionCrawlParserJob < ApplicationJob + queue_as :low + + def perform(assistant_id:, page_id:) + assistant = Captain::Assistant.find(assistant_id) + account = assistant.account + + if limit_exceeded?(account) + Rails.logger.info("Document limit exceeded for assistant #{assistant_id}") + return + end + + processor_service = Integrations::Notion::ProcessorService.new(account: account) + page_data = processor_service.full_page(page_id) + + if page_data[:error] + Rails.logger.error("Failed to fetch Notion page #{page_id}: #{page_data[:error]}") + return + end + + page_title = page_data['title'] || '' + content = page_data['md'] || '' + page_url = page_data['url'] || '' + + document = assistant.documents.find_or_initialize_by( + external_id: page_id, + document_type: :notion + ) + + document.update!( + name: page_title[0..254], + content: content[0..199_999], + external_link: page_url, + status: :available + ) + rescue StandardError => e + Rails.logger.error("Failed to parse Notion page: #{page_id} - #{e.message}") + raise "Failed to parse Notion page: #{page_id} #{e.message}" + end + + private + + def limit_exceeded?(account) + limits = account.usage_limits[:captain][:documents] + limits[:current_available].negative? || limits[:current_available].zero? + end +end \ No newline at end of file diff --git a/enterprise/app/services/captain/tools/notion_crawl_service.rb b/enterprise/app/services/captain/tools/notion_crawl_service.rb new file mode 100644 index 000000000..833b0b548 --- /dev/null +++ b/enterprise/app/services/captain/tools/notion_crawl_service.rb @@ -0,0 +1,42 @@ +class Captain::Tools::NotionCrawlService + attr_reader :account, :page_id + + def initialize(account, page_id) + @account = account + @page_id = page_id + end + + def page_ids + @page_ids ||= begin + result = [page_id] + child_pages = extract_child_pages(page_id) + result.concat(child_pages) + result.uniq + end + end + + private + + def extract_child_pages(page_id, visited = Set.new) + return [] if visited.include?(page_id) + + visited.add(page_id) + + child_pages = [] + page_data = processor_service.full_page(page_id) + + return child_pages if page_data[:error] + + page_data['child_pages']&.each do |child_page| + child_id = child_page['id'] + child_pages << child_id + child_pages.concat(extract_child_pages(child_id, visited)) + end + + child_pages + end + + def processor_service + @processor_service ||= Integrations::Notion::ProcessorService.new(account: account) + end +end \ No newline at end of file