feat: add notion crawling

This crawling is only one level deep

enterprise/app/jobs/captain/documents/crawl_job.rb

@@ -2,10 +2,15 @@ class Captain::Documents::CrawlJob < ApplicationJob
   queue_as :low
 
   def perform(document)
-    if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
-      perform_firecrawl_crawl(document)
-    else
-      perform_simple_crawl(document)
+    case document.document_type
+    when 'notion'
+      perform_notion_crawl(document)
+    when 'web'
+      if InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
+        perform_firecrawl_crawl(document)
+      else
+        perform_simple_crawl(document)
+      end
     end
   end
 
@@ -13,6 +18,21 @@ class Captain::Documents::CrawlJob < ApplicationJob
 
   include Captain::FirecrawlHelper
 
+  def perform_notion_crawl(document)
+    # external_id contains the Notion page ID for notion documents
+    page_id = document.external_id
+    crawler = Captain::Tools::NotionCrawlService.new(document.account, page_id)
+
+    page_ids = crawler.page_ids
+
+    page_ids.each do |notion_page_id|
+      Captain::Tools::NotionCrawlParserJob.perform_later(
+        assistant_id: document.assistant_id,
+        page_id: notion_page_id
+      )
+    end
+  end
+
   def perform_simple_crawl(document)
     page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links
 

enterprise/app/jobs/captain/tools/notion_crawl_parser_job.rb

@@ -0,0 +1,47 @@
+class Captain::Tools::NotionCrawlParserJob < ApplicationJob
+  queue_as :low
+
+  def perform(assistant_id:, page_id:)
+    assistant = Captain::Assistant.find(assistant_id)
+    account = assistant.account
+
+    if limit_exceeded?(account)
+      Rails.logger.info("Document limit exceeded for assistant #{assistant_id}")
+      return
+    end
+
+    processor_service = Integrations::Notion::ProcessorService.new(account: account)
+    page_data = processor_service.full_page(page_id)
+
+    if page_data[:error]
+      Rails.logger.error("Failed to fetch Notion page #{page_id}: #{page_data[:error]}")
+      return
+    end
+
+    page_title = page_data['title'] || ''
+    content = page_data['md'] || ''
+    page_url = page_data['url'] || ''
+
+    document = assistant.documents.find_or_initialize_by(
+      external_id: page_id,
+      document_type: :notion
+    )
+
+    document.update!(
+      name: page_title[0..254],
+      content: content[0..199_999],
+      external_link: page_url,
+      status: :available
+    )
+  rescue StandardError => e
+    Rails.logger.error("Failed to parse Notion page: #{page_id} - #{e.message}")
+    raise "Failed to parse Notion page: #{page_id} #{e.message}"
+  end
+
+  private
+
+  def limit_exceeded?(account)
+    limits = account.usage_limits[:captain][:documents]
+    limits[:current_available].negative? || limits[:current_available].zero?
+  end
+end

enterprise/app/services/captain/tools/notion_crawl_service.rb

@@ -0,0 +1,42 @@
+class Captain::Tools::NotionCrawlService
+  attr_reader :account, :page_id
+
+  def initialize(account, page_id)
+    @account = account
+    @page_id = page_id
+  end
+
+  def page_ids
+    @page_ids ||= begin
+      result = [page_id]
+      child_pages = extract_child_pages(page_id)
+      result.concat(child_pages)
+      result.uniq
+    end
+  end
+
+  private
+
+  def extract_child_pages(page_id, visited = Set.new)
+    return [] if visited.include?(page_id)
+
+    visited.add(page_id)
+
+    child_pages = []
+    page_data = processor_service.full_page(page_id)
+
+    return child_pages if page_data[:error]
+
+    page_data['child_pages']&.each do |child_page|
+      child_id = child_page['id']
+      child_pages << child_id
+      child_pages.concat(extract_child_pages(child_id, visited))
+    end
+
+    child_pages
+  end
+
+  def processor_service
+    @processor_service ||= Integrations::Notion::ProcessorService.new(account: account)
+  end
+end