diff --git a/enterprise/app/services/captain/tools/firecrawl_service.rb b/enterprise/app/services/captain/tools/firecrawl_service.rb index 9013a8c43..1c17a6b98 100644 --- a/enterprise/app/services/captain/tools/firecrawl_service.rb +++ b/enterprise/app/services/captain/tools/firecrawl_service.rb @@ -50,6 +50,7 @@ class Captain::Tools::FirecrawlService def scrape_options { onlyMainContent: true, + maxAge: 0, formats: ['markdown'], excludeTags: FIRECRAWL_EXCLUDE_TAGS } diff --git a/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb b/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb index d003c5536..d4fd014c2 100644 --- a/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb +++ b/spec/enterprise/services/captain/tools/firecrawl_service_spec.rb @@ -59,6 +59,7 @@ RSpec.describe Captain::Tools::FirecrawlService do webhook: { url: webhook_url }, scrapeOptions: { onlyMainContent: true, + maxAge: 0, formats: ['markdown'], excludeTags: Captain::Tools::FirecrawlService::FIRECRAWL_EXCLUDE_TAGS }