Package com.firecrawl.models

Class CrawlOptions.Builder

java.lang.Object

com.firecrawl.models.CrawlOptions.Builder

Enclosing class:

CrawlOptions

public static final class CrawlOptions.Builder
extends java.lang.Object

Method Summary

Modifier and Type	Method	Description
CrawlOptions.Builder	allowExternalLinks(java.lang.Boolean allowExternalLinks)	Follow external links.
CrawlOptions.Builder	allowSubdomains(java.lang.Boolean allowSubdomains)	Follow subdomains.
CrawlOptions	build()
CrawlOptions.Builder	crawlEntireDomain(java.lang.Boolean crawlEntireDomain)	Whether to crawl the entire domain.
CrawlOptions.Builder	deduplicateSimilarURLs(java.lang.Boolean deduplicateSimilarURLs)	Deduplicate URLs that are similar.
CrawlOptions.Builder	delay(java.lang.Integer delay)	Delay in milliseconds between requests.
CrawlOptions.Builder	excludePaths(java.util.List<java.lang.String> excludePaths)	URL path patterns to exclude from crawling.
CrawlOptions.Builder	ignoreQueryParameters(java.lang.Boolean ignoreQueryParameters)	Ignore query parameters when deduplicating URLs.
CrawlOptions.Builder	includePaths(java.util.List<java.lang.String> includePaths)	URL path patterns to include in crawling.
CrawlOptions.Builder	integration(java.lang.String integration)	Integration identifier.
CrawlOptions.Builder	limit(java.lang.Integer limit)	Maximum number of pages to crawl.
CrawlOptions.Builder	maxConcurrency(java.lang.Integer maxConcurrency)	Maximum concurrent requests.
CrawlOptions.Builder	maxDiscoveryDepth(java.lang.Integer maxDiscoveryDepth)	Maximum depth to discover links.
CrawlOptions.Builder	prompt(java.lang.String prompt)	Natural language prompt to guide crawling.
CrawlOptions.Builder	regexOnFullURL(java.lang.Boolean regexOnFullURL)	Apply regex patterns to the full URL, not just the path.
CrawlOptions.Builder	scrapeOptions(ScrapeOptions scrapeOptions)	Scrape options applied to each crawled page.
CrawlOptions.Builder	sitemap(java.lang.String sitemap)	Sitemap handling: "skip", "include", or "only".
CrawlOptions.Builder	webhook(java.lang.Object webhook)	Webhook URL string or WebhookConfig object.
CrawlOptions.Builder	zeroDataRetention(java.lang.Boolean zeroDataRetention)	Do not store any scraped data on Firecrawl servers.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

prompt

public CrawlOptions.Builder prompt(java.lang.String prompt)

Natural language prompt to guide crawling.

excludePaths

public CrawlOptions.Builder excludePaths(java.util.List<java.lang.String> excludePaths)

URL path patterns to exclude from crawling.

includePaths

public CrawlOptions.Builder includePaths(java.util.List<java.lang.String> includePaths)

URL path patterns to include in crawling.

maxDiscoveryDepth

public CrawlOptions.Builder maxDiscoveryDepth(java.lang.Integer maxDiscoveryDepth)

Maximum depth to discover links.

sitemap

public CrawlOptions.Builder sitemap(java.lang.String sitemap)

Sitemap handling: "skip", "include", or "only".

ignoreQueryParameters

public CrawlOptions.Builder ignoreQueryParameters(java.lang.Boolean ignoreQueryParameters)

Ignore query parameters when deduplicating URLs.

deduplicateSimilarURLs

public CrawlOptions.Builder deduplicateSimilarURLs(java.lang.Boolean deduplicateSimilarURLs)

Deduplicate URLs that are similar.

limit

public CrawlOptions.Builder limit(java.lang.Integer limit)

Maximum number of pages to crawl.

crawlEntireDomain

public CrawlOptions.Builder crawlEntireDomain(java.lang.Boolean crawlEntireDomain)

Whether to crawl the entire domain.

allowExternalLinks

public CrawlOptions.Builder allowExternalLinks(java.lang.Boolean allowExternalLinks)

Follow external links.

allowSubdomains

public CrawlOptions.Builder allowSubdomains(java.lang.Boolean allowSubdomains)

Follow subdomains.

delay

public CrawlOptions.Builder delay(java.lang.Integer delay)

Delay in milliseconds between requests.

maxConcurrency

public CrawlOptions.Builder maxConcurrency(java.lang.Integer maxConcurrency)

Maximum concurrent requests.

webhook

public CrawlOptions.Builder webhook(java.lang.Object webhook)

Webhook URL string or WebhookConfig object.

scrapeOptions

public CrawlOptions.Builder scrapeOptions(ScrapeOptions scrapeOptions)

Scrape options applied to each crawled page.

regexOnFullURL

public CrawlOptions.Builder regexOnFullURL(java.lang.Boolean regexOnFullURL)

Apply regex patterns to the full URL, not just the path.

zeroDataRetention

public CrawlOptions.Builder zeroDataRetention(java.lang.Boolean zeroDataRetention)

Do not store any scraped data on Firecrawl servers.

integration

public CrawlOptions.Builder integration(java.lang.String integration)

Integration identifier.

build

public CrawlOptions build()