Crawl websites.

client.BrowserRendering.Crawl.New(ctx, params) (*string, error)

POST/accounts/{account_id}/browser-rendering/crawl

Starts a crawl job for the provided URL and its children. Check available options like gotoOptions and waitFor* to control page load behaviour.

Security

API Token

The preferred authorization scheme for interacting with the Cloudflare API. Create a token.

Example:Authorization: Bearer Sn3lZJTBX6kkg7OdcBUAxOO963GEIyGQqnFTOFYY

API Email + API Key

The previous authorization scheme for interacting with the Cloudflare API, used in conjunction with a Global API key.

Example:X-Auth-Email: user@example.com

The previous authorization scheme for interacting with the Cloudflare API. When possible, use API tokens instead of Global API keys.

Example:X-Auth-Key: 144c9defac04969c7bfad8efaa8ea194

Accepted Permissions (at least one required)

Browser Rendering Write

ParametersExpand Collapse

params CrawlNewParams

AccountID param.Field[string]

Path param: Account ID.

URL param.Field[string]

Body param: URL to navigate to, eg. https://example.com.

formaturi

CacheTTL param.Field[float64]optional

Query param: Cache TTL default is 5s. Set to 0 to disable.

maximum86400

ActionTimeout param.Field[float64]optional

Body param: The maximum duration allowed for the browser action to complete after the page has loaded (such as taking screenshots, extracting content, or generating PDFs). If this time limit is exceeded, the action stops and returns a timeout error.

maximum120000

AddScriptTag param.Field[[]CrawlNewParamsVariant0AddScriptTag]optional

Body param: Adds a <script> tag into the page with the desired URL or content.

ID stringoptional

Content stringoptional

Type stringoptional

URL stringoptional

AddStyleTag param.Field[[]CrawlNewParamsVariant0AddStyleTag]optional

Body param: Adds a <link rel="stylesheet"> tag into the page with the desired URL or a <style type="text/css"> tag with the content.

Content stringoptional

URL stringoptional

AllowRequestPattern param.Field[[]string]optional

Body param: Only allow requests that match the provided regex patterns, eg. '/^.*.(css)'.

AllowResourceTypes param.Field[[]CrawlNewParamsVariant0AllowResourceType]optional

Body param: Only allow requests that match the provided resource types, eg. 'image' or 'script'.

const CrawlNewParamsVariant0AllowResourceTypeDocument CrawlNewParamsVariant0AllowResourceType = "document"

const CrawlNewParamsVariant0AllowResourceTypeStylesheet CrawlNewParamsVariant0AllowResourceType = "stylesheet"

const CrawlNewParamsVariant0AllowResourceTypeImage CrawlNewParamsVariant0AllowResourceType = "image"

const CrawlNewParamsVariant0AllowResourceTypeMedia CrawlNewParamsVariant0AllowResourceType = "media"

const CrawlNewParamsVariant0AllowResourceTypeFont CrawlNewParamsVariant0AllowResourceType = "font"

const CrawlNewParamsVariant0AllowResourceTypeScript CrawlNewParamsVariant0AllowResourceType = "script"

const CrawlNewParamsVariant0AllowResourceTypeTexttrack CrawlNewParamsVariant0AllowResourceType = "texttrack"

const CrawlNewParamsVariant0AllowResourceTypeXHR CrawlNewParamsVariant0AllowResourceType = "xhr"

const CrawlNewParamsVariant0AllowResourceTypeFetch CrawlNewParamsVariant0AllowResourceType = "fetch"

const CrawlNewParamsVariant0AllowResourceTypePrefetch CrawlNewParamsVariant0AllowResourceType = "prefetch"

const CrawlNewParamsVariant0AllowResourceTypeEventsource CrawlNewParamsVariant0AllowResourceType = "eventsource"

const CrawlNewParamsVariant0AllowResourceTypeWebsocket CrawlNewParamsVariant0AllowResourceType = "websocket"

const CrawlNewParamsVariant0AllowResourceTypeManifest CrawlNewParamsVariant0AllowResourceType = "manifest"

const CrawlNewParamsVariant0AllowResourceTypeSignedexchange CrawlNewParamsVariant0AllowResourceType = "signedexchange"

const CrawlNewParamsVariant0AllowResourceTypePing CrawlNewParamsVariant0AllowResourceType = "ping"

const CrawlNewParamsVariant0AllowResourceTypeCspviolationreport CrawlNewParamsVariant0AllowResourceType = "cspviolationreport"

const CrawlNewParamsVariant0AllowResourceTypePreflight CrawlNewParamsVariant0AllowResourceType = "preflight"

const CrawlNewParamsVariant0AllowResourceTypeOther CrawlNewParamsVariant0AllowResourceType = "other"

Authenticate param.Field[CrawlNewParamsVariant0Authenticate]optional

Body param: Provide credentials for HTTP authentication.

Password string

minLength1

Username string

minLength1

BestAttempt param.Field[bool]optional

Body param: Attempt to proceed when 'awaited' events fail or timeout.

Cookies param.Field[[]CrawlNewParamsVariant0Cookie]optional

Body param: Check options.

Name string

Cookie name.

Value string

Domain stringoptional

Expires float64optional

HTTPOnly booloptional

PartitionKey stringoptional

Path stringoptional

Priority CrawlNewParamsVariant0CookiesPriorityoptional

One of the following:

const CrawlNewParamsVariant0CookiesPriorityLow CrawlNewParamsVariant0CookiesPriority = "Low"

const CrawlNewParamsVariant0CookiesPriorityMedium CrawlNewParamsVariant0CookiesPriority = "Medium"

const CrawlNewParamsVariant0CookiesPriorityHigh CrawlNewParamsVariant0CookiesPriority = "High"

SameParty booloptional

SameSite CrawlNewParamsVariant0CookiesSameSiteoptional

One of the following:

const CrawlNewParamsVariant0CookiesSameSiteStrict CrawlNewParamsVariant0CookiesSameSite = "Strict"

const CrawlNewParamsVariant0CookiesSameSiteLax CrawlNewParamsVariant0CookiesSameSite = "Lax"

const CrawlNewParamsVariant0CookiesSameSiteNone CrawlNewParamsVariant0CookiesSameSite = "None"

Secure booloptional

SourcePort float64optional

SourceScheme CrawlNewParamsVariant0CookiesSourceSchemeoptional

One of the following:

const CrawlNewParamsVariant0CookiesSourceSchemeUnset CrawlNewParamsVariant0CookiesSourceScheme = "Unset"

const CrawlNewParamsVariant0CookiesSourceSchemeNonSecure CrawlNewParamsVariant0CookiesSourceScheme = "NonSecure"

const CrawlNewParamsVariant0CookiesSourceSchemeSecure CrawlNewParamsVariant0CookiesSourceScheme = "Secure"

URL stringoptional

CrawlPurposes param.Field[[]CrawlNewParamsVariant0CrawlPurpose]optional

Body param: List of crawl purposes to respect Content-Signal directives in robots.txt. Allowed values: 'search', 'ai-input', 'ai-train'. Learn more: https://contentsignals.org/. Default: ['search', 'ai-input', 'ai-train'].

const CrawlNewParamsVariant0CrawlPurposeSearch CrawlNewParamsVariant0CrawlPurpose = "search"

const CrawlNewParamsVariant0CrawlPurposeAIInput CrawlNewParamsVariant0CrawlPurpose = "ai-input"

const CrawlNewParamsVariant0CrawlPurposeAITrain CrawlNewParamsVariant0CrawlPurpose = "ai-train"

Depth param.Field[float64]optional

Body param: Maximum number of levels deep the crawler will traverse from the starting URL.

maximum100000

minimum1

EmulateMediaType param.Field[string]optional

Body param

Formats param.Field[[]CrawlNewParamsVariant0Format]optional

Body param: Formats to return. Default is html.

const CrawlNewParamsVariant0FormatHTML CrawlNewParamsVariant0Format = "html"

const CrawlNewParamsVariant0FormatMarkdown CrawlNewParamsVariant0Format = "markdown"

const CrawlNewParamsVariant0FormatJson CrawlNewParamsVariant0Format = "json"

GotoOptions param.Field[CrawlNewParamsVariant0GotoOptions]optional

Body param: Check options.

Referer stringoptional

ReferrerPolicy stringoptional

Timeout float64optional

maximum60000

WaitUntil CrawlNewParamsVariant0GotoOptionsWaitUntilUnionoptional

One of the following:

CrawlNewParamsVariant0GotoOptionsWaitUntilString

One of the following:

const CrawlNewParamsVariant0GotoOptionsWaitUntilStringLoad CrawlNewParamsVariant0GotoOptionsWaitUntilString = "load"

const CrawlNewParamsVariant0GotoOptionsWaitUntilStringDomcontentloaded CrawlNewParamsVariant0GotoOptionsWaitUntilString = "domcontentloaded"

const CrawlNewParamsVariant0GotoOptionsWaitUntilStringNetworkidle0 CrawlNewParamsVariant0GotoOptionsWaitUntilString = "networkidle0"

const CrawlNewParamsVariant0GotoOptionsWaitUntilStringNetworkidle2 CrawlNewParamsVariant0GotoOptionsWaitUntilString = "networkidle2"

CrawlNewParamsVariant0GotoOptionsWaitUntilArray

One of the following:

const CrawlNewParamsVariant0GotoOptionsWaitUntilArrayLoad CrawlNewParamsVariant0GotoOptionsWaitUntilArray = "load"

const CrawlNewParamsVariant0GotoOptionsWaitUntilArrayDomcontentloaded CrawlNewParamsVariant0GotoOptionsWaitUntilArray = "domcontentloaded"

const CrawlNewParamsVariant0GotoOptionsWaitUntilArrayNetworkidle0 CrawlNewParamsVariant0GotoOptionsWaitUntilArray = "networkidle0"

const CrawlNewParamsVariant0GotoOptionsWaitUntilArrayNetworkidle2 CrawlNewParamsVariant0GotoOptionsWaitUntilArray = "networkidle2"

JsonOptions param.Field[CrawlNewParamsVariant0JsonOptions]optional

Body param: Options for JSON extraction.

CustomAI []CrawlNewParamsVariant0JsonOptionsCustomAIoptional

Optional list of custom AI models to use for the request. The models will be tried in the order provided, and in case a model returns an error, the next one will be used as fallback.

Authorization string

Authorization token for the AI model: Bearer <token>.

Model string

AI model to use for the request. Must be formed as <provider>/<model_name>, e.g. workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast.

Prompt stringoptional

ResponseFormat CrawlNewParamsVariant0JsonOptionsResponseFormatoptional

Type string

JsonSchema map[string, CrawlNewParamsVariant0JsonOptionsResponseFormatJsonSchemaUnion]optional

Schema for the response format. More information here: https://developers.cloudflare.com/workers-ai/json-mode/.

One of the following:

UnionString

UnionFloat

UnionBool

CrawlNewParamsVariant0JsonOptionsResponseFormatJsonSchemaArray

Limit param.Field[float64]optional

Body param: Maximum number of URLs to crawl.

maximum100000

minimum1

MaxAge param.Field[float64]optional

Body param: Maximum age of a resource that can be returned from cache in seconds. Default is 1 day.

maximum604800

minimum0

ModifiedSince param.Field[int64]optional

Body param: Unix timestamp (seconds since epoch) indicating to only crawl pages that were modified since this time. For sitemap URLs with a lastmod field, this is compared directly. For other URLs, the crawler will use If-Modified-Since header when fetching. URLs without modification information (no lastmod in sitemap and no Last-Modified header support) will be crawled. Note: This works in conjunction with maxAge - both filters must pass for a cached resource to be used. Must be within the last year and not in the future.

exclusiveMinimum

minimum0

Options param.Field[CrawlNewParamsVariant0Options]optional

Body param: Additional options for the crawler.

ExcludePatterns []stringoptional

Exclude links matching the provided wildcard patterns in the crawl job. Example: 'https://example.com/privacy/**'.

IncludeExternalLinks booloptional

Include external links in the crawl job. If set to true, includeSubdomains is ignored.

IncludePatterns []stringoptional

Include only links matching the provided wildcard patterns in the crawl job. Include patterns are evaluated before exclude patterns. URLs that match any of the specified include patterns will be included in the crawl job. Example: 'https://example.com/blog/**'.

IncludeSubdomains booloptional

Include links to subdomains in the crawl job. This option is ignored if includeExternalLinks is true.

RejectRequestPattern param.Field[[]string]optional

Body param: Block undesired requests that match the provided regex patterns, eg. '/^.*.(css)'.

RejectResourceTypes param.Field[[]CrawlNewParamsVariant0RejectResourceType]optional

Body param: Block undesired requests that match the provided resource types, eg. 'image' or 'script'.

const CrawlNewParamsVariant0RejectResourceTypeDocument CrawlNewParamsVariant0RejectResourceType = "document"

const CrawlNewParamsVariant0RejectResourceTypeStylesheet CrawlNewParamsVariant0RejectResourceType = "stylesheet"

const CrawlNewParamsVariant0RejectResourceTypeImage CrawlNewParamsVariant0RejectResourceType = "image"

const CrawlNewParamsVariant0RejectResourceTypeMedia CrawlNewParamsVariant0RejectResourceType = "media"

const CrawlNewParamsVariant0RejectResourceTypeFont CrawlNewParamsVariant0RejectResourceType = "font"

const CrawlNewParamsVariant0RejectResourceTypeScript CrawlNewParamsVariant0RejectResourceType = "script"

const CrawlNewParamsVariant0RejectResourceTypeTexttrack CrawlNewParamsVariant0RejectResourceType = "texttrack"

const CrawlNewParamsVariant0RejectResourceTypeXHR CrawlNewParamsVariant0RejectResourceType = "xhr"

const CrawlNewParamsVariant0RejectResourceTypeFetch CrawlNewParamsVariant0RejectResourceType = "fetch"

const CrawlNewParamsVariant0RejectResourceTypePrefetch CrawlNewParamsVariant0RejectResourceType = "prefetch"

const CrawlNewParamsVariant0RejectResourceTypeEventsource CrawlNewParamsVariant0RejectResourceType = "eventsource"

const CrawlNewParamsVariant0RejectResourceTypeWebsocket CrawlNewParamsVariant0RejectResourceType = "websocket"

const CrawlNewParamsVariant0RejectResourceTypeManifest CrawlNewParamsVariant0RejectResourceType = "manifest"

const CrawlNewParamsVariant0RejectResourceTypeSignedexchange CrawlNewParamsVariant0RejectResourceType = "signedexchange"

const CrawlNewParamsVariant0RejectResourceTypePing CrawlNewParamsVariant0RejectResourceType = "ping"

const CrawlNewParamsVariant0RejectResourceTypeCspviolationreport CrawlNewParamsVariant0RejectResourceType = "cspviolationreport"

const CrawlNewParamsVariant0RejectResourceTypePreflight CrawlNewParamsVariant0RejectResourceType = "preflight"

const CrawlNewParamsVariant0RejectResourceTypeOther CrawlNewParamsVariant0RejectResourceType = "other"

Render param.Field[CrawlNewParamsVariant0Render]optional

Body param: Whether to render the page or fetch static content. True by default.

const CrawlNewParamsVariant0RenderTrue CrawlNewParamsVariant0Render = true

SetExtraHTTPHeaders param.Field[map[string, string]]optional

Body param

SetJavaScriptEnabled param.Field[bool]optional

Body param

Source param.Field[CrawlNewParamsVariant0Source]optional

Body param: Source of links to crawl. 'sitemaps' - only crawl URLs from sitemaps, 'links' - only crawl URLs scraped from pages, 'all' - crawl both sitemap and scraped links (default).

const CrawlNewParamsVariant0SourceSitemaps CrawlNewParamsVariant0Source = "sitemaps"

const CrawlNewParamsVariant0SourceLinks CrawlNewParamsVariant0Source = "links"

const CrawlNewParamsVariant0SourceAll CrawlNewParamsVariant0Source = "all"

Viewport param.Field[CrawlNewParamsVariant0Viewport]optional

Body param: Check options.

Height float64

Width float64

DeviceScaleFactor float64optional

HasTouch booloptional

IsLandscape booloptional

IsMobile booloptional

WaitForSelector param.Field[CrawlNewParamsVariant0WaitForSelector]optional

Body param: Wait for the selector to appear in page. Check options.

Selector string

Hidden CrawlNewParamsVariant0WaitForSelectorHiddenoptional

Timeout float64optional

maximum120000

Visible CrawlNewParamsVariant0WaitForSelectorVisibleoptional

WaitForTimeout param.Field[float64]optional

Body param: Waits for a specified timeout before continuing.

maximum120000

ReturnsExpand Collapse

type CrawlNewResponseEnvelopeResult string

Crawl job ID.

Crawl websites.

package main

import (
  "context"
  "fmt"

  "github.com/cloudflare/cloudflare-go"
  "github.com/cloudflare/cloudflare-go/browser_rendering"
  "github.com/cloudflare/cloudflare-go/option"
)

func main() {
  client := cloudflare.NewClient(
    option.WithAPIToken("Sn3lZJTBX6kkg7OdcBUAxOO963GEIyGQqnFTOFYY"),
  )
  crawl, err := client.BrowserRendering.Crawl.New(context.TODO(), browser_rendering.CrawlNewParams{
    AccountID: cloudflare.F("account_id"),
    Body: browser_rendering.CrawlNewParamsBodyObject{
      URL: cloudflare.F("https://example.com"),
    },
  })
  if err != nil {
    panic(err.Error())
  }
  fmt.Printf("%+v\n", crawl)
}

{
  "result": "result",
  "success": true,
  "errors": [
    {
      "code": 0,
      "message": "message"
    }
  ]
}

{
  "errors": [
    {
      "code": 2001,
      "message": "Rate limit exceeded"
    }
  ],
  "success": false
}

Returns Examples

{
  "result": "result",
  "success": true,
  "errors": [
    {
      "code": 0,
      "message": "message"
    }
  ]
}

{
  "errors": [
    {
      "code": 2001,
      "message": "Rate limit exceeded"
    }
  ],
  "success": false
}