curl --request GET \
--url https://api.scrunchai.com/v1/{brand_id}/crawl-readiness \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.scrunchai.com/v1/{brand_id}/crawl-readiness"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.scrunchai.com/v1/{brand_id}/crawl-readiness', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.scrunchai.com/v1/{brand_id}/crawl-readiness",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.scrunchai.com/v1/{brand_id}/crawl-readiness"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.scrunchai.com/v1/{brand_id}/crawl-readiness")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.scrunchai.com/v1/{brand_id}/crawl-readiness")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"domain": "<string>",
"robots_txt_present": true,
"is_spa": true,
"is_blocking_bots": true,
"blocked_pages": [
{
"path": "<string>",
"url": "<string>",
"blocked_by": [
"<string>"
]
}
],
"pages_truncated": true
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Get AI Crawl Readiness
Whether AI crawlers can reach one of the brand’s domains, based on the brand’s most recent finished crawl of it. These are the same signals the AI Access card shows in the Scrunch dashboard.
Pass domain to choose which of the brand’s domains to describe; omit it for the primary website. The response is null when the brand owns the domain but no crawl of it has finished yet; requesting a domain the brand does not own returns 404.
Reading the fields:
robots_txt_present: whether a robots.txt was found. False means permissive, not blocked: with no robots.txt, crawlers are allowed by default.is_blocking_bots: the site turns crawlers away, derived from the share of pages that answered HTTP 403 across the whole crawl. This is a server-level verdict and is unrelated toblocked_pages, which is about robots.txt.is_spa: most pages render client-side, so a crawler that does not run JavaScript reads little or nothing.blocked_pages: pages robots.txt blocks, with the AI assistant families blocked from each. This is a sample, not a complete list: only the 50 shallowest pages of the crawl are checked.pages_truncatedis true when the crawl held more than that, so an empty or short list does not mean nothing else on the site is blocked.
curl --request GET \
--url https://api.scrunchai.com/v1/{brand_id}/crawl-readiness \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.scrunchai.com/v1/{brand_id}/crawl-readiness"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.scrunchai.com/v1/{brand_id}/crawl-readiness', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.scrunchai.com/v1/{brand_id}/crawl-readiness",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.scrunchai.com/v1/{brand_id}/crawl-readiness"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.scrunchai.com/v1/{brand_id}/crawl-readiness")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.scrunchai.com/v1/{brand_id}/crawl-readiness")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"domain": "<string>",
"robots_txt_present": true,
"is_spa": true,
"is_blocking_bots": true,
"blocked_pages": [
{
"path": "<string>",
"url": "<string>",
"blocked_by": [
"<string>"
]
}
],
"pages_truncated": true
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Path Parameters
The unique identifier for the brand.
Query Parameters
Which of the brand's domains to describe. Omit for the primary website.
1Response
Successful Response. null when the brand owns the domain but no crawl of it has finished yet.
AI crawl readiness signals for one of the brand's domains, from its most recent finished crawl.
The domain the readiness signals describe.
Whether a robots.txt was found for the domain. False means permissive, not blocked: with no robots.txt, crawlers are allowed by default.
Most pages render client-side, so a crawler that does not run JavaScript reads little or nothing.
The site turns crawlers away at the server level, derived from the share of pages that answered HTTP 403 across the whole crawl.
Pages robots.txt blocks from at least one AI assistant family. A sample from the 50 shallowest pages of the crawl, not a complete list.
Show child attributes
Show child attributes
True when the crawl held more pages than the 50-page scan cap, so blocked_pages is a sample.