{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/auditae-app/json-schema/auditae-app-crawl-settings-schema.json",
  "title": "CrawlSettings",
  "description": "What a crawl covers and how it fetches (Site Audit crawl settings). Every field is optional: a crawl uses the call's value, else the site's saved default (GET/PUT /api/v1/crawl/settings), else the built-in default, which crawls the whole site as before. Masks use robots.txt syntax against path + query: * any characters, $ the end, anything else a prefix (/shoes also matches /shoes-men; /shoes/ is the folder). Exclude beats include. The crawler always identifies as AuditAE-Bot.",
  "x-generated": "2026-11-02",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/auditae-app-openapi.json#/components/schemas/CrawlSettings",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "source": {
      "type": "string",
      "enum": [
        "site",
        "sitemaps",
        "sitemap_url",
        "urls"
      ],
      "description": "Built-in default: site. site: links and sitemap. sitemaps: only the robots.txt Sitemap: files (else /sitemap.xml). sitemap_url: only that file or index. urls: only the listed URLs (plus the homepage); links are counted, never followed, by every source but site."
    },
    "sitemap_url": {
      "type": "string",
      "description": "source sitemap_url: the sitemap file or index, on the site."
    },
    "urls": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "minItems": 1,
      "maxItems": 10000,
      "description": "source urls: the URLs to crawl, on the site (not stored with the run; url_count and urls_sha256 are)."
    },
    "include": {
      "type": "array",
      "items": {
        "type": "string",
        "maxLength": 200
      },
      "maxItems": 20,
      "description": "Only URLs matching one of these masks are crawled (the homepage is always fetched)."
    },
    "exclude": {
      "type": "array",
      "items": {
        "type": "string",
        "maxLength": 200
      },
      "maxItems": 50,
      "description": "URLs matching one of these masks are skipped (skipped_by_rules). A seed matching one is refused (code seed_excluded)."
    },
    "ignore_params": {
      "type": "array",
      "items": {
        "type": "string",
        "pattern": "^[\\w.\\-*]{1,60}$"
      },
      "maxItems": 30,
      "description": "Query parameters dropped from URLs (* wildcards); [\"*\"] drops every query string."
    },
    "subdomains": {
      "type": "boolean",
      "description": "Built-in default: false. Also crawl subdomains of the site's domain; each obeys its own robots.txt (up to 25 hosts)."
    },
    "robots_agent": {
      "type": "string",
      "enum": [
        "*",
        "Googlebot",
        "Bingbot",
        "OAI-SearchBot",
        "ChatGPT-User",
        "Claude-SearchBot",
        "PerplexityBot"
      ],
      "description": "Built-in default: *. Obey this bot's robots.txt group instead of User-agent: *. The user agent doesn't change."
    },
    "device": {
      "type": "string",
      "enum": [
        "desktop",
        "mobile"
      ],
      "description": "Built-in default: desktop. The AuditAE-Bot desktop or mobile browser string."
    },
    "speed": {
      "type": "string",
      "enum": [
        "normal",
        "polite",
        "robots"
      ],
      "description": "Built-in default: normal. normal: 3 pages at a time. polite: 1 page a second (10,000 pages ≈ 3 hours). robots: robots.txt Crawl-delay, up to 10 s, else polite."
    },
    "cwv_urls": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "maxItems": 10,
      "description": "Pages on the site to measure Core Web Vitals on, instead of the automatic sample."
    },
    "exclude_checks": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "maxItems": 60,
      "description": "Check ids left out of Site Health and the theme scores (not_run excluded). A site default applies to every crawl's scores at once."
    },
    "save_as_default": {
      "type": "boolean",
      "description": "POST /api/v1/crawl: also save these settings (not urls) as the site's default."
    }
  }
}