{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/auditae-app/json-schema/auditae-app-crawl-result-schema.json",
  "title": "CrawlResult",
  "description": "A crawl run. Runs crawled with Site Audit checks (coverage \"full\") add Site Health, counts, theme scores and the failing checks; runs crawled before them answer as before, plus coverage \"legacy\" and a note.",
  "x-generated": "2026-11-02",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/auditae-app-openapi.json#/components/schemas/CrawlResult",
  "type": "object",
  "required": [
    "run_id",
    "status",
    "seed_url",
    "pages_crawled",
    "pages_failed",
    "skipped_by_robots",
    "billed_cents",
    "sitemaps",
    "issues_summary",
    "pages"
  ],
  "properties": {
    "run_id": {
      "type": "string",
      "format": "uuid"
    },
    "status": {
      "type": "string",
      "enum": [
        "running",
        "complete",
        "partial",
        "failed"
      ],
      "description": "running: still crawling in the background — poll GET /api/v1/crawl/{id}. partial: stopped early (error_message out_of_credits when the balance ran out)."
    },
    "seed_url": {
      "type": "string",
      "format": "uri"
    },
    "max_pages": {
      "type": "integer"
    },
    "pages_crawled": {
      "type": "integer"
    },
    "pages_found": {
      "type": "integer",
      "description": "Pages queued so far (links found plus sitemap URLs), up to max_pages."
    },
    "pages_failed": {
      "type": "integer"
    },
    "skipped_by_robots": {
      "type": "integer"
    },
    "skipped_by_rules": {
      "type": "integer",
      "description": "Site Audit runs: URLs the crawl settings' include / exclude rules (or the subdomain cap) skipped, never fetched."
    },
    "settings": {
      "$ref": "#/$defs/CrawlRunSettings",
      "description": "The crawl settings this run used (runs from October 2026 on), plus url_count and urls_sha256 for a URL list."
    },
    "rejected": {
      "type": "object",
      "description": "POST with a URL list: the listed URLs that weren't crawled (off the site, not a URL, a file, a duplicate, or disallowed by robots.txt), with up to 10 samples.",
      "properties": {
        "count": {
          "type": "integer"
        },
        "samples": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "maxItems": 10
        },
        "reasons": {
          "type": "object",
          "additionalProperties": {
            "type": "integer"
          }
        },
        "robots": {
          "type": "integer"
        }
      }
    },
    "billed_cents": {
      "type": "integer",
      "description": "1¢ per 10 pages crawled, billed as the crawl runs."
    },
    "error_message": {
      "type": [
        "string",
        "null"
      ]
    },
    "coverage": {
      "type": "string",
      "enum": [
        "full",
        "partial",
        "legacy"
      ],
      "description": "full: graded with Site Audit checks. partial: graded, but the crawl stopped early (max_pages or the balance), so Site Health covers the pages crawled and some site-wide checks weren't judged (not_run; note says which). legacy: crawled before them — today's fields only; crawl again for Site Health."
    },
    "stage": {
      "type": [
        "string",
        "null"
      ],
      "enum": [
        "pages",
        "resources",
        "finalize",
        "done",
        null
      ],
      "description": "Where a Site Audit run is: crawling pages, checking links and resources, scoring, or done."
    },
    "provisional": {
      "type": "boolean",
      "description": "true while the run crawls: counts and checks cover the page checks of the pages crawled so far (n = pages); site_health is null."
    },
    "partial": {
      "type": "boolean",
      "description": "The crawl stopped early (max_pages or the balance), so some checks weren't judged."
    },
    "site_health": {
      "$ref": "#/$defs/SiteHealth"
    },
    "counts": {
      "type": "object",
      "required": [
        "errors",
        "warnings",
        "notices"
      ],
      "properties": {
        "errors": {
          "type": "integer"
        },
        "warnings": {
          "type": "integer"
        },
        "notices": {
          "type": "integer"
        }
      },
      "description": "Failing items by severity (Σ n over failing checks)."
    },
    "failing": {
      "type": "object",
      "required": [
        "errors",
        "warnings",
        "notices"
      ],
      "properties": {
        "errors": {
          "type": "integer"
        },
        "warnings": {
          "type": "integer"
        },
        "notices": {
          "type": "integer"
        }
      },
      "description": "Distinct failing checks by severity."
    },
    "themes": {
      "type": "object",
      "propertyNames": {
        "enum": [
          "crawlability",
          "https",
          "international",
          "core_web_vitals",
          "performance",
          "internal_linking",
          "markup",
          "ai_search",
          "on_page"
        ]
      },
      "additionalProperties": {
        "$ref": "#/$defs/ThemeScore"
      },
      "description": "Score per theme (0–100, null when none of its checks ran)."
    },
    "cwv_state": {
      "type": [
        "string",
        "null"
      ],
      "enum": [
        "pending",
        "done",
        "skipped",
        "failed",
        null
      ],
      "description": "Core Web Vitals (PageSpeed mobile on up to 10 pages) run a few minutes after the crawl completes; pending until then, skipped when there was nothing to measure, failed when no page could be measured. They score their theme only, never Site Health."
    },
    "checks": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/CrawlCheck"
      },
      "maxItems": 40,
      "description": "Failing checks, worst first (errors, then by n): id, severity, title, n, of. GET /api/v1/crawl/{id}/checks/{check} lists one check's URLs."
    },
    "not_run": {
      "type": "array",
      "items": {
        "type": "object",
        "required": [
          "id",
          "reason"
        ],
        "properties": {
          "id": {
            "type": "string"
          },
          "reason": {
            "type": "string",
            "enum": [
              "verified_bots",
              "capped",
              "no_sitemap",
              "sitemap_truncated",
              "sitemap_unreadable",
              "scoped",
              "no_probe",
              "probe_failed",
              "no_resources",
              "unknown_sizes",
              "cwv_pending",
              "excluded",
              "image_format_unverified"
            ]
          }
        }
      },
      "description": "Checks this run couldn't judge, and why."
    },
    "checks_summary": {
      "oneOf": [
        {
          "$ref": "#/$defs/ChecksSummary"
        },
        {
          "type": "null"
        }
      ],
      "description": "GET only: the run's full check results as stored (null while it crawls)."
    },
    "vs_previous": {
      "type": "object",
      "additionalProperties": true,
      "description": "The change since the site's previous scored crawl, in brief: base_run_id, base_started_at, mode, health_delta, new_errors, fixed_errors, new_items, fixed_items, top_new and top_fixed ([{ id, severity, n }], ≤5), incomplete_baseline? (true when a crawl stopped at max_pages: new can include pages the other crawl didn't reach) and note?. Absent on a first crawl."
    },
    "comparison": {
      "oneOf": [
        {
          "$ref": "#/$defs/CrawlComparison"
        },
        {
          "type": "null"
        }
      ],
      "description": "Site Audit runs: the full comparison with the site's previous scored crawl, stored when the run finished. null on a first crawl and while crawling."
    },
    "next_step": {
      "type": "string"
    },
    "note": {
      "type": "string",
      "description": "What to know about this answer (still crawling, legacy run, scoring failed)."
    },
    "sitemaps": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Up to 50 page URLs from the site's sitemaps."
    },
    "issues_summary": {
      "type": [
        "object",
        "null"
      ],
      "additionalProperties": true,
      "description": "Legacy issue rollup (broken links, missing meta, duplicates …), kept for existing clients; prefer checks. null while running. On runs crawled with Site Audit checks each list holds at most 500 entries."
    },
    "check": {
      "type": "string",
      "description": "GET with ?check=: the check the pages were filtered by."
    },
    "pages_total": {
      "type": "integer",
      "description": "GET with ?check, ?limit or ?offset: pages matching, before pagination."
    },
    "offset": {
      "type": "integer"
    },
    "next_offset": {
      "type": [
        "integer",
        "null"
      ],
      "description": "Pass as offset for the next pages; null on the last."
    },
    "pages": {
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": true,
        "properties": {
          "checks": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Site Audit runs: the ids of the checks this page fails."
          },
          "inlinks": {
            "type": [
              "integer",
              "null"
            ],
            "description": "Site Audit runs: internal links pointing at this page."
          },
          "in_sitemap": {
            "type": [
              "boolean",
              "null"
            ],
            "description": "Site Audit runs: listed in the site's sitemaps."
          },
          "ilr": {
            "type": [
              "integer",
              "null"
            ],
            "minimum": 0,
            "maximum": 100,
            "description": "Site Audit runs: Internal LinkRank, 0–100 (the homepage is 100)."
          }
        }
      },
      "description": "Every crawled page once the run ends (or one page of them with ?limit / ?offset / ?check); empty while running and with ?pages=none."
    }
  },
  "$defs": {
    "ChecksSummary": {
      "type": "object",
      "description": "A Site Audit run's check results as stored (version 1, ≤96 KB). Keys may be added; read v before relying on the shape.",
      "required": [
        "v",
        "health",
        "counts",
        "failing",
        "themes",
        "fail",
        "pass",
        "not_run"
      ],
      "additionalProperties": true,
      "properties": {
        "v": {
          "type": "integer",
          "const": 1
        },
        "catalog": {
          "type": "integer",
          "description": "Check catalog version the run was graded with."
        },
        "rev": {
          "type": "string"
        },
        "partial": {
          "type": "boolean"
        },
        "health": {
          "type": "integer",
          "minimum": 0,
          "maximum": 100
        },
        "counts": {
          "type": "object",
          "required": [
            "errors",
            "warnings",
            "notices"
          ],
          "properties": {
            "errors": {
              "type": "integer"
            },
            "warnings": {
              "type": "integer"
            },
            "notices": {
              "type": "integer"
            }
          },
          "description": "Failing items by severity."
        },
        "failing": {
          "type": "object",
          "required": [
            "errors",
            "warnings",
            "notices"
          ],
          "properties": {
            "errors": {
              "type": "integer"
            },
            "warnings": {
              "type": "integer"
            },
            "notices": {
              "type": "integer"
            }
          },
          "description": "Distinct failing checks by severity."
        },
        "themes": {
          "type": "object",
          "propertyNames": {
            "enum": [
              "crawlability",
              "https",
              "international",
              "core_web_vitals",
              "performance",
              "internal_linking",
              "markup",
              "ai_search",
              "on_page"
            ]
          },
          "additionalProperties": {
            "$ref": "#/$defs/ThemeScore"
          }
        },
        "fail": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "required": [
              "n",
              "of",
              "s"
            ],
            "properties": {
              "n": {
                "type": "integer"
              },
              "of": {
                "type": "integer"
              },
              "s": {
                "type": "array",
                "items": {
                  "type": "string"
                },
                "maxItems": 5
              }
            }
          },
          "description": "Failing check id → n failing, of checked, s samples."
        },
        "pass": {
          "type": "object",
          "additionalProperties": {
            "type": "integer"
          },
          "description": "Passing check id → items checked."
        },
        "not_run": {
          "type": "object",
          "additionalProperties": {
            "type": "string",
            "enum": [
              "verified_bots",
              "capped",
              "no_sitemap",
              "sitemap_truncated",
              "sitemap_unreadable",
              "scoped",
              "no_probe",
              "probe_failed",
              "no_resources",
              "unknown_sizes",
              "cwv_pending",
              "excluded",
              "image_format_unverified"
            ]
          },
          "description": "Check id → why it wasn't judged."
        },
        "pages": {
          "type": "object",
          "additionalProperties": true,
          "description": "crawled, parsed, indexable, broken, redirected, blocked_by_robots, skipped_by_rules, limit_reached."
        },
        "resources": {
          "type": "object",
          "additionalProperties": true,
          "description": "found, checked, unverified, unchecked."
        },
        "groups": {
          "type": "object",
          "additionalProperties": true,
          "description": "Duplicate groups, broken links and resources, redirect chains, hreflang problems."
        },
        "markup": {
          "type": "object",
          "additionalProperties": true,
          "description": "Structured-data items, valid items and the top types."
        },
        "details": {
          "type": "object",
          "additionalProperties": true,
          "description": "Per-theme details (status codes, depth, sitemaps, timings …)."
        }
      }
    },
    "CrawlCheck": {
      "type": "object",
      "description": "One of the 174 Site Audit checks. In a crawl's checks list only id, severity, title, n and of are sent.",
      "required": [
        "id",
        "severity",
        "title"
      ],
      "properties": {
        "id": {
          "type": "string",
          "description": "Stable check id (never renamed)."
        },
        "title": {
          "type": "string"
        },
        "severity": {
          "type": "string",
          "enum": [
            "error",
            "warning",
            "notice"
          ]
        },
        "themes": {
          "type": "array",
          "items": {
            "type": "string",
            "enum": [
              "crawlability",
              "https",
              "international",
              "core_web_vitals",
              "performance",
              "internal_linking",
              "markup",
              "ai_search",
              "on_page"
            ]
          },
          "description": "First = primary theme."
        },
        "scope": {
          "type": "string",
          "enum": [
            "page",
            "cross",
            "site",
            "resource",
            "cwv"
          ],
          "description": "page: per page · cross: across pages · site: once per run · resource: links, images, scripts · cwv: Core Web Vitals pass."
        },
        "rule": {
          "type": "string",
          "description": "The exact rule, with thresholds."
        },
        "why": {
          "type": "string"
        },
        "fix": {
          "type": "string"
        },
        "fix_tools": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "MCP tools that act on it."
        },
        "semrush_ids": {
          "type": "array",
          "items": {
            "type": "integer"
          },
          "description": "The Semrush Site Audit issue ids it maps to."
        },
        "status": {
          "type": "string",
          "enum": [
            "fail",
            "pass",
            "not_run"
          ]
        },
        "n": {
          "type": "integer",
          "description": "Items failing (0 when it passed)."
        },
        "of": {
          "type": [
            "integer",
            "null"
          ],
          "description": "Items checked (pages, links, images …); null while provisional."
        },
        "samples": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "maxItems": 5,
          "description": "Up to 5 examples, \"url\" or \"url → detail\"."
        },
        "reason": {
          "type": "string",
          "enum": [
            "verified_bots",
            "capped",
            "no_sitemap",
            "sitemap_truncated",
            "sitemap_unreadable",
            "scoped",
            "no_probe",
            "probe_failed",
            "no_resources",
            "unknown_sizes",
            "cwv_pending",
            "excluded",
            "image_format_unverified"
          ],
          "description": "not_run only: why it wasn't judged."
        }
      }
    },
    "CrawlComparison": {
      "type": "object",
      "description": "What changed between two Site Audit runs of a site (version 1, ≤40 KB). mode page: same check version, so new / fixed / removed are counted from the stored per-page results (pages for page checks, 0/1 for site-wide checks, the change in n for link, image and script checks). mode summary: different check versions — totals only; new / fixed and the deltas are null.",
      "required": [
        "v",
        "base_run_id",
        "base_started_at",
        "mode",
        "versions_match",
        "health",
        "counts",
        "themes",
        "totals",
        "checks",
        "pages",
        "robots",
        "sitemap"
      ],
      "additionalProperties": true,
      "properties": {
        "v": {
          "type": "integer",
          "const": 1
        },
        "base_run_id": {
          "type": "string",
          "format": "uuid"
        },
        "base_started_at": {
          "type": "string",
          "format": "date-time"
        },
        "mode": {
          "type": "string",
          "enum": [
            "page",
            "summary"
          ]
        },
        "versions_match": {
          "type": "boolean"
        },
        "scope_changed": {
          "type": "boolean",
          "description": "The runs' crawl settings scope differs (source, URL list, include/exclude rules, ignored parameters, subdomains or robots group); pages added and removed are then null."
        },
        "note": {
          "type": "string"
        },
        "health": {
          "type": "object",
          "properties": {
            "from": {
              "type": [
                "integer",
                "null"
              ]
            },
            "to": {
              "type": "integer"
            },
            "delta": {
              "type": [
                "integer",
                "null"
              ],
              "description": "null when the check versions differ."
            }
          }
        },
        "counts": {
          "type": "object",
          "additionalProperties": true,
          "description": "errors, warnings, notices: { from, to } failing items."
        },
        "themes": {
          "type": "object",
          "propertyNames": {
            "enum": [
              "crawlability",
              "https",
              "international",
              "core_web_vitals",
              "performance",
              "internal_linking",
              "markup",
              "ai_search",
              "on_page"
            ]
          },
          "additionalProperties": {
            "type": "object",
            "properties": {
              "from": {
                "type": [
                  "integer",
                  "null"
                ]
              },
              "to": {
                "type": [
                  "integer",
                  "null"
                ]
              },
              "delta": {
                "type": [
                  "integer",
                  "null"
                ]
              }
            }
          },
          "description": "Per theme score from → to. Core Web Vitals fill in when they're measured, a few minutes after the crawl."
        },
        "totals": {
          "type": "object",
          "properties": {
            "new_errors": {
              "type": [
                "integer",
                "null"
              ]
            },
            "fixed_errors": {
              "type": [
                "integer",
                "null"
              ]
            },
            "new_items": {
              "type": [
                "integer",
                "null"
              ]
            },
            "fixed_items": {
              "type": [
                "integer",
                "null"
              ]
            }
          }
        },
        "checks": {
          "type": "array",
          "maxItems": 60,
          "description": "Changed checks only, biggest change in Site Health penalty first.",
          "items": {
            "type": "object",
            "required": [
              "id",
              "severity",
              "from",
              "to",
              "new",
              "fixed"
            ],
            "properties": {
              "id": {
                "type": "string"
              },
              "severity": {
                "type": "string",
                "enum": [
                  "error",
                  "warning",
                  "notice"
                ]
              },
              "from": {
                "type": "integer",
                "description": "Failing items on the base run."
              },
              "to": {
                "type": "integer",
                "description": "Failing items now."
              },
              "new": {
                "type": [
                  "integer",
                  "null"
                ]
              },
              "fixed": {
                "type": [
                  "integer",
                  "null"
                ]
              },
              "removed": {
                "type": [
                  "integer",
                  "null"
                ],
                "description": "Page checks: pages failing on the base run that this crawl didn't reach."
              },
              "new_s": {
                "type": "array",
                "items": {
                  "type": "string"
                },
                "maxItems": 3,
                "description": "Sample URLs newly failing."
              },
              "fixed_s": {
                "type": "array",
                "items": {
                  "type": "string"
                },
                "maxItems": 3,
                "description": "Sample URLs fixed (page checks only)."
              }
            }
          }
        },
        "pages": {
          "type": [
            "object",
            "null"
          ],
          "additionalProperties": true,
          "description": "Page mode: comparable (false when a crawl stopped at max_pages: added and removed are then null), added, removed, status_changed, became_broken, became_noindex, became_indexable, title_changed, samples { added, removed, status: [{ url, from, to }] }."
        },
        "robots": {
          "type": "object",
          "properties": {
            "changed": {
              "type": "boolean"
            },
            "added_rules": {
              "type": "array",
              "items": {
                "type": "string"
              },
              "description": "User-agent: * rules added, e.g. \"Disallow: /cart\"."
            },
            "removed_rules": {
              "type": "array",
              "items": {
                "type": "string"
              }
            }
          }
        },
        "sitemap": {
          "type": "object",
          "properties": {
            "urls_from": {
              "type": "integer"
            },
            "urls_to": {
              "type": "integer"
            }
          }
        }
      }
    },
    "CrawlRunSettings": {
      "type": "object",
      "additionalProperties": false,
      "description": "The settings a run used: the request's settings resolved against the site's defaults (the URL list itself isn't stored).",
      "properties": {
        "source": {
          "type": "string",
          "enum": [
            "site",
            "sitemaps",
            "sitemap_url",
            "urls"
          ],
          "description": "Built-in default: site. site: links and sitemap. sitemaps: only the robots.txt Sitemap: files (else /sitemap.xml). sitemap_url: only that file or index. urls: only the listed URLs (plus the homepage); links are counted, never followed, by every source but site."
        },
        "sitemap_url": {
          "type": "string",
          "description": "source sitemap_url: the sitemap file or index, on the site."
        },
        "urls": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "minItems": 1,
          "maxItems": 10000,
          "description": "source urls: the URLs to crawl, on the site (not stored with the run; url_count and urls_sha256 are)."
        },
        "include": {
          "type": "array",
          "items": {
            "type": "string",
            "maxLength": 200
          },
          "maxItems": 20,
          "description": "Only URLs matching one of these masks are crawled (the homepage is always fetched)."
        },
        "exclude": {
          "type": "array",
          "items": {
            "type": "string",
            "maxLength": 200
          },
          "maxItems": 50,
          "description": "URLs matching one of these masks are skipped (skipped_by_rules). A seed matching one is refused (code seed_excluded)."
        },
        "ignore_params": {
          "type": "array",
          "items": {
            "type": "string",
            "pattern": "^[\\w.\\-*]{1,60}$"
          },
          "maxItems": 30,
          "description": "Query parameters dropped from URLs (* wildcards); [\"*\"] drops every query string."
        },
        "subdomains": {
          "type": "boolean",
          "description": "Built-in default: false. Also crawl subdomains of the site's domain; each obeys its own robots.txt (up to 25 hosts)."
        },
        "robots_agent": {
          "type": "string",
          "enum": [
            "*",
            "Googlebot",
            "Bingbot",
            "OAI-SearchBot",
            "ChatGPT-User",
            "Claude-SearchBot",
            "PerplexityBot"
          ],
          "description": "Built-in default: *. Obey this bot's robots.txt group instead of User-agent: *. The user agent doesn't change."
        },
        "device": {
          "type": "string",
          "enum": [
            "desktop",
            "mobile"
          ],
          "description": "Built-in default: desktop. The AuditAE-Bot desktop or mobile browser string."
        },
        "speed": {
          "type": "string",
          "enum": [
            "normal",
            "polite",
            "robots"
          ],
          "description": "Built-in default: normal. normal: 3 pages at a time. polite: 1 page a second (10,000 pages ≈ 3 hours). robots: robots.txt Crawl-delay, up to 10 s, else polite."
        },
        "cwv_urls": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "maxItems": 10,
          "description": "Pages on the site to measure Core Web Vitals on, instead of the automatic sample."
        },
        "exclude_checks": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "maxItems": 60,
          "description": "Check ids left out of Site Health and the theme scores (not_run excluded). A site default applies to every crawl's scores at once."
        },
        "save_as_default": {
          "type": "boolean",
          "description": "POST /api/v1/crawl: also save these settings (not urls) as the site's default."
        },
        "url_count": {
          "type": "integer",
          "description": "How many listed URLs it was given."
        },
        "urls_sha256": {
          "type": "string",
          "description": "The hash of its URL list's keys."
        }
      }
    },
    "SiteHealth": {
      "type": [
        "integer",
        "null"
      ],
      "minimum": 0,
      "maximum": 100,
      "description": "Site Health, 0–100: 100 − Σ w × (0.5 + 0.5 × failing ÷ checked) over failing error (w 4) and warning (w 1) checks, floored at 0. Notices and Core Web Vitals don't count. null while the run crawls and on legacy runs."
    },
    "ThemeScore": {
      "type": "object",
      "required": [
        "score",
        "failing",
        "ran"
      ],
      "properties": {
        "score": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0,
          "maximum": 100
        },
        "failing": {
          "type": "integer",
          "description": "Failing checks in the theme."
        },
        "ran": {
          "type": "integer",
          "description": "Checks in the theme that ran."
        }
      }
    }
  }
}