{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/ragfeed-de/json-schema/ragfeed-de-crawl-request-schema.json",
  "title": "CrawlRequest",
  "x-generated": "2026-10-31",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/ragfeed-de-openapi.json#/components/schemas/CrawlRequest",
  "properties": {
    "url": {
      "type": "string",
      "title": "Url"
    },
    "prompt": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Prompt"
    },
    "excludePaths": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "title": "Excludepaths"
    },
    "includePaths": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "title": "Includepaths"
    },
    "maxDiscoveryDepth": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Maxdiscoverydepth"
    },
    "sitemap": {
      "type": "string",
      "enum": [
        "skip",
        "include",
        "only"
      ],
      "title": "Sitemap",
      "default": "include"
    },
    "ignoreQueryParameters": {
      "type": "boolean",
      "title": "Ignorequeryparameters",
      "default": false
    },
    "regexOnFullURL": {
      "type": "boolean",
      "title": "Regexonfullurl",
      "default": false
    },
    "limit": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Limit",
      "default": 10000
    },
    "crawlEntireDomain": {
      "type": "boolean",
      "title": "Crawlentiredomain",
      "default": false
    },
    "allowExternalLinks": {
      "type": "boolean",
      "title": "Allowexternallinks",
      "default": false
    },
    "allowSubdomains": {
      "type": "boolean",
      "title": "Allowsubdomains",
      "default": false
    },
    "ignoreRobotsTxt": {
      "type": "boolean",
      "title": "Ignorerobotstxt",
      "default": false
    },
    "robotsUserAgent": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Robotsuseragent"
    },
    "delay": {
      "anyOf": [
        {
          "type": "number"
        },
        {
          "type": "null"
        }
      ],
      "title": "Delay"
    },
    "maxConcurrency": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Maxconcurrency"
    },
    "webhook": {
      "anyOf": [
        {
          "$ref": "#/$defs/Webhook"
        },
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Webhook"
    },
    "scrapeOptions": {
      "anyOf": [
        {
          "$ref": "#/$defs/ScrapeOptions"
        },
        {
          "type": "null"
        }
      ]
    },
    "zeroDataRetention": {
      "type": "boolean",
      "title": "Zerodataretention",
      "default": false
    }
  },
  "additionalProperties": true,
  "type": "object",
  "required": [
    "url"
  ],
  "$defs": {
    "Location": {
      "properties": {
        "country": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "title": "Country",
          "default": "US"
        },
        "languages": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Languages"
        }
      },
      "additionalProperties": true,
      "type": "object",
      "title": "Location"
    },
    "ScrapeOptions": {
      "properties": {
        "formats": {
          "items": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "additionalProperties": true,
                "type": "object"
              }
            ]
          },
          "type": "array",
          "title": "Formats"
        },
        "onlyMainContent": {
          "type": "boolean",
          "title": "Onlymaincontent",
          "default": true
        },
        "onlyCleanContent": {
          "type": "boolean",
          "title": "Onlycleancontent",
          "default": false
        },
        "includeTags": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Includetags"
        },
        "excludeTags": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Excludetags"
        },
        "maxAge": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Maxage"
        },
        "minAge": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Minage"
        },
        "headers": {
          "anyOf": [
            {
              "additionalProperties": {
                "type": "string"
              },
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "title": "Headers"
        },
        "waitFor": {
          "type": "integer",
          "title": "Waitfor",
          "default": 0
        },
        "mobile": {
          "type": "boolean",
          "title": "Mobile",
          "default": false
        },
        "skipTlsVerification": {
          "type": "boolean",
          "title": "Skiptlsverification",
          "default": true
        },
        "timeout": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Timeout"
        },
        "parsers": {
          "anyOf": [
            {
              "items": {
                "anyOf": [
                  {
                    "type": "string"
                  },
                  {
                    "additionalProperties": true,
                    "type": "object"
                  }
                ]
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Parsers"
        },
        "actions": {
          "anyOf": [
            {
              "items": {
                "additionalProperties": true,
                "type": "object"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Actions"
        },
        "location": {
          "anyOf": [
            {
              "$ref": "#/$defs/Location"
            },
            {
              "type": "null"
            }
          ]
        },
        "removeBase64Images": {
          "type": "boolean",
          "title": "Removebase64Images",
          "default": true
        },
        "blockAds": {
          "type": "boolean",
          "title": "Blockads",
          "default": true
        },
        "proxy": {
          "anyOf": [
            {
              "type": "string",
              "enum": [
                "basic",
                "enhanced",
                "auto",
                "stealth"
              ]
            },
            {
              "type": "null"
            }
          ],
          "title": "Proxy",
          "default": "auto"
        },
        "storeInCache": {
          "type": "boolean",
          "title": "Storeincache",
          "default": true
        },
        "lockdown": {
          "type": "boolean",
          "title": "Lockdown",
          "default": false
        },
        "redactPII": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "additionalProperties": true,
              "type": "object"
            }
          ],
          "title": "Redactpii",
          "default": false
        },
        "zeroDataRetention": {
          "type": "boolean",
          "title": "Zerodataretention",
          "default": false
        }
      },
      "additionalProperties": true,
      "type": "object",
      "title": "ScrapeOptions"
    },
    "Webhook": {
      "properties": {
        "url": {
          "type": "string",
          "title": "Url"
        },
        "headers": {
          "anyOf": [
            {
              "additionalProperties": {
                "type": "string"
              },
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "title": "Headers"
        },
        "metadata": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "title": "Metadata"
        },
        "events": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "title": "Events"
        }
      },
      "additionalProperties": true,
      "type": "object",
      "required": [
        "url"
      ],
      "title": "Webhook"
    }
  }
}