{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/orq/json-schema/orq-evaluator-response-llm-schema.json",
  "title": "LLM",
  "x-generated": "2026-11-10",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/orq-evals-api-openapi.yml#/components/schemas/EvaluatorResponseLlm",
  "type": "object",
  "properties": {
    "_id": {
      "type": "string"
    },
    "description": {
      "type": "string"
    },
    "created": {
      "type": "string",
      "default": "2026-08-05T15:12:51.947Z"
    },
    "updated": {
      "type": "string",
      "default": "2026-08-05T15:12:51.947Z"
    },
    "updated_by_id": {
      "type": [
        "string",
        "null"
      ]
    },
    "project_id": {
      "type": "string",
      "description": "Unique identifier of the project owning this evaluator."
    },
    "guardrail_config": {
      "anyOf": [
        {
          "type": "object",
          "properties": {
            "enabled": {
              "type": "boolean",
              "default": true
            },
            "alert_on_failure": {
              "type": "boolean",
              "default": false
            },
            "type": {
              "type": "string",
              "enum": [
                "boolean"
              ]
            },
            "value": {
              "type": "boolean"
            }
          },
          "required": [
            "type",
            "value"
          ],
          "title": "Boolean"
        },
        {
          "type": "object",
          "properties": {
            "enabled": {
              "type": "boolean",
              "default": true
            },
            "alert_on_failure": {
              "type": "boolean",
              "default": false
            },
            "type": {
              "type": "string",
              "enum": [
                "categorical"
              ]
            },
            "values": {
              "type": "array",
              "items": {
                "type": "string",
                "minLength": 1
              }
            }
          },
          "required": [
            "type",
            "values"
          ],
          "title": "Categorical"
        },
        {
          "type": "object",
          "properties": {
            "enabled": {
              "type": "boolean",
              "default": true
            },
            "alert_on_failure": {
              "type": "boolean",
              "default": false
            },
            "type": {
              "type": "string",
              "enum": [
                "number"
              ]
            },
            "value": {
              "type": "number"
            },
            "operator": {
              "type": "string",
              "enum": [
                "eq",
                "ne",
                "gt",
                "gte",
                "lt",
                "lte"
              ]
            }
          },
          "required": [
            "type",
            "value",
            "operator"
          ],
          "title": "Number"
        },
        {
          "type": "null"
        }
      ],
      "x-speakeasy-type-override": "any"
    },
    "type": {
      "type": "string",
      "enum": [
        "llm_eval"
      ]
    },
    "output_type": {
      "type": "string",
      "enum": [
        "boolean",
        "categorical",
        "number",
        "string"
      ],
      "description": "The type of output expected from the evaluator"
    },
    "repetitions": {
      "type": [
        "integer",
        "null"
      ],
      "minimum": 1,
      "maximum": 3
    },
    "prompt": {
      "type": "string"
    },
    "categories": {
      "type": [
        "array",
        "null"
      ],
      "items": {
        "type": "string"
      }
    },
    "categorical_labels": {
      "type": [
        "array",
        "null"
      ],
      "items": {
        "type": "object",
        "properties": {
          "value": {
            "type": "string",
            "minLength": 1
          },
          "description": {
            "type": "string"
          }
        },
        "required": [
          "value"
        ]
      }
    },
    "dataset_id": {
      "type": [
        "string",
        "null"
      ]
    },
    "key": {
      "type": "string",
      "minLength": 1,
      "pattern": "^[a-zA-Z0-9]([a-zA-Z0-9_-]*[a-zA-Z0-9])?$"
    },
    "mode": {
      "type": "string",
      "enum": [
        "single",
        "jury"
      ]
    },
    "model": {
      "type": "string"
    },
    "jury": {
      "type": "object",
      "properties": {
        "judges": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "model": {
                "type": "string"
              },
              "retry": {
                "type": "object",
                "properties": {
                  "count": {
                    "type": "integer",
                    "minimum": 1,
                    "maximum": 5,
                    "default": 2
                  },
                  "on_codes": {
                    "type": "array",
                    "items": {
                      "type": "integer",
                      "minimum": 100,
                      "maximum": 599
                    },
                    "minItems": 1,
                    "default": [
                      429,
                      500,
                      502,
                      503,
                      504
                    ]
                  }
                }
              },
              "fallbacks": {
                "type": "array",
                "items": {
                  "type": "object",
                  "properties": {
                    "model": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "model"
                  ]
                },
                "default": []
              }
            },
            "required": [
              "model"
            ]
          },
          "minItems": 2
        },
        "replacement_judges": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "model": {
                "type": "string"
              },
              "retry": {
                "type": "object",
                "properties": {
                  "count": {
                    "type": "integer",
                    "minimum": 1,
                    "maximum": 5,
                    "default": 2
                  },
                  "on_codes": {
                    "type": "array",
                    "items": {
                      "type": "integer",
                      "minimum": 100,
                      "maximum": 599
                    },
                    "minItems": 1,
                    "default": [
                      429,
                      500,
                      502,
                      503,
                      504
                    ]
                  }
                }
              },
              "fallbacks": {
                "type": "array",
                "items": {
                  "type": "object",
                  "properties": {
                    "model": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "model"
                  ]
                },
                "default": []
              }
            },
            "required": [
              "model"
            ]
          },
          "default": []
        },
        "min_successful_judges": {
          "type": "integer",
          "minimum": 2,
          "default": 2
        },
        "tie_value": {
          "type": "string",
          "enum": [
            "Tie"
          ],
          "default": "Tie"
        }
      },
      "required": [
        "judges"
      ]
    }
  },
  "required": [
    "_id",
    "description",
    "type",
    "output_type",
    "prompt",
    "key",
    "mode"
  ]
}