{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://raw.githubusercontent.com/api-evangelist/trajectory/main/json-schema/trajectory-eval-run-schema.json",
  "title": "EvalRun",
  "x-generated": "2026-10-24",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/trajectory-openapi.json#/components/schemas/EvalRun",
  "properties": {
    "task_selection": {
      "anyOf": [
        {
          "$ref": "#/$defs/EvalTaskSelection"
        },
        {
          "type": "null"
        }
      ],
      "description": "Task coverage fixed at launch; null for legacy or diagnostic runs."
    },
    "agent_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Agent Id",
      "description": "Persisted owner; null for unassigned legacy runs."
    },
    "eval_run_id": {
      "type": "string",
      "title": "Eval Run Id",
      "description": "Unique evaluation run identifier."
    },
    "bench_id": {
      "type": "string",
      "title": "Bench Id",
      "description": "Benchmark evaluated by this run."
    },
    "rollout_xid": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Rollout Xid",
      "description": "XID containing this run's evaluation trajectories."
    },
    "display_name": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Display Name",
      "description": "User-facing name for the evaluation run; null for legacy rows."
    },
    "base_model_slug": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Base Model Slug",
      "description": "Base model evaluated by this run."
    },
    "options": {
      "$ref": "#/$defs/RecordedRunOptions"
    },
    "status": {
      "anyOf": [
        {
          "$ref": "#/$defs/EvalRunLifecycleStatus"
        },
        {
          "type": "null"
        }
      ],
      "description": "Current evaluation lifecycle status; null for legacy rows."
    },
    "deployment_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Deployment Id",
      "description": "Hosted deployment snapshotted at launch."
    },
    "parent_checkpoint_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Parent Checkpoint Id",
      "description": "Checkpoint evaluated by this run, when applicable."
    },
    "training_run_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Training Run Id",
      "description": "Training run that produced the evaluated checkpoint."
    },
    "step": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Step",
      "description": "Evaluated checkpoint training step."
    },
    "reward_mean": {
      "anyOf": [
        {
          "type": "number"
        },
        {
          "type": "null"
        }
      ],
      "title": "Reward Mean",
      "description": "Completed mean of effective evaluation rewards; cut-off trajectories can contribute zero without a logged grader reward."
    },
    "total_rollouts": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "title": "Total Rollouts",
      "description": "Rollout total fixed at launch; null for legacy or diagnostic runs."
    },
    "created_at": {
      "type": "string",
      "format": "date-time",
      "title": "Created At",
      "description": "Time when the evaluation run was created."
    },
    "updated_at": {
      "type": "string",
      "format": "date-time",
      "title": "Updated At",
      "description": "Time when the evaluation run was last updated."
    },
    "completed_at": {
      "anyOf": [
        {
          "type": "string",
          "format": "date-time"
        },
        {
          "type": "null"
        }
      ],
      "title": "Completed At",
      "description": "Time when the evaluation reached a terminal status."
    },
    "failure": {
      "anyOf": [
        {
          "$ref": "#/$defs/AsyncFailure"
        },
        {
          "type": "null"
        }
      ],
      "description": "Structured failure when status is failed."
    }
  },
  "type": "object",
  "required": [
    "agent_id",
    "eval_run_id",
    "bench_id",
    "display_name",
    "options",
    "status",
    "created_at",
    "updated_at"
  ],
  "$defs": {
    "AsyncFailure": {
      "properties": {
        "type": {
          "type": "string",
          "title": "Type",
          "description": "Stable category used for exception mapping."
        },
        "code": {
          "type": "string",
          "title": "Code",
          "description": "Stable programmatic reason for the asynchronous failure."
        },
        "message": {
          "type": "string",
          "title": "Message",
          "description": "Human-readable explanation for this failure occurrence."
        },
        "context": {
          "additionalProperties": true,
          "type": "object",
          "title": "Context",
          "description": "Machine-readable values specific to this failure code."
        }
      },
      "type": "object",
      "required": [
        "type",
        "code",
        "message"
      ],
      "title": "AsyncFailure"
    },
    "EvalRunLifecycleStatus": {
      "type": "string",
      "enum": [
        "pending",
        "registered",
        "running",
        "completed",
        "failed",
        "cancelled"
      ],
      "title": "EvalRunLifecycleStatus"
    },
    "EvalTaskSelection": {
      "properties": {
        "split": {
          "$ref": "#/$defs/EvalTaskSplit"
        },
        "available_task_count": {
          "type": "integer",
          "minimum": 0.0,
          "title": "Available Task Count",
          "description": "Tasks in the selected scope at launch."
        },
        "selected_task_count": {
          "type": "integer",
          "minimum": 0.0,
          "title": "Selected Task Count",
          "description": "Distinct tasks selected for execution."
        },
        "max_samples": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Samples",
          "description": "Explicit options.evaluation_max_samples, or null to run every task in the selected scope."
        }
      },
      "type": "object",
      "required": [
        "split",
        "available_task_count",
        "selected_task_count"
      ],
      "title": "EvalTaskSelection"
    },
    "EvalTaskSplit": {
      "type": "string",
      "enum": [
        "test",
        "all"
      ],
      "title": "EvalTaskSplit"
    },
    "RecordedRunOptions": {
      "properties": {
        "intelligent_density_training": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Intelligent Density Training"
        },
        "algorithm": {
          "anyOf": [
            {
              "$ref": "#/$defs/TrainingAlgorithm"
            },
            {
              "type": "null"
            }
          ]
        },
        "train_on_truncation": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Train On Truncation"
        },
        "train_on_error": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Train On Error"
        },
        "train_on_timeout": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Train On Timeout"
        },
        "evaluation_enabled": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Enabled"
        },
        "evaluation_frequency": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Frequency"
        },
        "evaluation_max_samples": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Max Samples"
        },
        "evaluation_samples_per_task": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Samples Per Task"
        },
        "evaluation_max_active_rollouts": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Max Active Rollouts"
        },
        "evaluation_collection_timeout_seconds": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Collection Timeout Seconds"
        },
        "learning_rate": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Learning Rate"
        },
        "lr_warmup_steps": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Lr Warmup Steps"
        },
        "lr_hold_steps": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Lr Hold Steps"
        },
        "min_lr_ratio": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Min Lr Ratio"
        },
        "reasoning_effort": {
          "anyOf": [
            {
              "type": "string",
              "enum": [
                "off",
                "on",
                "minimal",
                "low",
                "medium",
                "high",
                "xhigh",
                "max"
              ]
            },
            {
              "type": "null"
            }
          ],
          "title": "Reasoning Effort"
        },
        "temperature": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Temperature"
        },
        "top_p": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Top P"
        },
        "top_k": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Top K"
        },
        "evaluation_reasoning_effort": {
          "anyOf": [
            {
              "type": "string",
              "enum": [
                "off",
                "on",
                "minimal",
                "low",
                "medium",
                "high",
                "xhigh",
                "max"
              ]
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Reasoning Effort"
        },
        "evaluation_temperature": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Temperature"
        },
        "evaluation_top_p": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Top P"
        },
        "evaluation_top_k": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Evaluation Top K"
        },
        "num_steps": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Num Steps"
        },
        "train_batch_size": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Train Batch Size"
        },
        "group_size": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Group Size"
        },
        "samples_per_instance": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Samples Per Instance"
        },
        "min_samples_per_group": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Min Samples Per Group"
        },
        "max_active_rollouts": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Active Rollouts"
        },
        "max_steps_off_policy": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Steps Off Policy"
        },
        "execution_timeout_seconds": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Execution Timeout Seconds"
        },
        "batch_collection_idle_timeout_seconds": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Batch Collection Idle Timeout Seconds"
        },
        "max_output_tokens_per_step": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Output Tokens Per Step"
        },
        "max_total_tokens_per_trajectory": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Total Tokens Per Trajectory"
        },
        "max_turns_per_trajectory": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Max Turns Per Trajectory"
        }
      },
      "additionalProperties": false,
      "type": "object",
      "title": "RecordedRunOptions",
      "description": "Recorded settings retain values outside the public launch limits."
    },
    "TrainingAlgorithm": {
      "type": "string",
      "enum": [
        "gspo",
        "grpo",
        "reinforce_plus_plus"
      ],
      "title": "TrainingAlgorithm"
    }
  }
}