{
  "opencollection": "1.0.0",
  "info": {
    "name": "Omni AI AI Eval API",
    "version": "1.0.0"
  },
  "request": {
    "auth": {
      "type": "bearer",
      "token": "{{bearerToken}}"
    }
  },
  "items": [
    {
      "info": {
        "name": "AI Eval",
        "type": "folder"
      },
      "items": [
        {
          "info": {
            "name": "List eval prompt sets",
            "type": "http"
          },
          "http": {
            "method": "GET",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/prompt-sets",
            "params": [
              {
                "name": "archived",
                "value": "",
                "type": "query",
                "description": "When `true`, returns archived prompt sets instead of active ones. Defaults to `false`."
              },
              {
                "name": "model_ids",
                "value": "",
                "type": "query",
                "description": "Optional list of model IDs to filter prompt sets by. When omitted, returns prompt sets for every model the caller can access. Supply multiple times to filter by more than one model (e.g., `?model_ids=A&model_ids=B`)."
              }
            ]
          },
          "docs": "List eval prompt sets, sorted alphabetically by name. When `model_ids` is omitted, returns prompt sets for every shared model the caller can access. Requires at least the Querier role on each requested model."
        },
        {
          "info": {
            "name": "Create an eval prompt set",
            "type": "http"
          },
          "http": {
            "method": "POST",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/prompt-sets",
            "body": {
              "type": "json",
              "data": "{}"
            }
          },
          "docs": "Create a new eval prompt set bound to a shared model. Initial prompts can be supplied; additional prompts can be added later via PATCH."
        },
        {
          "info": {
            "name": "Get an eval prompt set",
            "type": "http"
          },
          "http": {
            "method": "GET",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/prompt-sets/:promptSetId",
            "params": [
              {
                "name": "promptSetId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval prompt set."
              }
            ]
          },
          "docs": "Get a single prompt set with all of its prompts."
        },
        {
          "info": {
            "name": "Update an eval prompt set",
            "type": "http"
          },
          "http": {
            "method": "PATCH",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/prompt-sets/:promptSetId",
            "params": [
              {
                "name": "promptSetId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval prompt set."
              }
            ],
            "body": {
              "type": "json",
              "data": "{}"
            }
          },
          "docs": "Update a prompt set's name, description, and/or prompts. When `prompts` is supplied, it fully replaces the existing list — existing prompts omitted from the list are deleted, entries without an `id` are created, and entries with a matching `id` are updated in place."
        },
        {
          "info": {
            "name": "Archive an eval prompt set",
            "type": "http"
          },
          "http": {
            "method": "DELETE",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/prompt-sets/:promptSetId",
            "params": [
              {
                "name": "promptSetId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval prompt set."
              }
            ]
          },
          "docs": "Archive (soft-delete) a prompt set. Omni also attempts to cancel every in-flight agentic job associated with the set; the returned `cancelled_job_count` reports how many were cancelled. Cancellation is best-effort — the archive is committed first, so if a run-cancellation later fails the endpoint returns 500 even though the set is already archived. The call is idempotent: retrying drains any remaining runs."
        },
        {
          "info": {
            "name": "Restore an archived eval prompt set",
            "type": "http"
          },
          "http": {
            "method": "POST",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/prompt-sets/:promptSetId/unarchive",
            "params": [
              {
                "name": "promptSetId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval prompt set."
              }
            ]
          },
          "docs": "Restore an archived prompt set."
        },
        {
          "info": {
            "name": "List eval runs",
            "type": "http"
          },
          "http": {
            "method": "GET",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/runs",
            "params": [
              {
                "name": "archived",
                "value": "",
                "type": "query",
                "description": "When `true`, returns archived runs instead of active ones. Defaults to `false`."
              },
              {
                "name": "prompt_set_id",
                "value": "",
                "type": "query",
                "description": "Required — the prompt set whose runs should be listed."
              }
            ]
          },
          "docs": "List runs for a prompt set, newest first, filtered to runs whose model the caller can access. The `prompt_set_id` query parameter is required."
        },
        {
          "info": {
            "name": "Start an eval run",
            "type": "http"
          },
          "http": {
            "method": "POST",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/runs",
            "body": {
              "type": "json",
              "data": "{}"
            }
          },
          "docs": "Create and start a new run against an existing prompt set. The run enqueues one agentic job per prompt and begins executing immediately. Returns the newly created run with its initial per-prompt result rows."
        },
        {
          "info": {
            "name": "Get an eval run",
            "type": "http"
          },
          "http": {
            "method": "GET",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/runs/:runId",
            "params": [
              {
                "name": "runId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval run."
              }
            ]
          },
          "docs": "Get an eval run with every per-prompt result row, including the underlying agentic job state and any scoring data."
        },
        {
          "info": {
            "name": "Archive an eval run",
            "type": "http"
          },
          "http": {
            "method": "DELETE",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/runs/:runId",
            "params": [
              {
                "name": "runId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval run."
              }
            ]
          },
          "docs": "Archive (soft-delete) an eval run. Any non-terminal per-prompt agentic jobs are cancelled as part of the archive (best-effort), and a still-RUNNING run is flipped to CANCELLED before archival. The call is idempotent; archiving an already-terminal or already-archived run is a no-op."
        },
        {
          "info": {
            "name": "Cancel an eval run",
            "type": "http"
          },
          "http": {
            "method": "POST",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/runs/:runId/cancel",
            "params": [
              {
                "name": "runId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval run."
              }
            ]
          },
          "docs": "Cancel an in-flight eval run. Marks the run cancelled first so no in-progress per-prompt job can flip it back to COMPLETE, then cancels every non-terminal agentic job associated with the run. The run is also archived as part of the cancel — the response returns the updated run inline (`status: CANCELLED`, `is_archived: true`); use `/unarchive` to surface it in the default `archived=false` list again."
        },
        {
          "info": {
            "name": "Restore an archived eval run",
            "type": "http"
          },
          "http": {
            "method": "POST",
            "url": "https://{instance}.omniapp.co/api/v1/ai/eval/runs/:runId/unarchive",
            "params": [
              {
                "name": "runId",
                "value": "",
                "type": "path",
                "description": "The unique identifier of the eval run."
              }
            ]
          },
          "docs": "Restore an archived eval run."
        }
      ]
    }
  ],
  "bundled": true
}