{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://artifacts.apis.io/roark/json-schema/roark-get-simulation-plan-job-response-schema.json", "title": "GetSimulationPlanJobResponse", "description": "Simulation run plan job with all associated simulation jobs", "x-generated": "2026-10-29", "x-method": "derived", "x-generator": "derive-json-schema.py", "x-source": "openapi/roark-openapi.json#/components/schemas/GetSimulationPlanJobResponse", "type": "object", "properties": { "simulationRunPlanJobId": { "type": "string", "format": "uuid", "description": "ID of the simulation run plan job" }, "simulationRunPlanId": { "type": "string", "format": "uuid", "description": "ID of the simulation run plan" }, "status": { "type": "string", "enum": [ "PENDING", "QUEUED", "CREATING_SNAPSHOTS", "CREATING_SIMULATIONS", "PREPARING_CAPACITY", "RUNNING_SIMULATIONS", "COMPLETED", "FAILED", "TIMED_OUT", "CANCELLED", "CANCELLING", "ENDING_SIMULATIONS" ], "description": "Job status" }, "createdAt": { "type": "string", "description": "When the job was created" }, "startedAt": { "type": [ "string", "null" ], "description": "When the job started" }, "endedAt": { "type": [ "string", "null" ], "description": "When the job ended" }, "simulationJobs": { "type": "array", "items": { "type": "object", "properties": { "simulationJobId": { "type": "string", "format": "uuid", "description": "Simulation job ID" }, "status": { "type": "string", "enum": [ "PENDING", "QUEUED", "PROCESSING", "COMPLETED", "FAILED", "TIMED_OUT", "CANCELLED", "CANCELLING", "RETRY_SCHEDULED" ], "description": "Job status. `RETRY_SCHEDULED` is a retry of a simulation your agent never spoke on, waiting out the plan’s `noResponseRetryBackoffSeconds` before it dials." }, "processingStatus": { "type": "string", "enum": [ "PENDING", "RESERVING_CAPACITY", "CONNECTING", "WAITING_FOR_OUTBOUND_CALL", "SIMULATING", "ENDING", "ANALYZING", "WAITING_FOR_LIVE_CONVERSATION", "EVALUATING", "COLLECTING_METRICS", "COMPLETED" ], "description": "Processing status. PENDING until the job starts connecting." }, "callId": { "type": [ "string", "null" ], "format": "uuid", "description": "ID of the call created for this simulation job. Null if the call has not been created yet." }, "roarkPhoneNumber": { "type": [ "string", "null" ], "description": "Phone number provisioned by Roark for this simulation job in E.164 format. Null if the simulation job is queued and has not been assigned a phone number yet." }, "persona": { "type": "object", "properties": { "id": { "type": "string", "format": "uuid", "description": "Unique identifier of the persona" }, "name": { "type": "string", "description": "The name the agent will identify as during conversations" }, "displayName": { "type": [ "string", "null" ], "description": "Label shown in place of the name across the dashboard (e.g. a short descriptor like \"Irate Escalator\"). The persona still identifies as `name` on calls. Omit or set null to display the name itself." }, "description": { "type": [ "string", "null" ], "description": "Human-readable description of the persona" }, "language": { "type": "string", "enum": [ "EN", "ES", "DE", "HI", "FR", "NL", "AR", "EL", "IT", "ID", "TH", "JA", "TL", "MS", "ZH", "TR", "PT", "HE" ], "description": "Primary language ISO 639-1 code for the persona" }, "secondaryLanguage": { "type": [ "string", "null" ], "enum": [ "EN" ], "description": "Secondary language ISO 639-1 code for code-switching (e.g., Hinglish, Spanglish)" }, "understoodLanguages": { "type": "array", "items": { "type": "string", "enum": [ "EN", "ES", "DE", "HI", "FR", "NL", "AR", "EL", "IT", "ID", "TH", "JA", "TL", "MS", "ZH", "TR", "PT", "HE" ] }, "minItems": 1, "description": "Languages the persona can understand. Multilingual combinations are limited by multilingual speech recognition support." }, "accent": { "type": "string", "enum": [ "US", "US_X_SOUTH", "GB", "ES", "DE", "IN", "FR", "NL", "SA", "GR", "AU", "IT", "ID", "TH", "JP", "NZ", "PH", "SG", "MY", "HK", "TR", "PT", "IL" ], "description": "Accent of the persona, defined using ISO 3166-1 alpha-2 country codes with optional variants" }, "age": { "type": "string", "enum": [ "CHILD", "TEENAGER", "ADULT", "ELDERLY" ], "default": "ADULT", "description": "How old the caller sounds and behaves. Only ages the persona's accent has a voice for are accepted; defaults to ADULT, which every accent supports." }, "gender": { "type": "string", "enum": [ "MALE", "FEMALE" ], "description": "Gender of the persona" }, "backgroundNoise": { "type": "string", "enum": [ "NONE", "AIRPORT", "CHILDREN_PLAYING", "CITY", "COFFEE_SHOP", "CONSTRUCTION", "CRYING_BABY", "DRIVING", "LIBRARY", "OFFICE", "THUNDERSTORM", "TRAIN" ], "default": "NONE", "description": "Background noise setting" }, "speechPace": { "type": "string", "enum": [ "SUPER_SLOW", "SLOW", "NORMAL", "FAST", "SUPER_FAST" ], "default": "NORMAL", "description": "Speech pace of the persona" }, "speechClarity": { "type": "string", "enum": [ "CLEAR", "VAGUE", "RAMBLING" ], "default": "CLEAR", "description": "Speech clarity of the persona" }, "hasDisfluencies": { "type": "boolean", "default": false, "description": "Whether the persona uses filler words like \"um\" and \"uh\"" }, "baseEmotion": { "type": "string", "enum": [ "NEUTRAL", "CHEERFUL", "CONFUSED", "FRUSTRATED", "SKEPTICAL", "RUSHED", "DISTRACTED", "ANGRY", "ANXIOUS", "SAD" ], "default": "NEUTRAL", "description": "Base emotional state of the persona" }, "intentClarity": { "type": "string", "enum": [ "CLEAR", "INDIRECT", "VAGUE" ], "default": "CLEAR", "description": "How clearly the persona expresses their intentions" }, "confirmationStyle": { "type": "string", "enum": [ "EXPLICIT", "VAGUE" ], "default": "EXPLICIT", "description": "How the persona confirms information" }, "memoryReliability": { "type": "string", "enum": [ "HIGH", "LOW" ], "default": "HIGH", "description": "How reliable the persona's memory is" }, "responseTiming": { "type": "string", "enum": [ "RELAXED", "NORMAL", "QUICK", "BARGE_IN" ], "default": "NORMAL", "description": "Deprecated and inert: it no longer affects the call. It set how long the persona waited once the agent stopped talking, and measured across production simulations it moved the reply gap by less than the noise floor, because model and speech latency dominate it. Every persona now uses one voice-activity profile. Use `interruption` for a caller who talks over the agent. Still accepted and stored so existing clients keep working. BARGE_IN is stored as `responseTiming: QUICK` with `interruption: OCCASIONAL`." }, "interruption": { "type": "string", "enum": [ "OFF", "BACKCHANNEL", "OCCASIONAL", "HEAVY" ], "default": "OFF", "description": "How much the persona talks over the agent while it is still speaking. OFF waits its turn. BACKCHANNEL makes listening noises (\"mm-hm\") over the agent without taking the floor, which tests whether the agent wrongly stops for them. OCCASIONAL adds cutting in on some long agent turns, HEAVY on most of them. Timing is randomised per turn, so two runs of the same persona do not interrupt at identical moments." }, "backstoryPrompt": { "type": [ "string", "null" ], "description": "Background story and behavioral patterns for the persona" }, "idleMessages": { "type": [ "array", "null" ], "items": { "type": "string" }, "description": "Messages the persona will say when the agent goes silent during a call. null = \"Automatic\": language-appropriate defaults are used at call time." }, "idleTimeoutSeconds": { "type": "integer", "minimum": 5, "maximum": 60, "default": 10, "description": "Seconds of silence before the persona sends an idle message" }, "idleMessageMaxSpokenCount": { "type": "integer", "minimum": 1, "maximum": 10, "default": 3, "description": "Maximum number of idle messages the persona will send before giving up" }, "idleMessageResetCountOnUserSpeechEnabled": { "type": "boolean", "default": true, "description": "Whether the idle message counter resets when the agent speaks" }, "properties": { "type": "object", "additionalProperties": {}, "default": {}, "description": "Additional custom properties about the persona" }, "phoneNumber": { "type": "string", "description": "The E.164 number every call with this persona uses, when Roark has pinned one for your project. Present only when set; read-only." }, "createdAt": { "type": "string", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "description": "Last update timestamp" } }, "required": [ "id", "name", "language", "understoodLanguages", "accent", "age", "gender", "backgroundNoise", "speechPace", "speechClarity", "hasDisfluencies", "baseEmotion", "intentClarity", "confirmationStyle", "memoryReliability", "responseTiming", "interruption", "idleMessages", "idleTimeoutSeconds", "idleMessageMaxSpokenCount", "idleMessageResetCountOnUserSpeechEnabled", "properties", "createdAt", "updatedAt" ] }, "backgroundNoise": { "$ref": "#/$defs/SimulationJobBackgroundNoise" }, "scenario": { "$ref": "#/$defs/ScenarioResponse" }, "agentEndpoint": { "$ref": "#/$defs/AgentEndpointResponse" }, "invalidation": { "oneOf": [ { "$ref": "#/$defs/SimulationJobInvalidation" }, { "type": "null" } ], "description": "Present when the result does not count (a strict flow went off script and its policy invalidated the run); null otherwise." }, "retryOfSimulationJobId": { "type": [ "string", "null" ], "format": "uuid", "description": "The simulation this one retries, because your agent never spoke on it. Null on a test case’s first attempt." }, "attemptNumber": { "type": "integer", "exclusiveMinimum": 0, "description": "This simulation’s place among its test case’s attempts: 1 for the first, 2 for the first retry. Above 1 only when the plan retries simulations your agent never spoke on." }, "scheduledAt": { "type": [ "string", "null" ], "description": "When a `RETRY_SCHEDULED` retry dials, ISO 8601. Null on a first attempt." }, "createdAt": { "type": "string", "description": "When the simulation job was created" }, "startedAt": { "type": [ "string", "null" ], "description": "When the simulation job started" }, "completedAt": { "type": [ "string", "null" ], "description": "When the simulation job completed" } }, "required": [ "simulationJobId", "status", "processingStatus", "persona", "backgroundNoise", "scenario", "agentEndpoint", "invalidation", "retryOfSimulationJobId", "attemptNumber", "scheduledAt", "createdAt" ] }, "description": "List of simulation jobs (calls) in this run plan job, every attempt included: a simulation your agent never spoke on stays listed after a retry replaces it." }, "attemptSummary": { "$ref": "#/$defs/SimulationRunPlanJobAttemptSummary" }, "pendingRetries": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobPendingRetry" }, "description": "Retries waiting out their backoff, soonest first. While this is not empty the run is waiting, not stuck: it settles only once every test case has a final attempt. Empty when nothing is scheduled." }, "verdict": { "oneOf": [ { "$ref": "#/$defs/SimulationRunPlanJobVerdict" }, { "type": "null" } ], "description": "Pass/fail verdict for this run. Null when the plan carried no pass/fail metric to judge, so an absent verdict means \"nothing to judge\", never a pass." }, "sweepAttribution": { "oneOf": [ { "$ref": "#/$defs/SimulationRunPlanJobSweepAttribution" }, { "type": "null" } ], "description": "For a run that swept a property: which check failures the property caused, which were not specific to it, and which values had too few simulations to compare. Null for a run that swept nothing, and until the run has ended." } }, "required": [ "simulationRunPlanJobId", "simulationRunPlanId", "status", "createdAt", "simulationJobs", "attemptSummary", "pendingRetries" ], "$defs": { "AgentEndpointResponse": { "type": "object", "properties": { "id": { "type": "string", "format": "uuid", "description": "Agent endpoint ID" }, "name": { "type": "string", "description": "Agent endpoint name" }, "phoneNumber": { "type": [ "string", "null" ], "description": "Agent endpoint phone number" }, "type": { "type": "string", "enum": [ "PHONE", "WEBSOCKET", "LIVEKIT", "SMALL_WEBRTC", "ELEVENLABS_WS", "KORE", "GOOGLE_CES", "DAILY" ], "description": "Agent endpoint type" } }, "required": [ "id", "name", "phoneNumber", "type" ], "description": "Agent endpoint used in a simulation" }, "ScenarioResponse": { "type": "object", "properties": { "id": { "type": "string", "format": "uuid", "description": "Scenario ID" }, "description": { "type": [ "string", "null" ], "description": "Scenario description" } }, "required": [ "id" ], "description": "Scenario used in a simulation" }, "SimulationJobBackgroundNoise": { "type": "object", "properties": { "backgroundNoise": { "type": "string", "enum": [ "NONE", "AIRPORT", "CHILDREN_PLAYING", "CITY", "COFFEE_SHOP", "CONSTRUCTION", "CRYING_BABY", "DRIVING", "LIBRARY", "OFFICE", "THUNDERSTORM", "TRAIN" ], "description": "The noise bed the simulated caller was placed with. NONE when the call ran in silence." }, "backgroundNoiseVolume": { "type": "number", "minimum": 0, "maximum": 1, "description": "Linear gain (0..1) the bed played at. 1 is as loud as the caller’s voice." }, "source": { "type": "string", "enum": [ "ENVIRONMENT", "PERSONA" ], "description": "`ENVIRONMENT` when the flow’s environment decided the bed (a sweep arm, or an environment with noise), `PERSONA` when the persona’s own setting did." } }, "required": [ "backgroundNoise", "backgroundNoiseVolume", "source" ], "description": "The background noise the call actually ran with. `persona.backgroundNoise` is only what the persona was set to; a flow’s environment overrides it, and on a background-noise sweep every arm’s persona says NONE while the arms differ here. Filter by arm on this field." }, "SimulationJobInvalidation": { "type": "object", "properties": { "reason": { "type": "string", "enum": [ "SCRIPT_DIVERGED", "REQUIRED_STEP_NOT_REACHED", "REQUIRED_STAGE_INCOMPLETE", "CALLER_NEVER_TOOK_OVER", "AGENT_NEVER_SPOKE" ], "description": "Why the result does not count. `SCRIPT_DIVERGED`: a strict flow went off script at a step whose off-script policy is HANG_UP_INVALIDATE. `REQUIRED_STAGE_INCOMPLETE`: the call ended before it got through a stage its flow requires. `CALLER_NEVER_TOOK_OVER`: the call opened on the persona of a preceding flow, which never handed the phone to the persona under test, so none of that persona's properties were exercised. `AGENT_NEVER_SPOKE`: your agent answered and never said a word (it hung up within seconds, or the line stayed silent until our caller gave up), so there was nothing to grade." }, "detail": { "type": [ "string", "null" ], "description": "One sentence: where the script was left and what your agent did instead." }, "invalidatedAt": { "type": "string", "description": "When the run was invalidated." } }, "required": [ "reason", "invalidatedAt" ], "description": "Present when the run was invalidated: the call ended before a step its flow requires for a valid run, or a strict flow went off script at a step whose policy invalidates the run. It keeps its transcript and recording, but nothing scored it and it is excluded from every run total." }, "SimulationRunPlanJobAttemptSummary": { "type": "object", "properties": { "testCaseCount": { "type": "integer", "minimum": 0, "description": "Test cases in the run. Each one has a single final attempt that its result is read from." }, "attemptCount": { "type": "integer", "minimum": 0, "description": "Every simulation placed, retries included. Each is a separate, billed call." }, "silentAttemptCount": { "type": "integer", "minimum": 0, "description": "Simulations your agent never spoke on, including the ones a retry replaced." }, "retryCount": { "type": "integer", "minimum": 0, "description": "Simulations placed as a retry of one your agent never spoke on." }, "stillSilentTestCaseCount": { "type": "integer", "minimum": 0, "description": "Test cases whose final attempt your agent still never spoke on. This is what the `AGENT_NEVER_SPOKE` verdict failure counts." } }, "required": [ "testCaseCount", "attemptCount", "silentAttemptCount", "retryCount", "stillSilentTestCaseCount" ], "description": "The run’s simulations against its test cases. Differs from a plain job count only when the plan retries simulations your agent never spoke on." }, "SimulationRunPlanJobCheck": { "type": "object", "properties": { "metricDefinitionId": { "type": "string", "format": "uuid" }, "metricName": { "type": [ "string", "null" ] }, "passRate": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "This check's own pass rate, 0-100: the share of sims it evaluated that passed it. Null when it evaluated nothing." }, "minPassRate": { "type": "number", "minimum": 0, "maximum": 100, "description": "THE BAR it was judged against: the share of the run's simulations it had to pass." }, "inherited": { "type": "boolean", "description": "Whether `minPassRate` is the 80% default (`true`) or a minimum this metric set for itself." }, "passedSims": { "type": "integer", "minimum": 0 }, "evaluatedSims": { "type": "integer", "minimum": 0 }, "passed": { "type": "boolean", "description": "`passRate >= minPassRate`." } }, "required": [ "metricDefinitionId", "passRate", "minPassRate", "inherited", "passedSims", "evaluatedSims", "passed" ], "description": "How one check did. Present for every check the run was judged on, passing or not.\n\nA check is a metric that yields a pass/fail: a threshold (`Silence Duration <= 2s`) or a\nyes/no metric you authored. Provider readings such as `Comprehension Failure` are\nobservations, not checks: their `true` is whatever the underlying field happens to mean, so\nthey carry no passing side and never appear here. Put a threshold on one to judge it." }, "SimulationRunPlanJobPendingRetry": { "type": "object", "properties": { "simulationJobId": { "type": "string", "format": "uuid", "description": "The scheduled retry." }, "attemptNumber": { "type": "integer", "exclusiveMinimum": 0, "description": "Its place among its test case’s attempts, 2 for the first retry." }, "maxAttempts": { "type": "integer", "exclusiveMinimum": 0, "description": "The most attempts a test case can get on this run: the plan’s `maxNoResponseRetries` plus 1." }, "scheduledAt": { "type": [ "string", "null" ], "description": "When the retry dials, ISO 8601. It may wait longer behind the run’s concurrency limit." } }, "required": [ "simulationJobId", "attemptNumber", "maxAttempts", "scheduledAt" ], "description": "A retry of a simulation your agent never spoke on, waiting out its backoff before it dials." }, "SimulationRunPlanJobSweepArm": { "type": "object", "properties": { "key": { "type": "string", "description": "The arm's identity: the override signature its calls ran with, e.g. `BACKGROUND_NOISE=DRIVING;BACKGROUND_NOISE_VOLUME=0.7`. Join on this rather than on `value`: a sweep can run one value as several arms (Driving at 70% and at 100%), and those share a value." }, "value": { "type": "string", "description": "The stored value of the swept property, e.g. `US`." }, "label": { "type": "string", "description": "The value in words, with what else the arm pinned, e.g. `American` or `Driving (70% noise)`." }, "isBaseline": { "type": "boolean", "description": "Whether this is the baseline the plan named." }, "attempted": { "type": "integer", "minimum": 0, "description": "Simulations run at this value. Simulations that failed on the Roark platform are left out, since they say nothing about your agent." }, "counted": { "type": "integer", "minimum": 0, "description": "Simulations that actually tested the property: not invalidated, and graded by at least one check. Only these are used in the comparison." }, "silentAttempts": { "type": "integer", "minimum": 0, "description": "Of `attempted`, the simulations your agent never spoke on, whether or not a retry followed. They are invalidated, so they are never in `counted`." }, "retries": { "type": "integer", "minimum": 0, "description": "Of `attempted`, the retries of simulations your agent never spoke on. Zero unless the plan retries silent simulations." }, "testable": { "type": "boolean", "description": "Whether this value had at least `minArmCalls` counted simulations. A value below that is \"insufficient data\": it is reported with its numbers but never compared or ranked, because a rate from one or two calls cannot be told apart from luck." }, "score": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "The mean of each check's pass rate at this value, 0-100, the same rule as the run's `score`. Descriptive only: a lower score alone never makes a value worse. Null when nothing counted." } }, "required": [ "key", "value", "label", "isBaseline", "attempted", "counted", "silentAttempts", "retries", "testable", "score" ], "description": "One value of the swept property." }, "SimulationRunPlanJobSweepArmCheck": { "type": "object", "properties": { "key": { "type": "string", "description": "The arm's identity: the override signature its calls ran with, e.g. `BACKGROUND_NOISE=DRIVING;BACKGROUND_NOISE_VOLUME=0.7`. Join on this rather than on `value`: a sweep can run one value as several arms (Driving at 70% and at 100%), and those share a value." }, "value": { "type": "string" }, "label": { "type": "string" }, "failed": { "type": "integer", "minimum": 0 }, "evaluated": { "type": "integer", "minimum": 0 }, "failureRate": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "Share of the counted simulations at this value that failed the check, 0-100." }, "restFailureRate": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "The same share across every other value combined, 0-100. This is what the value is compared with." }, "adjustedPValue": { "type": "number", "minimum": 0, "maximum": 1, "description": "How likely a difference at least this large would be by chance alone, after correcting for every comparison in the run (Benjamini-Hochberg). The value is called worse only when this is at most `falseDiscoveryRate`." } }, "required": [ "key", "value", "label", "failed", "evaluated", "failureRate", "restFailureRate", "adjustedPValue" ], "description": "A value that failed a check significantly more often than the rest of the run." }, "SimulationRunPlanJobSweepAttribution": { "type": "object", "properties": { "values": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobSweepArm" }, "description": "Every value of the swept property, baseline first." }, "checks": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobSweepCheck" }, "description": "Every check the run was graded on: `PROPERTY_ATTRIBUTABLE` first, then `FOUND_UNATTRIBUTED`, then `WITHIN_NOISE`." }, "testableValueCount": { "type": "integer", "minimum": 0, "description": "How many values had enough counted simulations to be compared." }, "testedComparisonCount": { "type": "integer", "minimum": 0, "description": "How many value and check pairs were actually tested. A value can have enough simulations and still go untested (no other value graded that check, or too few verdicts on it). When this is 0 no comparison ran, so an empty `worseValues` everywhere means nothing was tested, not that no value did worse." }, "minimumDetectableGap": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "Roughly how many percentage points more often a value would need to fail than the rest of the run to be flagged at this sample size. Optimistic: it uses simulation counts rather than the verdicts on each check and ignores the correction, so checks graded on fewer simulations need larger gaps. Large when few simulations ran per value: finding no significant difference then means the run could not see one, not that none exists." }, "caveats": { "type": "array", "items": { "type": "string" }, "description": "Plain sentences on what this run could not see, ready to show to a reader." }, "neverSpokeValues": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobSweepNeverSpokeValue" }, "description": "The values your agent never spoke on significantly more often than the rest of the run, most significant first, tested the same way as `worseValues`. Empty on a run with no silent simulations." }, "minArmCalls": { "type": "integer", "exclusiveMinimum": 0, "description": "Counted simulations a value needs before it is compared." }, "falseDiscoveryRate": { "type": "number", "minimum": 0, "maximum": 1, "description": "The Benjamini-Hochberg false discovery rate the comparisons are held to." }, "foundUnattributedFailureRate": { "type": "number", "minimum": 0, "maximum": 100, "description": "The overall failure rate, 0-100, at which a check with no standout value is reported as `FOUND_UNATTRIBUTED`." } }, "required": [ "values", "checks", "testableValueCount", "testedComparisonCount", "minimumDetectableGap", "caveats", "neverSpokeValues", "minArmCalls", "falseDiscoveryRate", "foundUnattributedFailureRate" ], "description": "For a run that swept a property (accent, background noise, speech pace and so on): which check failures the property caused.\n\nEach value is compared with every other value combined using a one-sided Fisher exact test, and every comparison in the run is corrected together with Benjamini-Hochberg. A value is only called worse when the difference is statistically significant, so a value that happened to fail a few more simulations by chance is not reported as a problem. Invalidated simulations and simulations that failed on the Roark platform are left out." }, "SimulationRunPlanJobSweepCheck": { "type": "object", "properties": { "metricDefinitionId": { "type": "string", "format": "uuid" }, "metricName": { "type": "string" }, "attribution": { "type": "string", "enum": [ "PROPERTY_ATTRIBUTABLE", "FOUND_UNATTRIBUTED", "WITHIN_NOISE" ], "description": "How this check relates to the swept property:\n\n- `PROPERTY_ATTRIBUTABLE`: at least one value failed it significantly more often than every other value combined. `worseValues` names them. This is the only case in which a value is called worse.\n- `FOUND_UNATTRIBUTED`: no value stands out, but the check failed on at least `foundUnattributedFailureRate`% of counted simulations overall. A real issue with the agent on which no value stood out, so the run cannot tie it to the property. It does not show the property had no effect: a small run may be unable to see one.\n- `WITHIN_NOISE`: neither. Any differences between values are within what chance produces." }, "failed": { "type": "integer", "minimum": 0 }, "evaluated": { "type": "integer", "minimum": 0 }, "failureRate": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "Share of counted simulations across every value that failed the check, 0-100." }, "worseValues": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobSweepArmCheck" }, "description": "The values significantly worse than the rest, most significant first. Empty unless `attribution` is `PROPERTY_ATTRIBUTABLE`." } }, "required": [ "metricDefinitionId", "metricName", "attribution", "failed", "evaluated", "failureRate", "worseValues" ], "description": "How one check's failures relate to the swept property." }, "SimulationRunPlanJobSweepNeverSpokeValue": { "type": "object", "properties": { "key": { "type": "string", "description": "The arm's identity: the override signature its calls ran with, e.g. `BACKGROUND_NOISE=DRIVING;BACKGROUND_NOISE_VOLUME=0.7`. Join on this rather than on `value`: a sweep can run one value as several arms (Driving at 70% and at 100%), and those share a value." }, "value": { "type": "string" }, "label": { "type": "string" }, "silentAttempts": { "type": "integer", "minimum": 0, "description": "Simulations at this value your agent never spoke on, retried or not." }, "attempted": { "type": "integer", "minimum": 0, "description": "Simulations run at this value." }, "retries": { "type": "integer", "minimum": 0, "description": "Retries placed at this value because your agent never spoke." }, "restSilentAttempts": { "type": "integer", "minimum": 0, "description": "Simulations your agent never spoke on across every other value combined." }, "restAttempted": { "type": "integer", "minimum": 0, "description": "Simulations run across every other value combined." }, "adjustedPValue": { "type": "number", "minimum": 0, "maximum": 1, "description": "How likely a difference at least this large would be by chance alone, after correcting across the values (Benjamini-Hochberg)." } }, "required": [ "key", "value", "label", "silentAttempts", "attempted", "retries", "restSilentAttempts", "restAttempted", "adjustedPValue" ], "description": "A value your agent never spoke on significantly more often than the rest of the run." }, "SimulationRunPlanJobVerdict": { "type": "object", "properties": { "passed": { "type": "boolean", "description": "Whether every check cleared its own `minPassRate` (and the run completed with full coverage).\nUse this as the CI exit status.\n\nNever derived from `score`: a run can score 95 and still fail, or score 40 and still pass." }, "score": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "The run's headline quality number, 0-100: the mean of each check's own pass rate.\n\nEvery check weighs the same, however many simulations it evaluated, which is the same way\n`passed` treats them. It is the number the Roark dashboard shows for this run.\n\nREPORTING ONLY, for dashboards and trend lines. Nothing is judged against it. Null when\nnothing was evaluated." }, "checks": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobCheck" }, "description": "Every check the run was judged on, with its rate and the minimum it had to reach.\n\nThresholds and authored yes/no metrics only. See `SimulationRunPlanJobCheck` for why a\nprovider reading is not one." }, "failures": { "type": "array", "items": { "$ref": "#/$defs/SimulationRunPlanJobVerdictFailure" }, "description": "Every criterion the run missed. Empty when it passed." } }, "required": [ "passed", "score", "checks", "failures" ], "description": "Pass/fail verdict for the run, judged against the success criteria pinned on the run plan when the run started." }, "SimulationRunPlanJobVerdictFailure": { "oneOf": [ { "type": "object", "properties": { "type": { "type": "string", "const": "RUN_NOT_COMPLETED" }, "status": { "type": "string", "enum": [ "PENDING", "QUEUED", "CREATING_SNAPSHOTS", "CREATING_SIMULATIONS", "PREPARING_CAPACITY", "RUNNING_SIMULATIONS", "COMPLETED", "FAILED", "TIMED_OUT", "CANCELLED", "CANCELLING", "ENDING_SIMULATIONS" ], "description": "The status the run actually ended in." } }, "required": [ "type", "status" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "INCOMPLETE_COVERAGE" }, "evaluatedCalls": { "type": "integer", "minimum": 0 }, "expectedCalls": { "type": "integer", "minimum": 0 } }, "required": [ "type", "evaluatedCalls", "expectedCalls" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "AGENT_NEVER_SPOKE" }, "neverSpokeCalls": { "type": "integer", "minimum": 0, "description": "Test cases your agent never spoke on, judged on each test case’s last attempt: a silent simulation a retry later reached your agent on does not count here." }, "totalCalls": { "type": "integer", "minimum": 0, "description": "Every test case of the run whose last attempt reached your agent." }, "maxShare": { "type": "number", "minimum": 0, "maximum": 100, "description": "The largest share of simulations, 0-100, your agent may never speak on while the run can pass." }, "agentName": { "type": [ "string", "null" ], "description": "The agent as the run tested it. Null when the simulations tested more than one agent." }, "silentAttempts": { "type": "integer", "minimum": 0, "description": "Every simulation your agent never spoke on, including the ones a retry replaced. Above `neverSpokeCalls` only when the plan retries silent simulations (`maxNoResponseRetries`)." } }, "required": [ "type", "neverSpokeCalls", "totalCalls", "maxShare" ], "description": "Your agent never spoke on more than `maxShare` percent of the run. Those simulations are left out of every check, so the run fails on them whatever the checks say." }, { "type": "object", "properties": { "type": { "type": "string", "const": "METRIC_NOT_EVALUATED" }, "metricDefinitionId": { "type": "string", "format": "uuid" }, "metricName": { "type": [ "string", "null" ], "description": "The check’s name, for rendering the failure." } }, "required": [ "type", "metricDefinitionId" ] }, { "type": "object", "properties": { "type": { "type": "string", "const": "METRIC_BELOW_MIN_PASS_RATE" }, "metricDefinitionId": { "type": "string", "format": "uuid" }, "metricName": { "type": [ "string", "null" ], "description": "The check’s name, for rendering the failure." }, "passRate": { "type": "number", "minimum": 0, "maximum": 100 }, "minPassRate": { "type": "number", "minimum": 0, "maximum": 100 }, "inherited": { "type": "boolean", "description": "Whether the missed minimum was the 80% default (`true`) or this metric's own." } }, "required": [ "type", "metricDefinitionId", "passRate", "minPassRate", "inherited" ] } ], "description": "One reason the run failed. Every reason is a hard fail: the run passes only when this list is empty." } } }