{ "info": { "_postman_id": "b1660dbe-1152-4fdd-81da-7ce120db83f2", "name": "Runloop Benchmark API", "description": "Run and manage Benchmarks and Benchmark Runs \u2014 the evaluation framework for AI coding agents. Supports SWE-Bench, SWE-smith, and custom benchmark definitions, scenario aggregation, run lifecycle (start/cancel/complete), scoring, and log retrieval.\n\nContact Support:\n Name: Runloop AI Support\n Email: support@runloop.ai", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "lastUpdatedBy": "35240", "uid": "35240-b1660dbe-1152-4fdd-81da-7ce120db83f2" }, "item": [ { "name": "v1", "item": [ { "name": "benchmark_jobs", "item": [ { "name": "{id}", "item": [ { "name": "[Beta] Get a previously created BenchmarkJob.", "id": "1a26b4a3-a1a3-42d6-91be-6d5fbd4126c4", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_jobs/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_jobs", ":id" ], "variable": [ { "id": "be686fdb-5a27-4b5b-a363-1499693de670", "key": "id", "value": "", "description": "(Required) The BenchmarkJob ID." } ] }, "description": "[Beta] Get a BenchmarkJob given ID." }, "response": [ { "id": "dee39b3a-11f7-4b9b-839c-52b3377d9a70", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_jobs/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_jobs", ":id" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"state\": \"cancelled\",\n \"create_time_ms\": \"\",\n \"job_source\": {\n \"inline_yaml\": \"\",\n \"type\": \"harbor\"\n },\n \"job_spec\": {\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"agent_configs\": [\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"id0\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"exercitationdd\": \"\",\n \"dolore8\": \"\"\n },\n \"secrets\": {\n \"mollit__b\": \"\",\n \"velit8\": \"\",\n \"deserunt_c8\": \"\"\n }\n }\n },\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"in_6\": \"\",\n \"estc3\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"reprehenderit_756\": \"\"\n },\n \"secrets\": {\n \"sunt_5c2\": \"\"\n }\n }\n }\n ],\n \"orchestrator_config\": {\n \"n_concurrent_trials\": \"\",\n \"n_attempts\": \"\",\n \"timeout_multiplier\": \"\",\n \"quiet\": \"\"\n }\n },\n \"failure_reason\": \"\",\n \"benchmark_outcomes\": [\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"FAILED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"CANCELED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n }\n ],\n \"in_progress_runs\": [\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"failed\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"failed\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n }\n ]\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-dee39b3a-11f7-4b9b-839c-52b3377d9a70" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-1a26b4a3-a1a3-42d6-91be-6d5fbd4126c4" } ], "id": "7804f5a7-f303-45b4-84bd-56d415ac85d4", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-7804f5a7-f303-45b4-84bd-56d415ac85d4" }, { "name": "[Beta] Create a BenchmarkJob.", "id": "30b8f865-bf82-4b6f-a9ff-33922bc19467", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"name\": \"\",\n \"spec\": {\n \"inline_yaml\": \"\",\n \"type\": \"harbor\"\n }\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmark_jobs", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_jobs" ] }, "description": "[Beta] Create a BenchmarkJob that runs a set of scenarios entirely on runloop." }, "response": [ { "id": "b67edd97-0cdc-4277-8611-fd3a76869644", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "body": { "mode": "raw", "raw": "{\n \"name\": \"\",\n \"spec\": {\n \"inline_yaml\": \"\",\n \"type\": \"harbor\"\n }\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmark_jobs", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_jobs" ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"state\": \"cancelled\",\n \"create_time_ms\": \"\",\n \"job_source\": {\n \"inline_yaml\": \"\",\n \"type\": \"harbor\"\n },\n \"job_spec\": {\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"agent_configs\": [\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"id0\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"exercitationdd\": \"\",\n \"dolore8\": \"\"\n },\n \"secrets\": {\n \"mollit__b\": \"\",\n \"velit8\": \"\",\n \"deserunt_c8\": \"\"\n }\n }\n },\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"in_6\": \"\",\n \"estc3\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"reprehenderit_756\": \"\"\n },\n \"secrets\": {\n \"sunt_5c2\": \"\"\n }\n }\n }\n ],\n \"orchestrator_config\": {\n \"n_concurrent_trials\": \"\",\n \"n_attempts\": \"\",\n \"timeout_multiplier\": \"\",\n \"quiet\": \"\"\n }\n },\n \"failure_reason\": \"\",\n \"benchmark_outcomes\": [\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"FAILED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"CANCELED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n }\n ],\n \"in_progress_runs\": [\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"failed\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"failed\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n }\n ]\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-b67edd97-0cdc-4277-8611-fd3a76869644" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-30b8f865-bf82-4b6f-a9ff-33922bc19467" }, { "name": "[Beta] List BenchmarkJobs.", "id": "9f00b43e-c0c1-43fd-b5ca-7690fc03ef21", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_jobs?name=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_jobs" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] }, "description": "[Beta] List all BenchmarkJobs matching filter." }, "response": [ { "id": "5375673f-2808-4497-a03a-8ec7c65a8ab5", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_jobs?name=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_jobs" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"jobs\": [\n {\n \"id\": \"\",\n \"name\": \"\",\n \"state\": \"running\",\n \"create_time_ms\": \"\",\n \"job_source\": {\n \"inline_yaml\": \"\",\n \"type\": \"harbor\"\n },\n \"job_spec\": {\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"agent_configs\": [\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"et_36\": \"\",\n \"id__\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"esse49\": \"\",\n \"culpa_02\": \"\",\n \"labored3\": \"\",\n \"occaecat9_\": \"\"\n },\n \"secrets\": {\n \"nostrud_660\": \"\",\n \"aliquip_eb\": \"\"\n }\n }\n },\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"ina\": \"\",\n \"eu3\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"Ut_87f\": \"\"\n },\n \"secrets\": {\n \"do_6\": \"\",\n \"exercitation_0c\": \"\",\n \"adb8\": \"\"\n }\n }\n }\n ],\n \"orchestrator_config\": {\n \"n_concurrent_trials\": \"\",\n \"n_attempts\": \"\",\n \"timeout_multiplier\": \"\",\n \"quiet\": \"\"\n }\n },\n \"failure_reason\": \"\",\n \"benchmark_outcomes\": [\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"FAILED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"FAILED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"TIMEOUT\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n }\n ],\n \"in_progress_runs\": [\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"running\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"canceled\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n }\n ]\n },\n {\n \"id\": \"\",\n \"name\": \"\",\n \"state\": \"queued\",\n \"create_time_ms\": \"\",\n \"job_source\": {\n \"inline_yaml\": \"\",\n \"type\": \"harbor\"\n },\n \"job_spec\": {\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"agent_configs\": [\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"eiusmod_107\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"Duisb\": \"\",\n \"incididunt_1_\": \"\",\n \"ut38\": \"\"\n },\n \"secrets\": {\n \"dolor_09\": \"\",\n \"laboreb0\": \"\"\n }\n }\n },\n {\n \"name\": \"\",\n \"type\": \"job_agent\",\n \"agent_id\": \"\",\n \"model_name\": \"\",\n \"timeout_seconds\": \"\",\n \"kwargs\": {\n \"ine82\": \"\"\n },\n \"agent_environment\": {\n \"environment_variables\": {\n \"exercitationb3\": \"\",\n \"tempor3c5\": \"\"\n },\n \"secrets\": {\n \"eu1\": \"\",\n \"ex_b4b\": \"\"\n }\n }\n }\n ],\n \"orchestrator_config\": {\n \"n_concurrent_trials\": \"\",\n \"n_attempts\": \"\",\n \"timeout_multiplier\": \"\",\n \"quiet\": \"\"\n }\n },\n \"failure_reason\": \"\",\n \"benchmark_outcomes\": [\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"COMPLETED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"agent_name\": \"\",\n \"n_completed\": \"\",\n \"n_failed\": \"\",\n \"n_timeout\": \"\",\n \"scenario_outcomes\": [\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"CANCELED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n },\n {\n \"scenario_definition_id\": \"\",\n \"scenario_name\": \"\",\n \"state\": \"CANCELED\",\n \"scenario_run_id\": \"\",\n \"score\": \"\",\n \"duration_ms\": \"\",\n \"failure_reason\": {\n \"exception_type\": \"\",\n \"exception_message\": \"\"\n }\n }\n ],\n \"model_name\": \"\",\n \"average_score\": \"\",\n \"duration_ms\": \"\"\n }\n ],\n \"in_progress_runs\": [\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"canceled\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n },\n {\n \"benchmark_run_id\": \"\",\n \"state\": \"running\",\n \"start_time_ms\": \"\",\n \"agent_config\": {\n \"type\": \"external_api\",\n \"info\": \"\"\n },\n \"duration_ms\": \"\"\n }\n ]\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-5375673f-2808-4497-a03a-8ec7c65a8ab5" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-9f00b43e-c0c1-43fd-b5ca-7690fc03ef21" } ], "id": "bb2a6a50-025d-4912-9493-e0f9cef32f70", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-bb2a6a50-025d-4912-9493-e0f9cef32f70" }, { "name": "benchmark_runs", "item": [ { "name": "{id}", "item": [ { "name": "cancel", "item": [ { "name": "Cancel a currently running Benchmark run.", "id": "47d357f4-92b4-4c73-b459-61672246e082", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/cancel", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "cancel" ], "variable": [ { "id": "93cc60dd-ed01-4c84-bef4-b60949060cad", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Cancel a Benchmark run. This will do the following: 1. Cancel all running scenarios and shutdown the underlying Devbox resources 2. Update the benchmark state to CANCELED 3. Calculate final score from completed scenarios" }, "response": [ { "id": "f1a23e49-b297-4d90-b69f-342270af64df", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/cancel", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "cancel" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-f1a23e49-b297-4d90-b69f-342270af64df" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-47d357f4-92b4-4c73-b459-61672246e082" } ], "id": "1a86e7e7-3df8-456a-b6bc-e578644ecaf1", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-1a86e7e7-3df8-456a-b6bc-e578644ecaf1" }, { "name": "complete", "item": [ { "name": "Complete a BenchmarkRun.", "id": "7950876c-ff1d-420f-ad34-e1b15e5588c6", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/complete", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "complete" ], "variable": [ { "id": "5311dd98-1c13-4b30-adc4-52746704841d", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Complete a currently running BenchmarkRun." }, "response": [ { "id": "e48325ea-5942-4da5-8d9f-56d01a8ca071", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/complete", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "complete" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-e48325ea-5942-4da5-8d9f-56d01a8ca071" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-7950876c-ff1d-420f-ad34-e1b15e5588c6" } ], "id": "cebfe3a1-c1c8-4640-b864-8f13bc5bcf66", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-cebfe3a1-c1c8-4640-b864-8f13bc5bcf66" }, { "name": "download_logs", "item": [ { "name": "Download logs for a Benchmark run.", "id": "2312477e-96aa-49ed-8c7f-392f09eb81b9", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/zip" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/download_logs", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "download_logs" ], "variable": [ { "id": "92f8ddda-b512-4153-9fb4-2419e9204bc8", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Download a zip file containing all logs for a Benchmark run." }, "response": [ { "id": "1528714b-bf26-4757-a941-8a4e5d0cf4f3", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/zip" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/download_logs", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "download_logs" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "text", "header": [ { "key": "Content-Type", "value": "application/zip" }, { "disabled": false, "description": { "content": "(Required) application/zip", "type": "text/plain" }, "key": "Content-Type", "value": "" }, { "disabled": false, "description": { "content": "(Required) attachment; filename=\"benchmark_run_logs.zip\"", "type": "text/plain" }, "key": "Content-Disposition", "value": "" } ], "cookie": [], "responseTime": null, "body": "{\n \"format\": \"binary\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-1528714b-bf26-4757-a941-8a4e5d0cf4f3" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-2312477e-96aa-49ed-8c7f-392f09eb81b9" } ], "id": "68918aaa-b11b-4bc4-a022-84ae1e627166", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-68918aaa-b11b-4bc4-a022-84ae1e627166" }, { "name": "scenario_runs", "item": [ { "name": "List started scenario runs for a benchmark run.", "id": "a26d79f7-fe1e-4627-9090-699ca8b6758e", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/scenario_runs?state=scoring&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "scenario_runs" ], "query": [ { "description": "Filter by Scenario Run state", "key": "state", "value": "scoring" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ], "variable": [ { "id": "4a6d10d1-e1c6-4c13-89da-4bdce5e3d3a2", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "List started scenario runs for a benchmark run." }, "response": [ { "id": "5bddac22-a691-40ad-8e4b-8b77c6e8aab4", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id/scenario_runs?state=scoring&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id", "scenario_runs" ], "query": [ { "description": "Filter by Scenario Run state", "key": "state", "value": "scoring" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"runs\": [\n {\n \"id\": \"\",\n \"scenario_id\": \"\",\n \"devbox_id\": \"\",\n \"state\": \"running\",\n \"metadata\": {\n \"dolore_0fb\": \"\"\n },\n \"name\": \"\",\n \"benchmark_run_id\": \"\",\n \"scoring_contract_result\": {\n \"score\": \"\",\n \"scoring_function_results\": [\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"complete\"\n },\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"unknown\"\n }\n ]\n },\n \"start_time_ms\": \"\",\n \"duration_ms\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"dolore9a\": \"\",\n \"eiusmod_cf\": \"\",\n \"in1\": \"\"\n },\n \"secrets_provided\": {\n \"anim_770\": \"\",\n \"tempor_4\": \"\"\n }\n },\n {\n \"id\": \"\",\n \"scenario_id\": \"\",\n \"devbox_id\": \"\",\n \"state\": \"running\",\n \"metadata\": {\n \"consequat1\": \"\",\n \"non_5\": \"\",\n \"none69\": \"\"\n },\n \"name\": \"\",\n \"benchmark_run_id\": \"\",\n \"scoring_contract_result\": {\n \"score\": \"\",\n \"scoring_function_results\": [\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"error\"\n },\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"unknown\"\n }\n ]\n },\n \"start_time_ms\": \"\",\n \"duration_ms\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"ex9b9\": \"\"\n },\n \"secrets_provided\": {\n \"anim_5e9\": \"\"\n }\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-5bddac22-a691-40ad-8e4b-8b77c6e8aab4" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-a26d79f7-fe1e-4627-9090-699ca8b6758e" } ], "id": "00ec6f27-e21e-4afa-bbc2-4cf7a75d9ac6", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-00ec6f27-e21e-4afa-bbc2-4cf7a75d9ac6" }, { "name": "Get a previously created BenchmarkRun.", "id": "c03ba9b9-7dfb-45e5-89c7-92d5650e00a6", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id" ], "variable": [ { "id": "3ce437a0-ab35-4512-8b01-91a033cf79cb", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Get a BenchmarkRun given ID." }, "response": [ { "id": "3e2103d2-d80a-4a18-bbf7-162e9c11bb8c", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs", ":id" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-3e2103d2-d80a-4a18-bbf7-162e9c11bb8c" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-c03ba9b9-7dfb-45e5-89c7-92d5650e00a6" } ], "id": "8ff405ee-6e5d-4e59-b0f4-84b08c8ed451", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-8ff405ee-6e5d-4e59-b0f4-84b08c8ed451" }, { "name": "List BenchmarkRuns.", "id": "dc865b5a-7d89-4c5a-a5e4-b9ddcd26adc6", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs?name=&benchmark_id=&state=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The Benchmark ID to filter by.", "key": "benchmark_id", "value": "" }, { "description": "Filter by state", "key": "state", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] }, "description": "List all BenchmarkRuns matching filter." }, "response": [ { "id": "726cc2b1-b2f8-47b7-aecb-2460301d36b3", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmark_runs?name=&benchmark_id=&state=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmark_runs" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The Benchmark ID to filter by.", "key": "benchmark_id", "value": "" }, { "description": "Filter by state", "key": "state", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"runs\": [\n {\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"et_410\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"deserunt4b\": \"\",\n \"culpa_fc7\": \"\"\n },\n \"secrets_provided\": {\n \"labore_aa9\": \"\",\n \"culpa_d90\": \"\"\n }\n },\n {\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"canceled\",\n \"metadata\": {\n \"minime3\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"commodo_a\": \"\"\n },\n \"secrets_provided\": {\n \"laborum8d0\": \"\",\n \"aliquipac5\": \"\",\n \"ametdc1\": \"\"\n }\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-726cc2b1-b2f8-47b7-aecb-2460301d36b3" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-dc865b5a-7d89-4c5a-a5e4-b9ddcd26adc6" } ], "id": "1bd2f33e-c87d-476d-bcf7-d8af084998cb", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-1bd2f33e-c87d-476d-bcf7-d8af084998cb" }, { "name": "benchmarks", "item": [ { "name": "list_public", "item": [ { "name": "List Public Benchmarks.", "id": "835e8034-7b24-4153-b185-698bed15984e", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/list_public?limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "list_public" ], "query": [ { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] }, "description": "List all public benchmarks matching filter." }, "response": [ { "id": "dc67406e-5350-4b1a-8f98-444b57552240", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/list_public?limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "list_public" ], "query": [ { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"benchmarks\": [\n {\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"cillum29e\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n },\n {\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"minim_9\": \"\",\n \"Lorem_a\": \"\",\n \"pariatur_e\": \"\",\n \"Ut3_b\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-dc67406e-5350-4b1a-8f98-444b57552240" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-835e8034-7b24-4153-b185-698bed15984e" } ], "id": "43e57393-b170-475d-8472-3e535a335f65", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-43e57393-b170-475d-8472-3e535a335f65" }, { "name": "metadata", "item": [ { "name": "keys", "item": [ { "name": "{key}", "item": [ { "name": "values", "item": [ { "name": "List values for a specific benchmark metadata key.", "id": "e75104d3-f351-4822-bc5d-3cec8629964c", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/metadata/keys/:key/values", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "metadata", "keys", ":key", "values" ], "variable": [ { "id": "d9038ac4-d770-4ad4-9d09-203d334f8e7e", "key": "key", "value": "", "description": "(Required) The metadata key to get values for." } ] }, "description": "Returns a list of all available metadata keys that can be used for filtering benchmarks." }, "response": [ { "id": "738cedde-655f-440f-9e0a-79dc82d68c61", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/metadata/keys/:key/values", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "metadata", "keys", ":key", "values" ], "variable": [ { "key": "key" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"key\": \"\",\n \"values\": [\n \"\",\n \"\"\n ]\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-738cedde-655f-440f-9e0a-79dc82d68c61" }, { "id": "e3a0a455-5034-4d61-9954-27c690b19f25", "name": "Invalid metadata key provided.", "originalRequest": { "method": "GET", "header": [ { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/metadata/keys/:key/values", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "metadata", "keys", ":key", "values" ], "variable": [ { "key": "key" } ] } }, "status": "Bad Request", "code": 400, "_postman_previewlanguage": "text", "header": [], "cookie": [], "responseTime": null, "body": null, "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-e3a0a455-5034-4d61-9954-27c690b19f25" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-e75104d3-f351-4822-bc5d-3cec8629964c" } ], "id": "640cd2d4-0e13-4b19-9ef2-6e75024de8c8", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-640cd2d4-0e13-4b19-9ef2-6e75024de8c8" } ], "id": "06058f58-b1b9-4587-bad6-28bcbcfc7a07", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-06058f58-b1b9-4587-bad6-28bcbcfc7a07" }, { "name": "List available benchmark metadata keys.", "id": "104b48a0-fa2e-407a-82eb-096487122014", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/metadata/keys", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "metadata", "keys" ] }, "description": "Returns a list of all available metadata keys that can be used for filtering benchmarks." }, "response": [ { "id": "f6fa8c8b-091d-40cc-951b-21109d442602", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/metadata/keys", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "metadata", "keys" ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"keys\": [\n \"\",\n \"\"\n ]\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-f6fa8c8b-091d-40cc-951b-21109d442602" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-104b48a0-fa2e-407a-82eb-096487122014" } ], "id": "3cc4c955-c749-4242-8f60-5f569c39a3b7", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-3cc4c955-c749-4242-8f60-5f569c39a3b7" } ], "id": "756b53b6-e4de-4c90-8e25-656091d7c1f3", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-756b53b6-e4de-4c90-8e25-656091d7c1f3" }, { "name": "runs", "item": [ { "name": "{id}", "item": [ { "name": "cancel", "item": [ { "name": "Cancel a currently running Benchmark run.", "id": "432cfdc5-230c-4f8b-a6df-ef7b0d1154cb", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/cancel", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "cancel" ], "variable": [ { "id": "4e85f17d-077a-48b7-b28e-00316099ad56", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Cancel a Benchmark run. This will do the following: 1. Cancel all running scenarios and shutdown the underlying Devbox resources 2. Update the benchmark state to CANCELED 3. Calculate final score from completed scenarios" }, "response": [ { "id": "56610718-6c72-4278-8814-60e2e537319b", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/cancel", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "cancel" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-56610718-6c72-4278-8814-60e2e537319b" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-432cfdc5-230c-4f8b-a6df-ef7b0d1154cb" } ], "id": "2dbb8774-5bda-45c5-8db9-853b1dbac24e", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-2dbb8774-5bda-45c5-8db9-853b1dbac24e" }, { "name": "complete", "item": [ { "name": "Complete a BenchmarkRun.", "id": "a7db046b-8153-4c8e-8340-0e8883a9081c", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/complete", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "complete" ], "variable": [ { "id": "6efb3abb-bd85-4e7a-ac1f-69346a2d2c00", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Complete a currently running BenchmarkRun." }, "response": [ { "id": "dcc1e83b-1a01-4328-a4f3-ef63290e34d7", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/complete", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "complete" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-dcc1e83b-1a01-4328-a4f3-ef63290e34d7" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-a7db046b-8153-4c8e-8340-0e8883a9081c" } ], "id": "74598a09-bfbd-409d-b5df-c7d37873117e", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-74598a09-bfbd-409d-b5df-c7d37873117e" }, { "name": "download_logs", "item": [ { "name": "Download logs for a Benchmark run.", "id": "02527a2b-05c4-40f9-a1fd-83d77c5072ec", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/zip" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/download_logs", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "download_logs" ], "variable": [ { "id": "1d829b44-9354-4cab-b0fd-dc6a5f523701", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Download a zip file containing all logs for a Benchmark run." }, "response": [ { "id": "dd4213e5-66fc-472d-a66c-1498c2870de0", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/zip" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/download_logs", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "download_logs" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "text", "header": [ { "key": "Content-Type", "value": "application/zip" }, { "disabled": false, "description": { "content": "(Required) application/zip", "type": "text/plain" }, "key": "Content-Type", "value": "" }, { "disabled": false, "description": { "content": "(Required) attachment; filename=\"benchmark_run_logs.zip\"", "type": "text/plain" }, "key": "Content-Disposition", "value": "" } ], "cookie": [], "responseTime": null, "body": "{\n \"format\": \"binary\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-dd4213e5-66fc-472d-a66c-1498c2870de0" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-02527a2b-05c4-40f9-a1fd-83d77c5072ec" } ], "id": "aee91d68-06f2-4252-b0fe-f23148efbd75", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-aee91d68-06f2-4252-b0fe-f23148efbd75" }, { "name": "scenario_runs", "item": [ { "name": "List started scenario runs for a benchmark run.", "id": "6687cad0-ac08-4619-9fb3-dd329272c4c3", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/scenario_runs?state=scoring&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "scenario_runs" ], "query": [ { "description": "Filter by Scenario Run state", "key": "state", "value": "scoring" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ], "variable": [ { "id": "7480e3d1-2cb1-4de1-bc88-c8b8fae61f9a", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "List started scenario runs for a benchmark run." }, "response": [ { "id": "a968067b-402c-40e0-a9ee-274b0bef85cf", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id/scenario_runs?state=scoring&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id", "scenario_runs" ], "query": [ { "description": "Filter by Scenario Run state", "key": "state", "value": "scoring" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"runs\": [\n {\n \"id\": \"\",\n \"scenario_id\": \"\",\n \"devbox_id\": \"\",\n \"state\": \"running\",\n \"metadata\": {\n \"dolore_0fb\": \"\"\n },\n \"name\": \"\",\n \"benchmark_run_id\": \"\",\n \"scoring_contract_result\": {\n \"score\": \"\",\n \"scoring_function_results\": [\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"complete\"\n },\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"unknown\"\n }\n ]\n },\n \"start_time_ms\": \"\",\n \"duration_ms\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"dolore9a\": \"\",\n \"eiusmod_cf\": \"\",\n \"in1\": \"\"\n },\n \"secrets_provided\": {\n \"anim_770\": \"\",\n \"tempor_4\": \"\"\n }\n },\n {\n \"id\": \"\",\n \"scenario_id\": \"\",\n \"devbox_id\": \"\",\n \"state\": \"running\",\n \"metadata\": {\n \"consequat1\": \"\",\n \"non_5\": \"\",\n \"none69\": \"\"\n },\n \"name\": \"\",\n \"benchmark_run_id\": \"\",\n \"scoring_contract_result\": {\n \"score\": \"\",\n \"scoring_function_results\": [\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"error\"\n },\n {\n \"score\": \"\",\n \"scoring_function_name\": \"\",\n \"output\": \"\",\n \"state\": \"unknown\"\n }\n ]\n },\n \"start_time_ms\": \"\",\n \"duration_ms\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"ex9b9\": \"\"\n },\n \"secrets_provided\": {\n \"anim_5e9\": \"\"\n }\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-a968067b-402c-40e0-a9ee-274b0bef85cf" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-6687cad0-ac08-4619-9fb3-dd329272c4c3" } ], "id": "ff3e04bc-faca-40d8-9326-4664e8872c5a", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-ff3e04bc-faca-40d8-9326-4664e8872c5a" }, { "name": "Get a previously created BenchmarkRun.", "id": "6d835293-a529-47d1-937e-926636aed0a9", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id" ], "variable": [ { "id": "a6cc323b-8719-4af7-941f-c9d90f0cb1a5", "key": "id", "value": "", "description": "(Required) The BenchmarkRun ID." } ] }, "description": "Get a BenchmarkRun given ID." }, "response": [ { "id": "4873a45e-b6a3-42f9-b2a0-fdf363e2e1bd", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs", ":id" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-4873a45e-b6a3-42f9-b2a0-fdf363e2e1bd" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-6d835293-a529-47d1-937e-926636aed0a9" } ], "id": "9d619ce4-c0f0-4425-9a46-12ea8846e258", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-9d619ce4-c0f0-4425-9a46-12ea8846e258" }, { "name": "List BenchmarkRuns.", "id": "6919480c-4b47-402d-859e-693451eadc55", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs?name=&benchmark_id=&state=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The Benchmark ID to filter by.", "key": "benchmark_id", "value": "" }, { "description": "Filter by state", "key": "state", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] }, "description": "List all BenchmarkRuns matching filter." }, "response": [ { "id": "f3bc058a-6035-49c9-abcd-bc6138d2d13a", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/runs?name=&benchmark_id=&state=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "runs" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The Benchmark ID to filter by.", "key": "benchmark_id", "value": "" }, { "description": "Filter by state", "key": "state", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"runs\": [\n {\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"et_410\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"deserunt4b\": \"\",\n \"culpa_fc7\": \"\"\n },\n \"secrets_provided\": {\n \"labore_aa9\": \"\",\n \"culpa_d90\": \"\"\n }\n },\n {\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"canceled\",\n \"metadata\": {\n \"minime3\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"commodo_a\": \"\"\n },\n \"secrets_provided\": {\n \"laborum8d0\": \"\",\n \"aliquipac5\": \"\",\n \"ametdc1\": \"\"\n }\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-f3bc058a-6035-49c9-abcd-bc6138d2d13a" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-6919480c-4b47-402d-859e-693451eadc55" } ], "id": "e9ded327-7ec7-40b8-b5c9-ff677326b1a4", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-e9ded327-7ec7-40b8-b5c9-ff677326b1a4" }, { "name": "start_run", "item": [ { "name": "Start a new BenchmarkRun.", "id": "01020345-a301-4c69-a13b-23d1da82ca1c", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"benchmark_id\": \"\",\n \"run_name\": \"\",\n \"metadata\": {\n \"aute_ba\": \"\"\n },\n \"runProfile\": {\n \"purpose\": \"\",\n \"envVars\": {\n \"culpa9ed\": \"\",\n \"fugiat__3\": \"\",\n \"in8_8\": \"\",\n \"commodo_c\": \"\"\n },\n \"secrets\": {\n \"Lorema3\": \"\"\n },\n \"launchParameters\": {\n \"launch_commands\": [\n \"\",\n \"\"\n ],\n \"resource_size_request\": \"XX_LARGE\",\n \"available_ports\": [\n \"\",\n \"\"\n ],\n \"keep_alive_time_seconds\": \"\",\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"shutdown\"\n },\n \"custom_cpu_cores\": \"\",\n \"custom_gb_memory\": \"\",\n \"custom_disk_size\": \"\",\n \"architecture\": \"x86_64\",\n \"user_parameters\": {\n \"username\": \"\",\n \"uid\": \"\"\n },\n \"required_services\": [\n \"\",\n \"\"\n ],\n \"network_policy_id\": \"\",\n \"lifecycle\": {\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"suspend\"\n },\n \"resume_triggers\": {\n \"http\": \"\",\n \"axon_event\": \"\"\n }\n }\n },\n \"mounts\": [\n {\n \"object_id\": \"\",\n \"object_path\": \"\",\n \"type\": \"object_mount\"\n },\n {\n \"object_id\": \"\",\n \"object_path\": \"\",\n \"type\": \"object_mount\"\n }\n ]\n }\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks/start_run", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "start_run" ] }, "description": "Start a new BenchmarkRun based on the provided Benchmark." }, "response": [ { "id": "bdc87e84-5c64-4a41-95e7-c6e8ff6272f8", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "body": { "mode": "raw", "raw": "{\n \"benchmark_id\": \"\",\n \"run_name\": \"\",\n \"metadata\": {\n \"aute_ba\": \"\"\n },\n \"runProfile\": {\n \"purpose\": \"\",\n \"envVars\": {\n \"culpa9ed\": \"\",\n \"fugiat__3\": \"\",\n \"in8_8\": \"\",\n \"commodo_c\": \"\"\n },\n \"secrets\": {\n \"Lorema3\": \"\"\n },\n \"launchParameters\": {\n \"launch_commands\": [\n \"\",\n \"\"\n ],\n \"resource_size_request\": \"XX_LARGE\",\n \"available_ports\": [\n \"\",\n \"\"\n ],\n \"keep_alive_time_seconds\": \"\",\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"shutdown\"\n },\n \"custom_cpu_cores\": \"\",\n \"custom_gb_memory\": \"\",\n \"custom_disk_size\": \"\",\n \"architecture\": \"x86_64\",\n \"user_parameters\": {\n \"username\": \"\",\n \"uid\": \"\"\n },\n \"required_services\": [\n \"\",\n \"\"\n ],\n \"network_policy_id\": \"\",\n \"lifecycle\": {\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"suspend\"\n },\n \"resume_triggers\": {\n \"http\": \"\",\n \"axon_event\": \"\"\n }\n }\n },\n \"mounts\": [\n {\n \"object_id\": \"\",\n \"object_path\": \"\",\n \"type\": \"object_mount\"\n },\n {\n \"object_id\": \"\",\n \"object_path\": \"\",\n \"type\": \"object_mount\"\n }\n ]\n }\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks/start_run", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", "start_run" ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"ut_fe1\": \"\",\n \"ametbf_\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"Excepteur_9\": \"\",\n \"labore9\": \"\"\n },\n \"secrets_provided\": {\n \"magna59\": \"\",\n \"dolor_b3\": \"\"\n }\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-bdc87e84-5c64-4a41-95e7-c6e8ff6272f8" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-01020345-a301-4c69-a13b-23d1da82ca1c" } ], "id": "621852e3-baae-4f76-ba2f-28ac1066bdb1", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-621852e3-baae-4f76-ba2f-28ac1066bdb1" }, { "name": "{id}", "item": [ { "name": "archive", "item": [ { "name": "Archive a Benchmark.", "id": "b2aaa20c-ac92-4cc6-affc-09a79a07901e", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/archive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "archive" ], "variable": [ { "id": "f3e1473f-c494-459e-bf19-3feeda4045cd", "key": "id", "value": "", "description": "(Required) The ID of the Benchmark to archive." } ] }, "description": "Archive a previously created Benchmark. The benchmark will no longer appear in list endpoints but can still be retrieved by ID." }, "response": [ { "id": "658c8fc7-fb55-432d-b2b8-cd62877774f6", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/archive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "archive" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"laborum__c5\": \"\",\n \"do_52\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-658c8fc7-fb55-432d-b2b8-cd62877774f6" }, { "id": "3b2c2db0-f140-4a5d-8f1e-ea4b9b1e1561", "name": "Cannot archive public benchmarks.", "originalRequest": { "method": "POST", "header": [ { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/archive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "archive" ], "variable": [ { "key": "id" } ] } }, "status": "Forbidden", "code": 403, "_postman_previewlanguage": "text", "header": [], "cookie": [], "responseTime": null, "body": null, "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-3b2c2db0-f140-4a5d-8f1e-ea4b9b1e1561" }, { "id": "4d5bfeb1-b77e-4f45-b462-cd1a30c594b1", "name": "Benchmark not found.", "originalRequest": { "method": "POST", "header": [ { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/archive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "archive" ], "variable": [ { "key": "id" } ] } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "text", "header": [], "cookie": [], "responseTime": null, "body": null, "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-4d5bfeb1-b77e-4f45-b462-cd1a30c594b1" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-b2aaa20c-ac92-4cc6-affc-09a79a07901e" } ], "id": "582e68ca-8729-4d4b-91ca-30b2fb989066", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-582e68ca-8729-4d4b-91ca-30b2fb989066" }, { "name": "definitions", "item": [ { "name": "Get scenario definitions for a Benchmark.", "id": "0095c30b-78d1-46de-9716-37155df848fc", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/definitions?limit=&starting_after=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "definitions" ], "query": [ { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" } ], "variable": [ { "id": "abb64c3c-a679-432c-8f0c-50f3e1b64889", "key": "id", "value": "", "description": "(Required) The Benchmark ID." } ] }, "description": "Get scenario definitions for a previously created Benchmark." }, "response": [ { "id": "04163832-0422-479b-8b18-770cf2112b0b", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/definitions?limit=&starting_after=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "definitions" ], "query": [ { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" } ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"scenarios\": [\n {\n \"id\": \"\",\n \"name\": \"\",\n \"input_context\": {\n \"problem_statement\": \"\",\n \"additional_context\": {}\n },\n \"scoring_contract\": {\n \"scoring_function_parameters\": [\n {\n \"name\": \"\",\n \"scorer\": {\n \"search_directory\": \"\",\n \"pattern\": \"\",\n \"type\": \"ast_grep_scorer\",\n \"lang\": \"\"\n },\n \"weight\": \"\"\n },\n {\n \"name\": \"\",\n \"scorer\": {\n \"search_directory\": \"\",\n \"pattern\": \"\",\n \"type\": \"ast_grep_scorer\",\n \"lang\": \"\"\n },\n \"weight\": \"\"\n }\n ]\n },\n \"metadata\": {\n \"ullamco58\": \"\"\n },\n \"status\": \"archived\",\n \"environment\": {\n \"blueprint_id\": \"\",\n \"snapshot_id\": \"\",\n \"launch_parameters\": {\n \"launch_commands\": [\n \"\",\n \"\"\n ],\n \"resource_size_request\": \"LARGE\",\n \"available_ports\": [\n \"\",\n \"\"\n ],\n \"keep_alive_time_seconds\": \"\",\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"suspend\"\n },\n \"custom_cpu_cores\": \"\",\n \"custom_gb_memory\": \"\",\n \"custom_disk_size\": \"\",\n \"architecture\": \"x86_64\",\n \"user_parameters\": {\n \"username\": \"\",\n \"uid\": \"\"\n },\n \"required_services\": [\n \"\",\n \"\"\n ],\n \"network_policy_id\": \"\",\n \"lifecycle\": {\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"suspend\"\n },\n \"resume_triggers\": {\n \"http\": \"\",\n \"axon_event\": \"\"\n }\n }\n },\n \"working_directory\": \"\"\n },\n \"reference_output\": \"\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"validation_type\": \"UNSPECIFIED\",\n \"scorer_timeout_sec\": \"\"\n },\n {\n \"id\": \"\",\n \"name\": \"\",\n \"input_context\": {\n \"problem_statement\": \"\",\n \"additional_context\": {}\n },\n \"scoring_contract\": {\n \"scoring_function_parameters\": [\n {\n \"name\": \"\",\n \"scorer\": {\n \"search_directory\": \"\",\n \"pattern\": \"\",\n \"type\": \"ast_grep_scorer\",\n \"lang\": \"\"\n },\n \"weight\": \"\"\n },\n {\n \"name\": \"\",\n \"scorer\": {\n \"search_directory\": \"\",\n \"pattern\": \"\",\n \"type\": \"ast_grep_scorer\",\n \"lang\": \"\"\n },\n \"weight\": \"\"\n }\n ]\n },\n \"metadata\": {\n \"idf4\": \"\"\n },\n \"status\": \"active\",\n \"environment\": {\n \"blueprint_id\": \"\",\n \"snapshot_id\": \"\",\n \"launch_parameters\": {\n \"launch_commands\": [\n \"\",\n \"\"\n ],\n \"resource_size_request\": \"XX_LARGE\",\n \"available_ports\": [\n \"\",\n \"\"\n ],\n \"keep_alive_time_seconds\": \"\",\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"suspend\"\n },\n \"custom_cpu_cores\": \"\",\n \"custom_gb_memory\": \"\",\n \"custom_disk_size\": \"\",\n \"architecture\": \"x86_64\",\n \"user_parameters\": {\n \"username\": \"\",\n \"uid\": \"\"\n },\n \"required_services\": [\n \"\",\n \"\"\n ],\n \"network_policy_id\": \"\",\n \"lifecycle\": {\n \"after_idle\": {\n \"idle_time_seconds\": \"\",\n \"on_idle\": \"suspend\"\n },\n \"resume_triggers\": {\n \"http\": \"\",\n \"axon_event\": \"\"\n }\n }\n },\n \"working_directory\": \"\"\n },\n \"reference_output\": \"\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"validation_type\": \"FORWARD\",\n \"scorer_timeout_sec\": \"\"\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-04163832-0422-479b-8b18-770cf2112b0b" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-0095c30b-78d1-46de-9716-37155df848fc" } ], "id": "ee4b3a0b-c169-42d6-809c-8fd503b4f786", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-ee4b3a0b-c169-42d6-809c-8fd503b4f786" }, { "name": "runs", "item": [ { "name": "Get runs for a provided Benchmark.", "id": "6a52e829-7d00-4914-b378-5ed685650895", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/runs?limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "runs" ], "query": [ { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ], "variable": [ { "id": "6932b9a4-15e8-491d-8f95-240d2d4f785f", "key": "id", "value": "", "description": "(Required) The Benchmark ID." } ] }, "description": "Get runs for a previously created Benchmark." }, "response": [ { "id": "83173ae8-b933-4817-84d2-c48d393b8aae", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/runs?limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "runs" ], "query": [ { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"runs\": [\n {\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"completed\",\n \"metadata\": {\n \"et_410\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"deserunt4b\": \"\",\n \"culpa_fc7\": \"\"\n },\n \"secrets_provided\": {\n \"labore_aa9\": \"\",\n \"culpa_d90\": \"\"\n }\n },\n {\n \"id\": \"\",\n \"start_time_ms\": \"\",\n \"state\": \"canceled\",\n \"metadata\": {\n \"minime3\": \"\"\n },\n \"benchmark_id\": \"\",\n \"name\": \"\",\n \"duration_ms\": \"\",\n \"score\": \"\",\n \"purpose\": \"\",\n \"environment_variables\": {\n \"commodo_a\": \"\"\n },\n \"secrets_provided\": {\n \"laborum8d0\": \"\",\n \"aliquipac5\": \"\",\n \"ametdc1\": \"\"\n }\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-83173ae8-b933-4817-84d2-c48d393b8aae" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-6a52e829-7d00-4914-b378-5ed685650895" } ], "id": "e7519811-70c8-4fa1-86e1-f60100bbfc6b", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-e7519811-70c8-4fa1-86e1-f60100bbfc6b" }, { "name": "scenarios", "item": [ { "name": "Modify scenarios for a Benchmark.", "id": "509aba71-a9c1-4eb4-acf2-f46a6bdcb205", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"scenarios_to_add\": [\n \"\",\n \"\"\n ],\n \"scenarios_to_remove\": [\n \"\",\n \"\"\n ]\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/scenarios", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "scenarios" ], "variable": [ { "id": "92ad4e9b-54fb-4309-825e-7ec5b99076a0", "key": "id", "value": "", "description": "(Required) The Benchmark ID." } ] }, "description": "Add and/or remove Scenario IDs from an existing Benchmark." }, "response": [ { "id": "877e2c0c-7db2-446c-8cea-2b1015283005", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "body": { "mode": "raw", "raw": "{\n \"scenarios_to_add\": [\n \"\",\n \"\"\n ],\n \"scenarios_to_remove\": [\n \"\",\n \"\"\n ]\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/scenarios", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "scenarios" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"laborum__c5\": \"\",\n \"do_52\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-877e2c0c-7db2-446c-8cea-2b1015283005" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-509aba71-a9c1-4eb4-acf2-f46a6bdcb205" } ], "id": "7610de3d-07ba-41ca-80e2-508d8e612322", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-7610de3d-07ba-41ca-80e2-508d8e612322" }, { "name": "unarchive", "item": [ { "name": "Unarchive a Benchmark.", "id": "b68caab9-1e4e-4954-8ea8-f6e1e5fb2d40", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/unarchive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "unarchive" ], "variable": [ { "id": "2907ce19-2029-43bf-a5ef-aef4dbf52dfd", "key": "id", "value": "", "description": "(Required) The ID of the Benchmark to unarchive." } ] }, "description": "Unarchive a previously archived Benchmark. The benchmark will appear in list endpoints again." }, "response": [ { "id": "c2c621fb-158c-4eb7-8177-b5b90b92e841", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/unarchive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "unarchive" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"laborum__c5\": \"\",\n \"do_52\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-c2c621fb-158c-4eb7-8177-b5b90b92e841" }, { "id": "b82cf0e8-07c3-44b4-84a6-992a95797702", "name": "Cannot unarchive public benchmarks.", "originalRequest": { "method": "POST", "header": [ { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/unarchive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "unarchive" ], "variable": [ { "key": "id" } ] } }, "status": "Forbidden", "code": 403, "_postman_previewlanguage": "text", "header": [], "cookie": [], "responseTime": null, "body": null, "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-b82cf0e8-07c3-44b4-84a6-992a95797702" }, { "id": "210b8ac2-7cf0-4f29-a92e-075679740313", "name": "Benchmark not found.", "originalRequest": { "method": "POST", "header": [ { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id/unarchive", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id", "unarchive" ], "variable": [ { "key": "id" } ] } }, "status": "Not Found", "code": 404, "_postman_previewlanguage": "text", "header": [], "cookie": [], "responseTime": null, "body": null, "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-210b8ac2-7cf0-4f29-a92e-075679740313" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-b68caab9-1e4e-4954-8ea8-f6e1e5fb2d40" } ], "id": "6b6105ef-3f4e-4610-8265-20a1f5caaf8b", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-6b6105ef-3f4e-4610-8265-20a1f5caaf8b" }, { "name": "Update a Benchmark.", "id": "3e779363-501a-4a4e-8527-57097864dc24", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"name\": \"\",\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"exercitation836\": \"\"\n },\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"attribution\": \"\",\n \"description\": \"\"\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id" ], "variable": [ { "id": "e2995fdb-8dff-41d8-a0b9-a5977918a040", "key": "id", "value": "", "description": "(Required) The Benchmark ID." } ] }, "description": "Update a Benchmark. Fields that are null will preserve the existing value. Fields that are provided (including empty values) will replace the existing value entirely." }, "response": [ { "id": "81d36641-8a03-4a4b-8a64-73b408b7f429", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "body": { "mode": "raw", "raw": "{\n \"name\": \"\",\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"exercitation836\": \"\"\n },\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"attribution\": \"\",\n \"description\": \"\"\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"laborum__c5\": \"\",\n \"do_52\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-81d36641-8a03-4a4b-8a64-73b408b7f429" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-3e779363-501a-4a4e-8527-57097864dc24" }, { "name": "Get a Benchmark.", "id": "e624af73-41ea-449c-bebf-fb6e24408e0d", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id" ], "variable": [ { "id": "880b653f-3ea0-4741-9244-9575d8c6cc54", "key": "id", "value": "", "description": "(Required) The Benchmark ID." } ] }, "description": "Get a previously created Benchmark." }, "response": [ { "id": "652e9ec6-8300-4ca1-b7f5-78849e85c5da", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks/:id", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks", ":id" ], "variable": [ { "key": "id" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"laborum__c5\": \"\",\n \"do_52\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-652e9ec6-8300-4ca1-b7f5-78849e85c5da" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-e624af73-41ea-449c-bebf-fb6e24408e0d" } ], "id": "74994450-16ae-482d-a143-9454c2e1025e", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-74994450-16ae-482d-a143-9454c2e1025e" }, { "name": "Create a Benchmark.", "id": "a328e336-5079-41ae-9b6c-6a9a48589b95", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" } ], "body": { "mode": "raw", "raw": "{\n \"name\": \"\",\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"eu_7b3\": \"\",\n \"consequat0a\": \"\"\n },\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"attribution\": \"\",\n \"description\": \"\"\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks" ] }, "description": "Create a Benchmark with a set of Scenarios." }, "response": [ { "id": "f83de5ab-5b63-4e48-b30a-c209b3fe65d9", "name": "OK", "originalRequest": { "method": "POST", "header": [ { "key": "Content-Type", "value": "application/json" }, { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "body": { "mode": "raw", "raw": "{\n \"name\": \"\",\n \"scenario_ids\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"eu_7b3\": \"\",\n \"consequat0a\": \"\"\n },\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"attribution\": \"\",\n \"description\": \"\"\n}", "options": { "raw": { "headerFamily": "json", "language": "json" } } }, "url": { "raw": "{{baseUrl}}/v1/benchmarks", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks" ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"laborum__c5\": \"\",\n \"do_52\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-f83de5ab-5b63-4e48-b30a-c209b3fe65d9" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-a328e336-5079-41ae-9b6c-6a9a48589b95" }, { "name": "List Benchmarks.", "id": "55e12283-c5a3-4c90-b490-37cfa6577a8e", "protocolProfileBehavior": { "disableBodyPruning": true }, "request": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks?name=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] }, "description": "List all Benchmarks matching filter." }, "response": [ { "id": "40e6a74d-40fd-4c2e-9b78-fae49ab0feb0", "name": "OK", "originalRequest": { "method": "GET", "header": [ { "key": "Accept", "value": "application/json" }, { "description": "Added as a part of security scheme: bearer", "key": "Authorization", "value": "Bearer " } ], "url": { "raw": "{{baseUrl}}/v1/benchmarks?name=&limit=&starting_after=&include_total_count=", "host": [ "{{baseUrl}}" ], "path": [ "v1", "benchmarks" ], "query": [ { "description": "Filter by name", "key": "name", "value": "" }, { "description": "The limit of items to return. Default is 20. Max is 5000.", "key": "limit", "value": "" }, { "description": "Load the next page of data starting after the item with the given ID.", "key": "starting_after", "value": "" }, { "description": "If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.", "key": "include_total_count", "value": "" } ] } }, "status": "OK", "code": 200, "_postman_previewlanguage": "json", "header": [ { "key": "Content-Type", "value": "application/json" } ], "cookie": [], "responseTime": null, "body": "{\n \"benchmarks\": [\n {\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"cillum29e\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n },\n {\n \"id\": \"\",\n \"name\": \"\",\n \"scenarioIds\": [\n \"\",\n \"\"\n ],\n \"metadata\": {\n \"minim_9\": \"\",\n \"Lorem_a\": \"\",\n \"pariatur_e\": \"\",\n \"Ut3_b\": \"\"\n },\n \"status\": \"active\",\n \"required_environment_variables\": [\n \"\",\n \"\"\n ],\n \"required_secret_names\": [\n \"\",\n \"\"\n ],\n \"is_public\": \"\",\n \"attribution\": \"\",\n \"description\": \"\"\n }\n ],\n \"has_more\": \"\",\n \"total_count\": \"\"\n}", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-40e6a74d-40fd-4c2e-9b78-fae49ab0feb0" } ], "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-55e12283-c5a3-4c90-b490-37cfa6577a8e" } ], "id": "b51e66c4-431a-43b9-9b36-761d25e6ae61", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-b51e66c4-431a-43b9-9b36-761d25e6ae61" } ], "id": "a4f2c0be-81bc-4a01-9d45-98f6a7bfcaaf", "createdAt": "2026-06-05T15:15:17.000Z", "updatedAt": "2026-06-05T15:15:17.000Z", "uid": "35240-a4f2c0be-81bc-4a01-9d45-98f6a7bfcaaf" } ], "auth": { "type": "bearer", "bearer": [ { "key": "token", "value": "{{bearerToken}}", "type": "string" } ] }, "variable": [ { "key": "baseUrl", "value": "https://api.runloop.ai" } ] }