{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/huggingbay-xyz/json-schema/huggingbay-xyz-canonical-evaluation-publication-schema.json",
  "title": "CanonicalEvaluationPublication",
  "x-generated": "2026-11-10",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/huggingbay-xyz-evals-api-openapi.yml#/components/schemas/CanonicalEvaluationPublication",
  "properties": {
    "schema_version": {
      "type": "string",
      "const": "bay-run.canonical-evals.public.v1",
      "title": "Schema Version",
      "default": "bay-run.canonical-evals.public.v1"
    },
    "availability": {
      "type": "boolean",
      "title": "Availability"
    },
    "artifact_schema_version": {
      "anyOf": [
        {
          "type": "string",
          "const": "bay-run.canonical-evals.artifact.v2"
        },
        {
          "type": "null"
        }
      ],
      "title": "Artifact Schema Version"
    },
    "required_cases_per_pin": {
      "type": "integer",
      "const": 40,
      "title": "Required Cases Per Pin",
      "default": 40
    },
    "dataset": {
      "$ref": "#/$defs/PublishedDataset"
    },
    "evaluated_at": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Evaluated At"
    },
    "evidence_type": {
      "anyOf": [
        {
          "type": "string",
          "const": "curated-public-fixture"
        },
        {
          "type": "null"
        }
      ],
      "title": "Evidence Type"
    },
    "results": {
      "items": {
        "$ref": "#/$defs/PublishedPinEvaluation"
      },
      "type": "array",
      "title": "Results"
    },
    "reason": {
      "anyOf": [
        {
          "type": "string",
          "enum": [
            "validated_artifact_absent",
            "validated_artifact_invalid",
            "validated_artifact_unreadable"
          ]
        },
        {
          "type": "null"
        }
      ],
      "title": "Reason"
    },
    "sentiment_candidate": {
      "anyOf": [
        {
          "$ref": "#/$defs/SentimentCandidateStatus"
        },
        {
          "type": "null"
        }
      ]
    }
  },
  "additionalProperties": false,
  "type": "object",
  "required": [
    "availability",
    "dataset",
    "results"
  ],
  "$defs": {
    "EvaluationScore": {
      "properties": {
        "metric": {
          "type": "string",
          "enum": [
            "accuracy",
            "f1",
            "macro_f1",
            "mrr",
            "ndcg",
            "ndcg_at_5"
          ],
          "title": "Metric"
        },
        "value": {
          "type": "number",
          "title": "Value"
        }
      },
      "additionalProperties": false,
      "type": "object",
      "required": [
        "metric",
        "value"
      ],
      "title": "EvaluationScore"
    },
    "PublishedDataset": {
      "properties": {
        "version": {
          "anyOf": [
            {
              "type": "string",
              "maxLength": 96,
              "minLength": 1,
              "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
            },
            {
              "type": "null"
            }
          ],
          "title": "Version"
        },
        "digest": {
          "anyOf": [
            {
              "type": "string",
              "pattern": "^sha256:[0-9a-f]{64}$"
            },
            {
              "type": "null"
            }
          ],
          "title": "Digest"
        }
      },
      "additionalProperties": false,
      "type": "object",
      "title": "PublishedDataset"
    },
    "PublishedPinEvaluation": {
      "properties": {
        "pin_id": {
          "type": "string",
          "title": "Pin Id"
        },
        "route_id": {
          "type": "string",
          "title": "Route Id"
        },
        "model": {
          "anyOf": [
            {
              "type": "string",
              "maxLength": 256,
              "minLength": 1,
              "pattern": "^[A-Za-z0-9][A-Za-z0-9._/@:+-]*$"
            },
            {
              "type": "null"
            }
          ],
          "title": "Model"
        },
        "case_count": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "title": "Case Count"
        },
        "score": {
          "anyOf": [
            {
              "$ref": "#/$defs/EvaluationScore"
            },
            {
              "type": "null"
            }
          ]
        }
      },
      "additionalProperties": false,
      "type": "object",
      "required": [
        "pin_id",
        "route_id"
      ],
      "title": "PublishedPinEvaluation"
    },
    "SentimentCandidateSource": {
      "properties": {
        "dataset": {
          "type": "string",
          "const": "cardiffnlp/tweet_eval",
          "title": "Dataset"
        },
        "config": {
          "type": "string",
          "const": "sentiment",
          "title": "Config"
        },
        "revision": {
          "type": "string",
          "const": "b3a375baf0f409c77e6bc7aa35102b7b3534f8be",
          "title": "Revision"
        },
        "file": {
          "type": "string",
          "const": "sentiment/test-00000-of-00001.parquet",
          "title": "File"
        },
        "file_oid": {
          "type": "string",
          "const": "770b73d16e62f3a93734decccaca43a83c12a1c9",
          "title": "File Oid"
        },
        "file_sha256": {
          "type": "string",
          "const": "sha256:f73aea52d43879223019e733b944780f7e9203ee32459ea006f554011379fcdf",
          "title": "File Sha256"
        },
        "split": {
          "type": "string",
          "const": "test",
          "title": "Split"
        },
        "source_url": {
          "type": "string",
          "const": "https://huggingface.co/datasets/cardiffnlp/tweet_eval/tree/b3a375baf0f409c77e6bc7aa35102b7b3534f8be",
          "title": "Source Url"
        },
        "license_note": {
          "type": "string",
          "maxLength": 512,
          "minLength": 1,
          "title": "License Note"
        },
        "license_status": {
          "type": "string",
          "const": "recorded_not_reviewed",
          "title": "License Status"
        },
        "attribution_status": {
          "type": "string",
          "const": "review_required",
          "title": "Attribution Status"
        },
        "platform_terms_status": {
          "type": "string",
          "const": "review_required",
          "title": "Platform Terms Status"
        },
        "label_mapping": {
          "additionalProperties": {
            "type": "string",
            "enum": [
              "NEGATIVE",
              "POSITIVE"
            ]
          },
          "type": "object",
          "title": "Label Mapping"
        },
        "excluded_source_label": {
          "additionalProperties": {
            "type": "string",
            "const": "NEUTRAL"
          },
          "type": "object",
          "title": "Excluded Source Label"
        }
      },
      "additionalProperties": false,
      "type": "object",
      "required": [
        "dataset",
        "config",
        "revision",
        "file",
        "file_oid",
        "file_sha256",
        "split",
        "source_url",
        "license_note",
        "license_status",
        "attribution_status",
        "platform_terms_status",
        "label_mapping",
        "excluded_source_label"
      ],
      "title": "SentimentCandidateSource",
      "description": "Recorded source metadata for the non-publishable sentiment intake."
    },
    "SentimentCandidateStatus": {
      "properties": {
        "schema_version": {
          "type": "string",
          "const": "bay-run.canonical-evals.sentiment-candidate.v1",
          "title": "Schema Version",
          "default": "bay-run.canonical-evals.sentiment-candidate.v1"
        },
        "pin_id": {
          "type": "string",
          "const": "route_1c7472e940dc02517f5af93792bf07ee",
          "title": "Pin Id"
        },
        "route_id": {
          "type": "string",
          "const": "route_1c7472e940dc02517f5af93792bf07ee",
          "title": "Route Id"
        },
        "status": {
          "type": "string",
          "const": "candidate-unreviewed",
          "title": "Status"
        },
        "reviewed": {
          "type": "boolean",
          "const": false,
          "title": "Reviewed"
        },
        "publication_status": {
          "type": "string",
          "const": "blocked",
          "title": "Publication Status"
        },
        "case_count": {
          "type": "integer",
          "const": 40,
          "title": "Case Count"
        },
        "case_file": {
          "type": "string",
          "const": "evals/candidates/sentiment-tweeteval-test-v1.jsonl",
          "title": "Case File"
        },
        "case_file_sha256": {
          "type": "string",
          "const": "sha256:63c3962c6cc9771ce4d16f66797fa20965a26fe887d8b1d622ca05c26c30dce6",
          "title": "Case File Sha256"
        },
        "candidate_bundle_file": {
          "type": "string",
          "const": "evals/candidates/candidate-bundle.v1.json",
          "title": "Candidate Bundle File"
        },
        "candidate_bundle_sha256": {
          "type": "string",
          "const": "sha256:f7502b6f0abd907dc2ec541510eb3a4578a2d7a6e0fa5e6fb139725baea2515f",
          "title": "Candidate Bundle Sha256"
        },
        "source": {
          "$ref": "#/$defs/SentimentCandidateSource"
        },
        "missing_evidence": {
          "items": {
            "$ref": "#/$defs/SentimentMissingEvidence"
          },
          "type": "array",
          "maxItems": 16,
          "minItems": 1,
          "title": "Missing Evidence"
        },
        "next_action": {
          "type": "string",
          "const": "review_then_measure_then_prove",
          "title": "Next Action"
        }
      },
      "additionalProperties": false,
      "type": "object",
      "required": [
        "pin_id",
        "route_id",
        "status",
        "reviewed",
        "publication_status",
        "case_count",
        "case_file",
        "case_file_sha256",
        "candidate_bundle_file",
        "candidate_bundle_sha256",
        "source",
        "missing_evidence",
        "next_action"
      ],
      "title": "SentimentCandidateStatus",
      "description": "Honest public status for a sentiment candidate that is not evidence."
    },
    "SentimentMissingEvidence": {
      "properties": {
        "code": {
          "type": "string",
          "enum": [
            "rights_attribution_and_platform_terms_review",
            "case_by_case_content_scope_and_pii_review",
            "live_model_outputs_and_metric",
            "evaluation_methodology_and_timestamp",
            "served_model_weight_and_configuration_binding",
            "named_reviewer_and_release_authority_proof"
          ],
          "title": "Code"
        },
        "status": {
          "type": "string",
          "const": "missing",
          "title": "Status"
        },
        "owner": {
          "type": "string",
          "enum": [
            "human_reviewer",
            "evaluation_operator",
            "release_authority"
          ],
          "title": "Owner"
        },
        "required_action": {
          "type": "string",
          "maxLength": 768,
          "minLength": 1,
          "title": "Required Action"
        }
      },
      "additionalProperties": false,
      "type": "object",
      "required": [
        "code",
        "status",
        "owner",
        "required_action"
      ],
      "title": "SentimentMissingEvidence",
      "description": "One concrete gate still missing before sentiment can be published."
    }
  }
}