{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://kaimahi.dev/schemas/agentsuite/v1/eval-evidence.schema.json",
  "title": "AgentSuite unsigned public evaluation evidence",
  "type": "object",
  "additionalProperties": false,
  "required": ["schemaVersion", "mediaType", "subject", "execution", "evaluation", "evaluator", "results"],
  "properties": {
    "schemaVersion": {"const": "1.0.0"},
    "mediaType": {"const": "application/vnd.agentsuite.eval.evidence.v1+json"},
    "subject": {"$ref": "#/$defs/subject"},
    "execution": {"$ref": "#/$defs/execution"},
    "evaluation": {"$ref": "#/$defs/evaluation"},
    "evaluator": {"$ref": "#/$defs/evaluator"},
    "results": {"type": "array", "maxItems": 128, "items": {"$ref": "#/$defs/caseResult"}}
  },
  "$defs": {
    "token": {
      "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
      "not": {"anyOf": [
        {"pattern": "^([sS][kK][-_]|[gG][hH][pPoOuUsSrR]_|[gG][iI][tT][hH][uU][bB]_[pP][aA][tT]_|[aA][kK][iI][aA]|[aA][sS][iI][aA]|[eE][yY][jJ])"},
        {"pattern": "^[A-Za-z0-9-]+(\\.[A-Za-z0-9-]+)*\\.[A-Za-z]{2,}$"},
        {"pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$"}
      ]}
    },
    "caseID": {"allOf": [{"$ref": "#/$defs/token"}, {"maxLength": 63}]},
    "digest": {"type": "string", "pattern": "^sha256:[a-f0-9]{64}$"},
    "outcome": {"enum": ["pass", "fail", "unknown"]},
    "time": {"type": "string", "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]{1,9})?Z$"},
    "descriptor": {
      "type": "object", "additionalProperties": false, "required": ["mediaType", "digest", "size"],
      "properties": {
        "mediaType": {"enum": ["application/vnd.oci.image.manifest.v1+json", "application/vnd.agentsuite.eval.index.v1+json", "application/vnd.agentsuite.eval.case.v1+json"]},
        "digest": {"$ref": "#/$defs/digest"}, "size": {"type": "integer", "minimum": 1, "maximum": 4194304}
      }
    },
    "manifestDescriptor": {"allOf": [{"$ref": "#/$defs/descriptor"}, {"properties": {"mediaType": {"const": "application/vnd.oci.image.manifest.v1+json"}}}]},
    "indexDescriptor": {"allOf": [{"$ref": "#/$defs/descriptor"}, {"properties": {"mediaType": {"const": "application/vnd.agentsuite.eval.index.v1+json"}}}]},
    "caseDescriptor": {"allOf": [{"$ref": "#/$defs/descriptor"}, {"properties": {"mediaType": {"const": "application/vnd.agentsuite.eval.case.v1+json"}}}]},
    "platform": {
      "type": "object", "additionalProperties": false, "required": ["os", "architecture", "variant"],
      "properties": {"os": {"$ref": "#/$defs/token"}, "architecture": {"$ref": "#/$defs/token"}, "variant": {"anyOf": [{"const": ""}, {"$ref": "#/$defs/token"}]}}
    },
    "subject": {
      "type": "object", "additionalProperties": false,
      "required": ["manifest", "memberID", "memberDigest", "compositionDigest", "buildProfileDigest", "platform"],
      "properties": {
        "manifest": {"$ref": "#/$defs/manifestDescriptor"}, "memberID": {"$ref": "#/$defs/token"},
        "memberDigest": {"$ref": "#/$defs/digest"}, "compositionDigest": {"$ref": "#/$defs/digest"},
        "buildProfileDigest": {"$ref": "#/$defs/digest"}, "platform": {"$ref": "#/$defs/platform"}
      }
    },
    "implementation": {
      "type": "object", "additionalProperties": false, "required": ["id", "version", "digest"],
      "properties": {"id": {"$ref": "#/$defs/token"}, "version": {"$ref": "#/$defs/token"}, "digest": {"$ref": "#/$defs/digest"}}
    },
    "model": {
      "type": "object", "additionalProperties": false, "required": ["protocol", "id", "configurationDigest"],
      "properties": {"protocol": {"$ref": "#/$defs/token"}, "id": {"$ref": "#/$defs/token"}, "configurationDigest": {"$ref": "#/$defs/digest"}}
    },
    "execution": {
      "type": "object", "additionalProperties": false, "required": ["runtime", "model", "harness", "renderer", "profile", "configurationDigest"],
      "properties": {
        "runtime": {"$ref": "#/$defs/implementation"}, "model": {"$ref": "#/$defs/model"}, "harness": {"$ref": "#/$defs/implementation"},
        "renderer": {"$ref": "#/$defs/implementation"}, "profile": {"$ref": "#/$defs/implementation"}, "configurationDigest": {"$ref": "#/$defs/digest"}
      }
    },
    "policy": {
      "type": "object", "additionalProperties": false, "required": ["id", "version", "configurationDigest"],
      "properties": {"id": {"$ref": "#/$defs/token"}, "version": {"$ref": "#/$defs/token"}, "configurationDigest": {"$ref": "#/$defs/digest"}}
    },
    "evaluatorIdentity": {
      "type": "object", "additionalProperties": false, "required": ["id", "version", "policy"],
      "properties": {"id": {"$ref": "#/$defs/token"}, "version": {"$ref": "#/$defs/token"}, "policy": {"$ref": "#/$defs/policy"}}
    },
    "evaluator": {
      "type": "object", "additionalProperties": false, "required": ["identity", "startedAt", "completedAt"],
      "properties": {"identity": {"$ref": "#/$defs/evaluatorIdentity"}, "startedAt": {"$ref": "#/$defs/time"}, "completedAt": {"$ref": "#/$defs/time"}}
    },
    "indexPath": {
      "type": "string", "pattern": "^evals/[A-Za-z0-9][A-Za-z0-9._-]{0,127}/index\\.json$",
      "not": {"anyOf": [
        {"pattern": "^evals/([sS][kK][-_]|[gG][hH][pPoOuUsSrR]_|[gG][iI][tT][hH][uU][bB]_[pP][aA][tT]_|[aA][kK][iI][aA]|[aA][sS][iI][aA]|[eE][yY][jJ])"},
        {"pattern": "^evals/[A-Za-z0-9-]+(\\.[A-Za-z0-9-]+)*\\.[A-Za-z]{2,}/"},
        {"pattern": "^evals/[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/"}
      ]}
    },
    "casePath": {
      "type": "string", "pattern": "^evals/[A-Za-z0-9][A-Za-z0-9._-]{0,127}/cases/[A-Za-z0-9][A-Za-z0-9._-]{0,127}\\.json$",
      "not": {"anyOf": [
        {"pattern": "^evals/([sS][kK][-_]|[gG][hH][pPoOuUsSrR]_|[gG][iI][tT][hH][uU][bB]_[pP][aA][tT]_|[aA][kK][iI][aA]|[aA][sS][iI][aA]|[eE][yY][jJ])"},
        {"pattern": "^evals/[A-Za-z0-9-]+(\\.[A-Za-z0-9-]+)*\\.[A-Za-z]{2,}/"},
        {"pattern": "^evals/[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+/"},
        {"pattern": "^evals/[^/]+/cases/([sS][kK][-_]|[gG][hH][pPoOuUsSrR]_|[gG][iI][tT][hH][uU][bB]_[pP][aA][tT]_|[aA][kK][iI][aA]|[aA][sS][iI][aA]|[eE][yY][jJ])"},
        {"pattern": "^evals/[^/]+/cases/[A-Za-z0-9-]+(\\.[A-Za-z0-9-]+)*\\.[A-Za-z]{2,}\\.json$"},
        {"pattern": "^evals/[^/]+/cases/[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\.json$"}
      ]}
    },
    "index": {
      "type": "object", "additionalProperties": false, "required": ["path", "descriptor"],
      "properties": {"path": {"$ref": "#/$defs/indexPath"}, "descriptor": {"$ref": "#/$defs/indexDescriptor"}}
    },
    "caseRef": {
      "type": "object", "additionalProperties": false, "required": ["id", "path", "descriptor", "runs", "minPasses"],
      "properties": {
        "id": {"$ref": "#/$defs/caseID"}, "path": {"$ref": "#/$defs/casePath"},
        "descriptor": {"$ref": "#/$defs/caseDescriptor"}, "runs": {"type": "integer", "minimum": 1, "maximum": 1000}, "minPasses": {"type": "integer", "minimum": 1, "maximum": 1000}
      }
    },
    "evaluation": {
      "type": "object", "additionalProperties": false, "required": ["index", "cases"],
      "properties": {"index": {"$ref": "#/$defs/index"}, "cases": {"type": "array", "maxItems": 128, "items": {"$ref": "#/$defs/caseRef"}}}
    },
    "caseResult": {
      "type": "object", "additionalProperties": false, "required": ["id", "outcome", "runs"],
      "properties": {"id": {"$ref": "#/$defs/caseID"}, "outcome": {"$ref": "#/$defs/outcome"}, "runs": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/run"}}}
    },
    "run": {
      "type": "object", "additionalProperties": false, "required": ["number", "status", "outcome", "assertions"],
      "properties": {
        "number": {"type": "integer", "minimum": 1, "maximum": 1000}, "status": {"enum": ["completed", "error", "cancelled", "unknown"]},
        "outcome": {"$ref": "#/$defs/outcome"}, "assertions": {"type": "array", "maxItems": 1000, "items": {"$ref": "#/$defs/assertion"}}
      }
    },
    "assertion": {
      "type": "object", "additionalProperties": false, "required": ["id", "type", "outcome"],
      "properties": {"id": {"$ref": "#/$defs/caseID"}, "type": {"$ref": "#/$defs/token"}, "outcome": {"$ref": "#/$defs/outcome"}, "judge": {"$ref": "#/$defs/judge"}},
      "if": {"properties": {"type": {"const": "llmJudge"}}}, "else": {"properties": {"judge": false}}
    },
    "judgeConfiguration": {
      "type": "object", "additionalProperties": false, "required": ["model", "version", "policy"],
      "properties": {"model": {"$ref": "#/$defs/token"}, "version": {"$ref": "#/$defs/token"}, "policy": {"$ref": "#/$defs/policy"}}
    },
    "judge": {
      "type": "object", "additionalProperties": false, "required": ["configuration", "observedModels", "scores", "minScore"],
      "properties": {
        "configuration": {"$ref": "#/$defs/judgeConfiguration"}, "minScore": {"type": "integer", "minimum": 0, "maximum": 100},
        "scores": {"type": "array", "items": {"type": "integer", "minimum": 0, "maximum": 100}},
        "observedModels": {"type": "array", "items": {"$ref": "#/$defs/token"}}
      },
      "oneOf": [
        {"properties": {"scores": {"maxItems": 0}, "observedModels": {"maxItems": 0}}},
        {"properties": {"scores": {"minItems": 2, "maxItems": 2}, "observedModels": {"minItems": 2, "maxItems": 2}}}
      ]
    }
  }
}