{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "urn:benchx:schema:comparison-document:0.1.0",
  "title": "BenchX comparison document",
  "description": "The comparator's output: verdicts for target results against their references, in history mode or run mode, with the evidence behind each. See docs/design/comparator.md section 5. Derived and recomputed on demand; the store keeps its inputs, not the document.",
  "$comment": "Rules the schema cannot express and the comparator enforces: inputs lists every result the document refers to; local_only is true when any input is not clean in subject_dirty or benchmark_dirty (benchmark-result-schema.md section 5.5); a unit affected by a failed invariant has no verdict and appears in no units entry; in history mode every reference result precedes the target under the recorded ordering.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "schema_version",
    "mode",
    "method",
    "estimator",
    "identity_schema",
    "local_only",
    "units",
    "missing",
    "exclusions",
    "inputs"
  ],
  "properties": {
    "schema_version": {
      "description": "Message contract version this object conforms to.",
      "const": "benchx/comparison-document/0.1.0"
    },
    "mode": {
      "description": "History mode compares a target series point against preceding points of its series; run mode compares two sides measured in one run, as a profile defines them.",
      "enum": [
        "history",
        "run"
      ]
    },
    "method": {
      "$ref": "#/$defs/method"
    },
    "estimator": {
      "description": "Estimator every compared value was computed with.",
      "$ref": "#/$defs/estimator"
    },
    "identity_schema": {
      "description": "Versioned name of the projection under which series membership and comparability were decided: for a profile, its own name and its identity policy's (benchmark-result-schema.md section 4.3).",
      "$ref": "#/$defs/token"
    },
    "local_only": {
      "description": "True when any input is not clean; such a comparison never promotes into tracked history.",
      "type": "boolean"
    },
    "run_key": {
      "description": "Run mode: the run both sides were measured in.",
      "$ref": "#/$defs/token"
    },
    "profile": {
      "description": "Run mode: the comparison profile the run was checked against (benchmark-result-schema.md section 5.5).",
      "enum": [
        "single-run",
        "variants",
        "revisions",
        "environments",
        "cross-machine"
      ]
    },
    "varying": {
      "description": "Run mode: path of the coordinate whose value distinguishes the sides, such as provenance.subject_tree or coordinates.workload.parameters.role.",
      "$ref": "#/$defs/token"
    },
    "baseline": {
      "description": "Run mode: the reference side, as its value of the varying coordinate.",
      "$ref": "#/$defs/side"
    },
    "contender": {
      "description": "Run mode: the target side. Absent when a failed invariant prevented identifying it.",
      "$ref": "#/$defs/side"
    },
    "sides": {
      "description": "Run mode: what each side declared about how it was built and entered, recorded as given and never judged, beside the environment the runner observed for it.",
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "baseline": {
          "$ref": "#/$defs/declared"
        },
        "contender": {
          "$ref": "#/$defs/declared"
        }
      }
    },
    "pairing": {
      "description": "Run mode: how the two sides' attempts are paired.",
      "$comment": "Only paired comparison is defined; comparator.md open question 2 asks whether an unpaired method should be allowed.",
      "const": "procedure.round"
    },
    "failed_invariants": {
      "description": "Run mode: profile invariants the run did not satisfy.",
      "type": "array",
      "items": {
        "$ref": "#/$defs/failedInvariant"
      }
    },
    "units": {
      "description": "One evaluation each: a series point in history mode, a workload variant and quantity in run mode.",
      "type": "array"
    },
    "missing": {
      "description": "Expected targets with no result, such as a side absent from a run or a series with no result from a batch request's target.",
      "type": "array",
      "items": {
        "$ref": "#/$defs/missing"
      }
    },
    "exclusions": {
      "description": "Results left out of the evidence, with the reason.",
      "type": "array",
      "items": {
        "$ref": "#/$defs/exclusion"
      }
    },
    "inputs": {
      "description": "Every result the comparison consumed.",
      "type": "array",
      "uniqueItems": true,
      "items": {
        "$ref": "#/$defs/resultRef"
      }
    }
  },
  "allOf": [
    {
      "if": {
        "properties": {
          "mode": {
            "const": "run"
          }
        }
      },
      "then": {
        "required": [
          "run_key",
          "profile",
          "varying",
          "baseline",
          "pairing",
          "failed_invariants"
        ],
        "properties": {
          "units": {
            "items": {
              "$ref": "#/$defs/runUnit"
            }
          }
        }
      },
      "else": {
        "properties": {
          "run_key": false,
          "profile": false,
          "varying": false,
          "baseline": false,
          "contender": false,
          "sides": false,
          "pairing": false,
          "failed_invariants": false,
          "units": {
            "items": {
              "$ref": "#/$defs/historyUnit"
            }
          }
        }
      }
    }
  ],
  "$defs": {
    "method": {
      "description": "Comparison method: a stable name and version, the effect measure it reports, and its complete configuration, such as k and minimum evidence.",
      "type": "object",
      "additionalProperties": false,
      "required": [
        "name",
        "version",
        "effect_measure",
        "configuration"
      ],
      "properties": {
        "name": {
          "$ref": "#/$defs/token"
        },
        "version": {
          "$ref": "#/$defs/token"
        },
        "effect_measure": {
          "description": "How effect is measured from the reference estimate, such as relative, absolute, ratio, or log-ratio.",
          "$ref": "#/$defs/token"
        },
        "configuration": {
          "type": "object"
        }
      }
    },
    "outcome": {
      "description": "What one evaluation found. Shared by both unit kinds.",
      "type": "object",
      "required": [
        "workload",
        "quantity",
        "verdict"
      ],
      "properties": {
        "workload": {
          "$ref": "#/$defs/workload"
        },
        "quantity": {
          "$ref": "#/$defs/token"
        },
        "reference_estimate": {
          "description": "Expected value from the reference points or the baseline side.",
          "type": "number"
        },
        "effect": {
          "description": "Change from the reference estimate, in the method's effect measure.",
          "type": "number"
        },
        "noise": {
          "description": "Variation on the effect's scale. Absent for a deterministic quantity.",
          "type": "number",
          "minimum": 0
        },
        "noise_source": {
          "description": "Where the noise came from, such as reference-points or paired-rounds.",
          "$ref": "#/$defs/token"
        },
        "provisional": {
          "description": "True when the noise came from a provisional rule rather than evidence.",
          "type": "boolean"
        },
        "threshold": {
          "description": "The effect magnitude the method required to indicate a change.",
          "type": "number",
          "minimum": 0
        },
        "verdict": {
          "enum": [
            "improved",
            "regressed",
            "changed",
            "no-change-detected",
            "indeterminate"
          ]
        },
        "reason": {
          "description": "Why the verdict is indeterminate, such as insufficient-history or duplicate-points.",
          "$ref": "#/$defs/token"
        },
        "unmatched_differences": {
          "description": "Coordinates the projection does not match, by JSON Pointer, whose values differ between target and reference.",
          "type": "array",
          "uniqueItems": true,
          "items": {
            "$ref": "#/$defs/token"
          }
        },
        "caveats": {
          "description": "Quality caveats the reader should see.",
          "type": "array",
          "items": {
            "$ref": "#/$defs/nonEmptyString"
          }
        }
      },
      "if": {
        "properties": {
          "verdict": {
            "const": "indeterminate"
          }
        }
      },
      "then": {
        "required": [
          "reason"
        ]
      },
      "else": {
        "required": [
          "reference_estimate",
          "effect"
        ]
      }
    },
    "runUnit": {
      "description": "One workload variant and quantity, its two sides paired by round.",
      "allOf": [
        {
          "$ref": "#/$defs/outcome"
        }
      ],
      "unevaluatedProperties": false,
      "required": [
        "rounds"
      ],
      "properties": {
        "rounds": {
          "description": "Paired rounds used as evidence.",
          "type": "integer",
          "minimum": 0
        },
        "series": {
          "description": "The series each side's results belong to, when the store assigns them one. Run mode reads no history from them.",
          "type": "object",
          "additionalProperties": false,
          "required": [
            "baseline",
            "contender"
          ],
          "properties": {
            "baseline": {
              "$ref": "#/$defs/sha256"
            },
            "contender": {
              "$ref": "#/$defs/sha256"
            }
          }
        }
      }
    },
    "historyUnit": {
      "description": "One target series point against preceding points of its series.",
      "allOf": [
        {
          "$ref": "#/$defs/outcome"
        }
      ],
      "unevaluatedProperties": false,
      "required": [
        "series",
        "target",
        "reference"
      ],
      "properties": {
        "series": {
          "description": "Series fingerprint of the target and every reference point.",
          "$ref": "#/$defs/sha256"
        },
        "target": {
          "$ref": "#/$defs/resultRef"
        },
        "reference": {
          "description": "The reference points the reference estimate was formed from, oldest first.",
          "type": "array",
          "items": {
            "$ref": "#/$defs/resultRef"
          }
        },
        "ordering": {
          "description": "How the reference points were ordered before the target. Absent when no order was available.",
          "enum": [
            "ancestry",
            "native-order",
            "native-time"
          ]
        },
        "boundaries": {
          "description": "Resets, change points, and continuity joins that truncated or joined the reference.",
          "type": "array",
          "items": {
            "$ref": "#/$defs/boundary"
          }
        }
      }
    },
    "boundary": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "kind"
      ],
      "properties": {
        "kind": {
          "enum": [
            "distribution-reset",
            "change-point",
            "continuity"
          ]
        },
        "at": {
          "description": "First result after the boundary.",
          "$ref": "#/$defs/resultRef"
        },
        "detail": {
          "$ref": "#/$defs/nonEmptyString"
        }
      }
    },
    "side": {
      "description": "A value of the varying coordinate, such as a tree id, a label, or a parameter value.",
      "not": {
        "type": "null"
      }
    },
    "declared": {
      "description": "One side's declared text, copied from its results' provenance.info, and the allowlisted environment variables observed for it.",
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "how_built": {
          "$ref": "#/$defs/nonEmptyString"
        },
        "activation": {
          "$ref": "#/$defs/nonEmptyString"
        },
        "shell": {
          "$ref": "#/$defs/token"
        },
        "target_provider": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "name"
          ],
          "properties": {
            "name": {
              "$ref": "#/$defs/nonEmptyString"
            },
            "version": {
              "$ref": "#/$defs/nonEmptyString"
            }
          }
        },
        "observed_env": {
          "type": "object",
          "additionalProperties": {
            "type": "string"
          }
        }
      }
    },
    "failedInvariant": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "invariant",
        "detail"
      ],
      "properties": {
        "invariant": {
          "description": "Name of the invariant, such as alternating-slots or equal-attempts.",
          "$ref": "#/$defs/token"
        },
        "detail": {
          "$ref": "#/$defs/nonEmptyString"
        },
        "workload": {
          "description": "The unit it affects, when it affects only one.",
          "$ref": "#/$defs/workload"
        },
        "quantity": {
          "$ref": "#/$defs/token"
        }
      }
    },
    "missing": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "workload",
        "quantity"
      ],
      "properties": {
        "workload": {
          "$ref": "#/$defs/workload"
        },
        "quantity": {
          "$ref": "#/$defs/token"
        },
        "side": {
          "description": "Run mode: the side with no result.",
          "enum": [
            "baseline",
            "contender"
          ]
        },
        "series": {
          "description": "History mode: the expected series.",
          "$ref": "#/$defs/sha256"
        }
      }
    },
    "exclusion": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "result",
        "reason"
      ],
      "properties": {
        "result": {
          "$ref": "#/$defs/resultRef"
        },
        "reason": {
          "$ref": "#/$defs/nonEmptyString"
        }
      }
    },
    "workload": {
      "description": "Workload variant: a named operation plus one parameter assignment.",
      "type": "object",
      "additionalProperties": false,
      "required": [
        "name"
      ],
      "properties": {
        "name": {
          "$ref": "#/$defs/token"
        },
        "parameters": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/parameterValue"
          }
        }
      }
    },
    "estimator": {
      "description": "Estimator declaration, as in the measurement-result schema.",
      "type": "object",
      "additionalProperties": false,
      "required": [
        "name",
        "method_version",
        "input_level"
      ],
      "properties": {
        "name": {
          "$ref": "#/$defs/token"
        },
        "method_version": {
          "$ref": "#/$defs/token"
        },
        "input_level": {
          "enum": [
            "observations",
            "estimates"
          ]
        },
        "parameters": {
          "type": "object"
        },
        "inputs": {
          "type": "array",
          "minItems": 1,
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "role",
              "quantity"
            ],
            "properties": {
              "role": {
                "$ref": "#/$defs/token"
              },
              "quantity": {
                "$ref": "#/$defs/token"
              }
            }
          }
        }
      }
    },
    "resultRef": {
      "description": "Stable reference to a stored result.",
      "type": "object",
      "additionalProperties": false,
      "required": [
        "producer",
        "ingest_key"
      ],
      "properties": {
        "producer": {
          "$ref": "#/$defs/token"
        },
        "ingest_key": {
          "$ref": "#/$defs/token"
        }
      }
    },
    "parameterValue": {
      "oneOf": [
        {
          "$ref": "#/$defs/primitive"
        },
        {
          "type": "array",
          "items": {
            "$ref": "#/$defs/primitive"
          }
        }
      ]
    },
    "primitive": {
      "type": [
        "string",
        "number",
        "boolean",
        "null"
      ]
    },
    "sha256": {
      "type": "string",
      "pattern": "^[0-9a-f]{64}$"
    },
    "token": {
      "description": "Non-empty string without whitespace.",
      "type": "string",
      "minLength": 1,
      "pattern": "^\\S+$"
    },
    "nonEmptyString": {
      "type": "string",
      "minLength": 1
    }
  }
}