243 lines
8.8 KiB
Python
243 lines
8.8 KiB
Python
|
"""FEATURE_PARITY_HARNESS_V1 contracts; formula ports are deliberately external."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
from collections.abc import Callable, Iterable, Mapping
|
||
|
|
from dataclasses import asdict, dataclass, replace
|
||
|
|
from enum import StrEnum
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[3]
|
||
|
|
FROZEN_ENGINEERING_MAP_V1 = ROOT / "archaeology" / "engineering_family_map_v1.json"
|
||
|
|
FROZEN_ENGINEERING_MAP_V1_SHA256 = (
|
||
|
|
"314a87ca468dd80e6f8bf1eaaa3e6350771ad5f53ee4f3f80db7c8949b2de1ab"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class ParityStatus(StrEnum):
|
||
|
|
PASS = "pass"
|
||
|
|
FAIL = "fail"
|
||
|
|
NOT_IMPLEMENTED = "not_implemented"
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True, slots=True)
|
||
|
|
class FeatureVersion:
|
||
|
|
"""One immutable implementation candidate in a parity run.
|
||
|
|
|
||
|
|
Coverage is intentionally calculated from the versions submitted to one
|
||
|
|
run, never from the historical registry or an implied implementation pool.
|
||
|
|
"""
|
||
|
|
|
||
|
|
feature_version: str
|
||
|
|
primitive_key: str
|
||
|
|
engineering_family: str
|
||
|
|
usage_slots: int
|
||
|
|
evaluator: Callable[..., np.ndarray] | None = None
|
||
|
|
status: ParityStatus = ParityStatus.NOT_IMPLEMENTED
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True, slots=True)
|
||
|
|
class ComparisonRecord:
|
||
|
|
name: str
|
||
|
|
status: ParityStatus
|
||
|
|
expected_dtype: str
|
||
|
|
actual_dtype: str
|
||
|
|
expected_shape: tuple[int, ...]
|
||
|
|
actual_shape: tuple[int, ...]
|
||
|
|
nan_mask_mismatches: int
|
||
|
|
value_mismatches: int
|
||
|
|
reason: str | None = None
|
||
|
|
|
||
|
|
@property
|
||
|
|
def passed(self) -> bool:
|
||
|
|
return self.status == ParityStatus.PASS
|
||
|
|
|
||
|
|
def artifact(self) -> dict[str, object]:
|
||
|
|
return asdict(self)
|
||
|
|
|
||
|
|
|
||
|
|
class FrozenArtifactError(ValueError):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def _digest(path: Path) -> str:
|
||
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def load_frozen_engineering_map(path: Path = FROZEN_ENGINEERING_MAP_V1) -> dict[str, object]:
|
||
|
|
"""Load only the pinned engineering map; reject substituted archaeology."""
|
||
|
|
if _digest(path) != FROZEN_ENGINEERING_MAP_V1_SHA256:
|
||
|
|
raise FrozenArtifactError(
|
||
|
|
"ENGINEERING_FAMILY_MAP_V1 digest does not match the frozen artifact"
|
||
|
|
)
|
||
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
|
|
if payload.get("artifact") != "ENGINEERING_FAMILY_MAP_V1" or payload.get("schema_version") != 1:
|
||
|
|
raise FrozenArtifactError("invalid ENGINEERING_FAMILY_MAP_V1 manifest")
|
||
|
|
return payload
|
||
|
|
|
||
|
|
|
||
|
|
def parity_manifest() -> dict[str, object]:
|
||
|
|
"""Return the reproducible run contract without claiming any formula parity."""
|
||
|
|
engineering_map = load_frozen_engineering_map()
|
||
|
|
return {
|
||
|
|
"schema_version": 1,
|
||
|
|
"artifact": "FEATURE_PARITY_HARNESS_V1_MANIFEST",
|
||
|
|
"frozen_inputs": {
|
||
|
|
"engineering_family_map": {
|
||
|
|
"path": "archaeology/engineering_family_map_v1.json",
|
||
|
|
"sha256": FROZEN_ENGINEERING_MAP_V1_SHA256,
|
||
|
|
"primitive_count": engineering_map["counts"]["primitives"],
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"oracle_windows": {"execution": "spark_offline", "network_access": "forbidden"},
|
||
|
|
"comparison": {
|
||
|
|
"dtype": "exact",
|
||
|
|
"shape": "exact",
|
||
|
|
"nan_mask": "exact",
|
||
|
|
"state_keys": "exact",
|
||
|
|
"tolerance_by_engineering_family": {},
|
||
|
|
"unknown_family_policy": "fail_closed_no_tolerance",
|
||
|
|
},
|
||
|
|
"coverage": "recomputed solely from submitted FeatureVersions with status=pass",
|
||
|
|
"scope": "architecture only; no new formula ports",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def enumerate_native_evaluators(
|
||
|
|
feature_versions: Iterable[FeatureVersion],
|
||
|
|
) -> tuple[FeatureVersion, ...]:
|
||
|
|
"""Return parameterized native evaluators and reject duplicate version IDs."""
|
||
|
|
versions = tuple(feature_versions)
|
||
|
|
ids = [item.feature_version for item in versions]
|
||
|
|
if len(ids) != len(set(ids)):
|
||
|
|
raise ValueError("FeatureVersion identifiers must be unique")
|
||
|
|
return tuple(item for item in versions if item.evaluator is not None)
|
||
|
|
|
||
|
|
|
||
|
|
def compare_array(name: str, expected: Any, actual: Any) -> ComparisonRecord:
|
||
|
|
"""Compare a feature output without numerical tolerance or dtype coercion."""
|
||
|
|
expected_array, actual_array = np.asarray(expected), np.asarray(actual)
|
||
|
|
dtype_match = expected_array.dtype == actual_array.dtype
|
||
|
|
shape_match = expected_array.shape == actual_array.shape
|
||
|
|
if not shape_match:
|
||
|
|
return ComparisonRecord(
|
||
|
|
name, ParityStatus.FAIL, str(expected_array.dtype), str(actual_array.dtype),
|
||
|
|
expected_array.shape, actual_array.shape, 0, 0, "shape_mismatch",
|
||
|
|
)
|
||
|
|
expected_nan = (
|
||
|
|
np.isnan(expected_array)
|
||
|
|
if np.issubdtype(expected_array.dtype, np.inexact)
|
||
|
|
else np.zeros(expected_array.shape, bool)
|
||
|
|
)
|
||
|
|
actual_nan = (
|
||
|
|
np.isnan(actual_array)
|
||
|
|
if np.issubdtype(actual_array.dtype, np.inexact)
|
||
|
|
else np.zeros(actual_array.shape, bool)
|
||
|
|
)
|
||
|
|
nan_mismatches = int(np.count_nonzero(expected_nan != actual_nan))
|
||
|
|
values_match = dtype_match and np.array_equal(expected_array, actual_array, equal_nan=True)
|
||
|
|
matching_values = (expected_array == actual_array) | (expected_nan & actual_nan)
|
||
|
|
value_mismatches = (
|
||
|
|
0
|
||
|
|
if values_match
|
||
|
|
else int(expected_array.size - np.count_nonzero(matching_values))
|
||
|
|
)
|
||
|
|
return ComparisonRecord(
|
||
|
|
name, ParityStatus.PASS if values_match else ParityStatus.FAIL,
|
||
|
|
str(expected_array.dtype),
|
||
|
|
str(actual_array.dtype),
|
||
|
|
expected_array.shape,
|
||
|
|
actual_array.shape,
|
||
|
|
nan_mismatches,
|
||
|
|
value_mismatches,
|
||
|
|
None if values_match else "dtype_or_value_mismatch",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def compare_float64_ulp(
|
||
|
|
name: str,
|
||
|
|
expected: Any,
|
||
|
|
actual: Any,
|
||
|
|
*,
|
||
|
|
max_ulps: int,
|
||
|
|
allowed_indexes: frozenset[int],
|
||
|
|
) -> ComparisonRecord:
|
||
|
|
"""Accept only explicitly listed finite float64 differences within a ULP bound."""
|
||
|
|
exact = compare_array(name, expected, actual)
|
||
|
|
if exact.passed:
|
||
|
|
return exact
|
||
|
|
expected_array, actual_array = np.asarray(expected), np.asarray(actual)
|
||
|
|
if (
|
||
|
|
expected_array.dtype != np.dtype(np.float64)
|
||
|
|
or actual_array.dtype != np.dtype(np.float64)
|
||
|
|
or expected_array.shape != actual_array.shape
|
||
|
|
or exact.nan_mask_mismatches
|
||
|
|
):
|
||
|
|
return exact
|
||
|
|
mismatches = np.flatnonzero(
|
||
|
|
~((expected_array == actual_array) | (np.isnan(expected_array) & np.isnan(actual_array)))
|
||
|
|
)
|
||
|
|
if (
|
||
|
|
not len(mismatches)
|
||
|
|
or not np.all(np.isfinite(expected_array[mismatches]))
|
||
|
|
or not np.all(np.isfinite(actual_array[mismatches]))
|
||
|
|
or any(int(index) not in allowed_indexes for index in mismatches)
|
||
|
|
):
|
||
|
|
return exact
|
||
|
|
for index in mismatches:
|
||
|
|
expected_bits = int(np.asarray(expected_array[index]).view(np.uint64))
|
||
|
|
actual_bits = int(np.asarray(actual_array[index]).view(np.uint64))
|
||
|
|
expected_ordered = _ordered_float64_bits(expected_bits)
|
||
|
|
actual_ordered = _ordered_float64_bits(actual_bits)
|
||
|
|
if abs(expected_ordered - actual_ordered) > max_ulps:
|
||
|
|
return exact
|
||
|
|
return replace(exact, status=ParityStatus.PASS, reason="accepted_documented_float64_ulp_drift")
|
||
|
|
|
||
|
|
|
||
|
|
def _ordered_float64_bits(bits: int) -> int:
|
||
|
|
return ~bits & ((1 << 64) - 1) if bits >> 63 else bits | (1 << 63)
|
||
|
|
|
||
|
|
|
||
|
|
def compare_state(
|
||
|
|
expected: Mapping[str, Any], actual: Mapping[str, Any],
|
||
|
|
) -> tuple[ComparisonRecord, ...]:
|
||
|
|
"""Produce auditable exact records for every state key, including missing keys."""
|
||
|
|
records: list[ComparisonRecord] = []
|
||
|
|
for name in sorted(set(expected) | set(actual)):
|
||
|
|
if name not in expected or name not in actual:
|
||
|
|
missing = expected.get(name, actual.get(name))
|
||
|
|
array = np.asarray(missing)
|
||
|
|
records.append(ComparisonRecord(
|
||
|
|
name, ParityStatus.FAIL, str(array.dtype) if name in expected else "<missing>",
|
||
|
|
str(array.dtype) if name in actual else "<missing>",
|
||
|
|
array.shape if name in expected else (),
|
||
|
|
array.shape if name in actual else (),
|
||
|
|
0,
|
||
|
|
0,
|
||
|
|
"state_key_mismatch",
|
||
|
|
))
|
||
|
|
else:
|
||
|
|
records.append(compare_array(name, expected[name], actual[name]))
|
||
|
|
return tuple(records)
|
||
|
|
|
||
|
|
|
||
|
|
def recompute_coverage(feature_versions: Iterable[FeatureVersion]) -> dict[str, object]:
|
||
|
|
"""Compute coverage only over the explicitly submitted FeatureVersions."""
|
||
|
|
versions = tuple(feature_versions)
|
||
|
|
total = sum(item.usage_slots for item in versions)
|
||
|
|
passed = tuple(item for item in versions if item.status == ParityStatus.PASS)
|
||
|
|
passed_slots = sum(item.usage_slots for item in passed)
|
||
|
|
return {
|
||
|
|
"submitted_feature_versions": len(versions),
|
||
|
|
"passed_feature_versions": len(passed),
|
||
|
|
"submitted_usage_slots": total,
|
||
|
|
"passed_usage_slots": passed_slots,
|
||
|
|
"coverage_percent": 0.0 if total == 0 else passed_slots * 100 / total,
|
||
|
|
"feature_versions": [item.feature_version for item in passed],
|
||
|
|
}
|