235 lines
8.3 KiB
Python
235 lines
8.3 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Validate research/experiment_registry.json without external dependencies."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
from collections import Counter
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
|
||
|
|
REQUIRED_STATUSES = {
|
||
|
|
"production_accepted",
|
||
|
|
"research_retained",
|
||
|
|
"architecture_rejected",
|
||
|
|
"performance_rejected",
|
||
|
|
"quality_rejected",
|
||
|
|
"temporarily_blocked",
|
||
|
|
"incomplete",
|
||
|
|
}
|
||
|
|
|
||
|
|
REQUIRED_FIELDS = {
|
||
|
|
"id",
|
||
|
|
"name",
|
||
|
|
"family",
|
||
|
|
"status",
|
||
|
|
"hypothesis",
|
||
|
|
"implementation_strategy",
|
||
|
|
"source_locations",
|
||
|
|
"active_source_location",
|
||
|
|
"commit_hash",
|
||
|
|
"benchmark_artifacts",
|
||
|
|
"profiler_artifacts",
|
||
|
|
"environment",
|
||
|
|
"metrics",
|
||
|
|
"correctness_evidence",
|
||
|
|
"decision_rationale",
|
||
|
|
"reproducer_commands",
|
||
|
|
"timestamp",
|
||
|
|
"evidence_missing",
|
||
|
|
"production_behavior",
|
||
|
|
"source_recovery",
|
||
|
|
}
|
||
|
|
|
||
|
|
NONEMPTY_STRINGS = {
|
||
|
|
"id",
|
||
|
|
"name",
|
||
|
|
"family",
|
||
|
|
"status",
|
||
|
|
"hypothesis",
|
||
|
|
"implementation_strategy",
|
||
|
|
"decision_rationale",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def type_name(value: Any) -> str:
|
||
|
|
if value is None:
|
||
|
|
return "null"
|
||
|
|
if isinstance(value, bool):
|
||
|
|
return "boolean"
|
||
|
|
if isinstance(value, dict):
|
||
|
|
return "object"
|
||
|
|
if isinstance(value, list):
|
||
|
|
return "array"
|
||
|
|
return type(value).__name__
|
||
|
|
|
||
|
|
|
||
|
|
def validate_path_metadata(value: Any, location: str, errors: list[str]) -> None:
|
||
|
|
if not isinstance(value, dict):
|
||
|
|
errors.append(f"{location}: expected path metadata object")
|
||
|
|
return
|
||
|
|
path = value.get("path")
|
||
|
|
exists = value.get("exists")
|
||
|
|
if not isinstance(path, str) or not path.strip():
|
||
|
|
errors.append(f"{location}.path: expected non-empty string")
|
||
|
|
if not isinstance(exists, bool):
|
||
|
|
errors.append(f"{location}.exists: expected boolean")
|
||
|
|
|
||
|
|
|
||
|
|
def validate_path_array(value: Any, location: str, errors: list[str]) -> None:
|
||
|
|
if not isinstance(value, list):
|
||
|
|
errors.append(f"{location}: expected array, got {type_name(value)}")
|
||
|
|
return
|
||
|
|
for index, item in enumerate(value):
|
||
|
|
validate_path_metadata(item, f"{location}[{index}]", errors)
|
||
|
|
|
||
|
|
|
||
|
|
def validate_source_array(value: Any, location: str, errors: list[str]) -> None:
|
||
|
|
validate_string_array(value, location, errors)
|
||
|
|
|
||
|
|
|
||
|
|
def validate_string_array(value: Any, location: str, errors: list[str]) -> None:
|
||
|
|
if not isinstance(value, list):
|
||
|
|
errors.append(f"{location}: expected array, got {type_name(value)}")
|
||
|
|
return
|
||
|
|
for index, item in enumerate(value):
|
||
|
|
if not isinstance(item, str) or not item.strip():
|
||
|
|
errors.append(f"{location}[{index}]: expected non-empty string")
|
||
|
|
|
||
|
|
|
||
|
|
def validate_record(record: Any, index: int, seen_ids: set[str], errors: list[str]) -> None:
|
||
|
|
location = f"experiments[{index}]"
|
||
|
|
if not isinstance(record, dict):
|
||
|
|
errors.append(f"{location}: expected object, got {type_name(record)}")
|
||
|
|
return
|
||
|
|
|
||
|
|
missing = sorted(REQUIRED_FIELDS - record.keys())
|
||
|
|
if missing:
|
||
|
|
errors.append(f"{location}: missing required fields: {', '.join(missing)}")
|
||
|
|
|
||
|
|
for field in NONEMPTY_STRINGS:
|
||
|
|
if field not in record:
|
||
|
|
continue
|
||
|
|
value = record[field]
|
||
|
|
if not isinstance(value, str) or not value.strip():
|
||
|
|
errors.append(f"{location}.{field}: expected non-empty string")
|
||
|
|
|
||
|
|
experiment_id = record.get("id")
|
||
|
|
if isinstance(experiment_id, str) and experiment_id.strip():
|
||
|
|
if experiment_id in seen_ids:
|
||
|
|
errors.append(f"{location}.id: duplicate id {experiment_id!r}")
|
||
|
|
seen_ids.add(experiment_id)
|
||
|
|
|
||
|
|
status = record.get("status")
|
||
|
|
if isinstance(status, str) and status not in REQUIRED_STATUSES:
|
||
|
|
errors.append(f"{location}.status: unknown status {status!r}")
|
||
|
|
|
||
|
|
if "source_locations" in record:
|
||
|
|
validate_source_array(record["source_locations"], f"{location}.source_locations", errors)
|
||
|
|
if "active_source_location" in record and record["active_source_location"] is not None:
|
||
|
|
value = record["active_source_location"]
|
||
|
|
if not isinstance(value, str) or not value.strip():
|
||
|
|
errors.append(f"{location}.active_source_location: expected non-empty string or null")
|
||
|
|
for field in ("benchmark_artifacts", "profiler_artifacts"):
|
||
|
|
if field in record:
|
||
|
|
validate_path_array(record[field], f"{location}.{field}", errors)
|
||
|
|
|
||
|
|
if "commit_hash" in record and record["commit_hash"] is not None:
|
||
|
|
if not isinstance(record["commit_hash"], str) or not record["commit_hash"].strip():
|
||
|
|
errors.append(f"{location}.commit_hash: expected non-empty string or null")
|
||
|
|
|
||
|
|
for field in ("environment", "metrics"):
|
||
|
|
if field in record and not isinstance(record[field], dict):
|
||
|
|
errors.append(f"{location}.{field}: expected object, got {type_name(record[field])}")
|
||
|
|
|
||
|
|
if "timestamp" in record and record["timestamp"] is not None:
|
||
|
|
if not isinstance(record["timestamp"], str) or not record["timestamp"].strip():
|
||
|
|
errors.append(f"{location}.timestamp: expected non-empty string or null")
|
||
|
|
|
||
|
|
if "source_recovery" in record:
|
||
|
|
value = record["source_recovery"]
|
||
|
|
if not isinstance(value, str) or not value.strip():
|
||
|
|
errors.append(f"{location}.source_recovery: expected non-empty string")
|
||
|
|
|
||
|
|
for field in ("correctness_evidence", "production_behavior"):
|
||
|
|
if field in record and record[field] is not None and not isinstance(
|
||
|
|
record[field], (dict, list, str)
|
||
|
|
):
|
||
|
|
errors.append(f"{location}.{field}: expected object, array, string, or null")
|
||
|
|
|
||
|
|
if "reproducer_commands" in record:
|
||
|
|
validate_string_array(record["reproducer_commands"], f"{location}.reproducer_commands", errors)
|
||
|
|
|
||
|
|
if "evidence_missing" in record:
|
||
|
|
value = record["evidence_missing"]
|
||
|
|
if not isinstance(value, (bool, list)):
|
||
|
|
errors.append(f"{location}.evidence_missing: expected boolean or array")
|
||
|
|
elif isinstance(value, list):
|
||
|
|
validate_string_array(value, f"{location}.evidence_missing", errors)
|
||
|
|
|
||
|
|
|
||
|
|
def validate_registry(data: Any) -> tuple[list[str], Counter[str]]:
|
||
|
|
errors: list[str] = []
|
||
|
|
counts: Counter[str] = Counter()
|
||
|
|
if not isinstance(data, dict):
|
||
|
|
return [f"top level: expected object, got {type_name(data)}"], counts
|
||
|
|
|
||
|
|
allowed = data.get("allowed_statuses")
|
||
|
|
if not isinstance(allowed, list) or any(not isinstance(item, str) for item in allowed):
|
||
|
|
errors.append("allowed_statuses: expected array of strings")
|
||
|
|
elif len(allowed) != len(set(allowed)) or set(allowed) != REQUIRED_STATUSES:
|
||
|
|
expected = ", ".join(sorted(REQUIRED_STATUSES))
|
||
|
|
errors.append(f"allowed_statuses: must contain exactly: {expected}")
|
||
|
|
|
||
|
|
experiments = data.get("experiments")
|
||
|
|
if not isinstance(experiments, list):
|
||
|
|
errors.append(f"experiments: expected array, got {type_name(experiments)}")
|
||
|
|
return errors, counts
|
||
|
|
|
||
|
|
seen_ids: set[str] = set()
|
||
|
|
for index, record in enumerate(experiments):
|
||
|
|
validate_record(record, index, seen_ids, errors)
|
||
|
|
if isinstance(record, dict) and record.get("status") in REQUIRED_STATUSES:
|
||
|
|
counts[record["status"]] += 1
|
||
|
|
return errors, counts
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
default_registry = Path(__file__).resolve().parents[1] / "research" / "experiment_registry.json"
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument("registry", nargs="?", type=Path, default=default_registry)
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
try:
|
||
|
|
with args.registry.open("r", encoding="utf-8") as handle:
|
||
|
|
data = json.load(handle)
|
||
|
|
except FileNotFoundError:
|
||
|
|
print(f"error: registry not found: {args.registry}", file=sys.stderr)
|
||
|
|
return 1
|
||
|
|
except (OSError, UnicodeError) as exc:
|
||
|
|
print(f"error: cannot read registry: {exc}", file=sys.stderr)
|
||
|
|
return 1
|
||
|
|
except json.JSONDecodeError as exc:
|
||
|
|
print(f"error: invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}", file=sys.stderr)
|
||
|
|
return 1
|
||
|
|
|
||
|
|
errors, counts = validate_registry(data)
|
||
|
|
if errors:
|
||
|
|
for error in errors:
|
||
|
|
print(f"error: {error}", file=sys.stderr)
|
||
|
|
return 1
|
||
|
|
|
||
|
|
experiments = data["experiments"]
|
||
|
|
status_counts = ", ".join(
|
||
|
|
f"{status}={counts[status]}" for status in sorted(REQUIRED_STATUSES)
|
||
|
|
)
|
||
|
|
print(f"valid: experiments={len(experiments)}; statuses: {status_counts}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|