201 lines
8.6 KiB
Python
201 lines
8.6 KiB
Python
|
"""Recovery helpers for the frozen HyperScalper Cohort 001 artifact."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import csv
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
from datetime import UTC, datetime
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
DATASET_RECORD_COUNT = 129_599
|
||
|
|
FOLD_COUNT = 4
|
||
|
|
TRAIN_BARS = 14_400
|
||
|
|
EMBARGO_BARS = 35
|
||
|
|
TEST_BARS = 7_200
|
||
|
|
MANIFEST_HASH_PREFIX = "4d20"
|
||
|
|
SEMANTIC_DIGEST = "cc697ea359f20ccc83c5d62f5b406a3532c1d7411191423059164aafe5931254"
|
||
|
|
PROVENANCE = "RECONSTRUCTED_FROM_FROZEN_SPEC_V1"
|
||
|
|
COHORT_MANIFEST = "cohort_manifest.json"
|
||
|
|
DATASET_MANIFEST = "dataset_manifest.json"
|
||
|
|
DATASET_CSV = "binance_btcusdt_spot_2m_180d.csv"
|
||
|
|
|
||
|
|
|
||
|
|
def canonical_json(value: Any) -> str:
|
||
|
|
return json.dumps(
|
||
|
|
value,
|
||
|
|
sort_keys=True,
|
||
|
|
indent=2,
|
||
|
|
ensure_ascii=True,
|
||
|
|
default=lambda item: item.isoformat() if isinstance(item, datetime) else str(item),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def digest(value: Any) -> str:
|
||
|
|
return hashlib.sha256(canonical_json(value).encode()).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def semantic_digest(members: list[dict[str, Any]]) -> str:
|
||
|
|
"""Hash the prior verifier's ordered, minimal semantic strategy projection."""
|
||
|
|
required = ("rank", "source_strategy_id", "base_family", "combo")
|
||
|
|
if any(not isinstance(member, dict) or set(required) - set(member) for member in members):
|
||
|
|
raise ValueError("Frozen cohort semantic records are incomplete.")
|
||
|
|
projection = [
|
||
|
|
{
|
||
|
|
"selection_position": member["rank"],
|
||
|
|
"source_strategy_id": member["source_strategy_id"],
|
||
|
|
"base_family": member["base_family"],
|
||
|
|
"combo": member["combo"],
|
||
|
|
}
|
||
|
|
for member in members
|
||
|
|
]
|
||
|
|
positions = [record["selection_position"] for record in projection]
|
||
|
|
if positions != sorted(positions) or len(set(positions)) != len(positions):
|
||
|
|
raise ValueError("Frozen cohort semantic records are not uniquely ordered.")
|
||
|
|
payload = json.dumps(projection, sort_keys=True, ensure_ascii=True, separators=(",", ":"))
|
||
|
|
return hashlib.sha256(payload.encode()).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def file_digest(path: Path) -> str:
|
||
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def validate_manifest(manifest: dict[str, Any], *, cohort: bool = False) -> None:
|
||
|
|
"""Validate the frozen manifest's canonical self hash and declared semantics."""
|
||
|
|
hash_key = "manifest_sha256" if "manifest_sha256" in manifest else "sha256"
|
||
|
|
stored = manifest.get(hash_key)
|
||
|
|
actual = digest({key: value for key, value in manifest.items() if key != hash_key})
|
||
|
|
if not isinstance(stored, str) or stored != actual:
|
||
|
|
raise ValueError("Frozen manifest canonical hash does not verify.")
|
||
|
|
if cohort and not stored.startswith(MANIFEST_HASH_PREFIX):
|
||
|
|
raise ValueError("Frozen cohort manifest is not the required 4d20 canonical manifest.")
|
||
|
|
if cohort:
|
||
|
|
members = manifest.get("members")
|
||
|
|
if not isinstance(members, list):
|
||
|
|
raise ValueError("Frozen cohort manifest is missing ordered strategy records.")
|
||
|
|
if semantic_digest(members) != SEMANTIC_DIGEST:
|
||
|
|
raise ValueError("Frozen cohort semantic digest does not verify.")
|
||
|
|
lineage = manifest.get("lineage")
|
||
|
|
if lineage is not None and manifest.get("lineage_sha256") != digest(lineage):
|
||
|
|
raise ValueError("Frozen manifest lineage hash does not verify.")
|
||
|
|
|
||
|
|
|
||
|
|
def validate_dataset_manifest(manifest: dict[str, Any]) -> None:
|
||
|
|
"""Validate dataset declarations; its sha256 is the CSV artifact hash, not a self-hash."""
|
||
|
|
required = ("artifact", "sha256", "row_count", "fields", "monotonic_timestamps", "schema_valid")
|
||
|
|
if any(key not in manifest for key in required):
|
||
|
|
raise ValueError("Dataset manifest is missing required artifact declarations.")
|
||
|
|
if (
|
||
|
|
not isinstance(manifest["artifact"], str)
|
||
|
|
or not isinstance(manifest["sha256"], str)
|
||
|
|
or len(manifest["sha256"]) != 64
|
||
|
|
or manifest["row_count"] != DATASET_RECORD_COUNT
|
||
|
|
or not isinstance(manifest["fields"], list)
|
||
|
|
or not manifest["monotonic_timestamps"]
|
||
|
|
or not manifest["schema_valid"]
|
||
|
|
):
|
||
|
|
raise ValueError("Dataset manifest declarations are invalid.")
|
||
|
|
|
||
|
|
|
||
|
|
def validate_selection_source(
|
||
|
|
cohort_manifest: dict[str, Any], selection: dict[str, Any], supplied_path: Path
|
||
|
|
) -> str:
|
||
|
|
"""Bind the unhashed declared source report to all 20 frozen member records."""
|
||
|
|
if cohort_manifest.get("source_report") != str(supplied_path):
|
||
|
|
raise ValueError("Supplied selection source does not match cohort source_report.")
|
||
|
|
selected = selection.get("selected")
|
||
|
|
members = cohort_manifest.get("members")
|
||
|
|
if not isinstance(selected, list) or not isinstance(members, list) or len(members) != 20:
|
||
|
|
raise ValueError("Selection source or frozen members are incomplete.")
|
||
|
|
for member, source in zip(members, selected[:20], strict=True):
|
||
|
|
if (
|
||
|
|
not isinstance(source, dict)
|
||
|
|
or source.get("selection") != member.get("old_selection_metrics")
|
||
|
|
or source.get("final") != member.get("old_final_metrics")
|
||
|
|
or source.get("selection", {}).get("id") != member.get("source_strategy_id")
|
||
|
|
or source.get("selection", {}).get("name") != member.get("source_name")
|
||
|
|
):
|
||
|
|
raise ValueError("Selection source does not reproduce the frozen member order.")
|
||
|
|
return file_digest(supplied_path)
|
||
|
|
|
||
|
|
|
||
|
|
def _timestamp(value: str) -> datetime:
|
||
|
|
raw = float(value)
|
||
|
|
if raw > 100_000_000_000:
|
||
|
|
raw /= 1_000
|
||
|
|
return datetime.fromtimestamp(raw, tz=UTC)
|
||
|
|
|
||
|
|
|
||
|
|
def load_csv(path: Path, timestamp_field: str) -> tuple[list[dict[str, Any]], list[str]]:
|
||
|
|
with path.open(newline="", encoding="utf-8") as handle:
|
||
|
|
reader = csv.DictReader(handle)
|
||
|
|
headers = reader.fieldnames or []
|
||
|
|
if timestamp_field not in headers:
|
||
|
|
raise ValueError(f"Dataset CSV is missing timestamp field {timestamp_field!r}.")
|
||
|
|
rows = []
|
||
|
|
for raw in reader:
|
||
|
|
row: dict[str, Any] = dict(raw)
|
||
|
|
row[timestamp_field] = _timestamp(raw[timestamp_field])
|
||
|
|
for field in ("open", "high", "low", "close", "volume"):
|
||
|
|
if field in row:
|
||
|
|
row[field] = float(row[field])
|
||
|
|
rows.append(row)
|
||
|
|
return rows, headers
|
||
|
|
|
||
|
|
|
||
|
|
def validate_rows(rows: list[dict[str, Any]], headers: list[str], manifest: dict[str, Any]) -> None:
|
||
|
|
if len(rows) != DATASET_RECORD_COUNT:
|
||
|
|
raise ValueError(f"Cohort001 requires exactly {DATASET_RECORD_COUNT} bars.")
|
||
|
|
expected_headers = manifest.get(
|
||
|
|
"fields", manifest.get("columns", manifest.get("schema", manifest.get("dataset_schema")))
|
||
|
|
)
|
||
|
|
if isinstance(expected_headers, dict):
|
||
|
|
expected_headers = list(expected_headers)
|
||
|
|
if expected_headers != headers:
|
||
|
|
raise ValueError("Dataset CSV schema differs from the frozen dataset manifest.")
|
||
|
|
timestamp_field = manifest.get("timestamp_field", "timestamp")
|
||
|
|
timestamps = [row[timestamp_field] for row in rows]
|
||
|
|
interval = timestamps[1] - timestamps[0]
|
||
|
|
if interval.total_seconds() <= 0 or any(
|
||
|
|
right - left != interval for left, right in zip(timestamps, timestamps[1:], strict=False)
|
||
|
|
):
|
||
|
|
raise ValueError("Dataset timestamp continuity does not verify.")
|
||
|
|
canonical_rows = [
|
||
|
|
{
|
||
|
|
key: value.isoformat() if isinstance(value, datetime) else value
|
||
|
|
for key, value in row.items()
|
||
|
|
}
|
||
|
|
for row in rows
|
||
|
|
]
|
||
|
|
expected = manifest.get("canonical_rows_sha256")
|
||
|
|
if expected is not None and expected != digest(canonical_rows):
|
||
|
|
raise ValueError("Dataset canonical row hash does not verify.")
|
||
|
|
|
||
|
|
|
||
|
|
def reconstruct_folds(rows: list[dict[str, Any]], timestamp_field: str) -> list[dict[str, Any]]:
|
||
|
|
block = TRAIN_BARS + EMBARGO_BARS + TEST_BARS
|
||
|
|
starts = [index * (len(rows) - block) // (FOLD_COUNT - 1) for index in range(FOLD_COUNT)]
|
||
|
|
folds = []
|
||
|
|
for number, start in enumerate(starts, start=1):
|
||
|
|
train_end = start + TRAIN_BARS
|
||
|
|
test_start = train_end + EMBARGO_BARS
|
||
|
|
test_end = test_start + TEST_BARS
|
||
|
|
fold = {"fold": f"Fold {number}", "provenance": PROVENANCE}
|
||
|
|
for name, begin, end in (
|
||
|
|
("train", start, train_end),
|
||
|
|
("embargo", train_end, test_start),
|
||
|
|
("test", test_start, test_end),
|
||
|
|
):
|
||
|
|
fold[name] = {
|
||
|
|
"start_index": begin,
|
||
|
|
"end_index": end - 1,
|
||
|
|
"bars": end - begin,
|
||
|
|
"start_at": rows[begin][timestamp_field].isoformat(),
|
||
|
|
"end_at": rows[end - 1][timestamp_field].isoformat(),
|
||
|
|
"sha256": digest(rows[begin:end]),
|
||
|
|
"status": "EXCLUDED" if name == "embargo" else "ACTIVE",
|
||
|
|
}
|
||
|
|
folds.append(fold)
|
||
|
|
return folds
|