daniel/Artifex
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add HS22 qualification and parity infrastructure

Browse source
This commit is contained in:
Daniel Maddern 2026-08-18 02:00:53 +07:00
parent 65f0d510ec
commit 48d82561f2
50 changed files with 10918 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

200
control_plane/trading_studio/cohort_materialization.py Normal file
View file

@ -0,0 +1,200 @@
"""Recovery helpers for the frozen HyperScalper Cohort 001 artifact."""
from __future__ import annotations
import csv
import hashlib
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
DATASET_RECORD_COUNT = 129_599
FOLD_COUNT = 4
TRAIN_BARS = 14_400
EMBARGO_BARS = 35
TEST_BARS = 7_200
MANIFEST_HASH_PREFIX = "4d20"
SEMANTIC_DIGEST = "cc697ea359f20ccc83c5d62f5b406a3532c1d7411191423059164aafe5931254"
PROVENANCE = "RECONSTRUCTED_FROM_FROZEN_SPEC_V1"
COHORT_MANIFEST = "cohort_manifest.json"
DATASET_MANIFEST = "dataset_manifest.json"
DATASET_CSV = "binance_btcusdt_spot_2m_180d.csv"
def canonical_json(value: Any) -> str:
return json.dumps(
value,
sort_keys=True,
indent=2,
ensure_ascii=True,
default=lambda item: item.isoformat() if isinstance(item, datetime) else str(item),
)
def digest(value: Any) -> str:
return hashlib.sha256(canonical_json(value).encode()).hexdigest()
def semantic_digest(members: list[dict[str, Any]]) -> str:
"""Hash the prior verifier's ordered, minimal semantic strategy projection."""
required = ("rank", "source_strategy_id", "base_family", "combo")
if any(not isinstance(member, dict) or set(required) - set(member) for member in members):
raise ValueError("Frozen cohort semantic records are incomplete.")
projection = [
{
"selection_position": member["rank"],
"source_strategy_id": member["source_strategy_id"],
"base_family": member["base_family"],
"combo": member["combo"],
}
for member in members
]
positions = [record["selection_position"] for record in projection]
if positions != sorted(positions) or len(set(positions)) != len(positions):
raise ValueError("Frozen cohort semantic records are not uniquely ordered.")
payload = json.dumps(projection, sort_keys=True, ensure_ascii=True, separators=(",", ":"))
return hashlib.sha256(payload.encode()).hexdigest()
def file_digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def validate_manifest(manifest: dict[str, Any], *, cohort: bool = False) -> None:
"""Validate the frozen manifest's canonical self hash and declared semantics."""
hash_key = "manifest_sha256" if "manifest_sha256" in manifest else "sha256"
stored = manifest.get(hash_key)
actual = digest({key: value for key, value in manifest.items() if key != hash_key})
if not isinstance(stored, str) or stored != actual:
raise ValueError("Frozen manifest canonical hash does not verify.")
if cohort and not stored.startswith(MANIFEST_HASH_PREFIX):
raise ValueError("Frozen cohort manifest is not the required 4d20 canonical manifest.")
if cohort:
members = manifest.get("members")
if not isinstance(members, list):
raise ValueError("Frozen cohort manifest is missing ordered strategy records.")
if semantic_digest(members) != SEMANTIC_DIGEST:
raise ValueError("Frozen cohort semantic digest does not verify.")
lineage = manifest.get("lineage")
if lineage is not None and manifest.get("lineage_sha256") != digest(lineage):
raise ValueError("Frozen manifest lineage hash does not verify.")
def validate_dataset_manifest(manifest: dict[str, Any]) -> None:
"""Validate dataset declarations; its sha256 is the CSV artifact hash, not a self-hash."""
required = ("artifact", "sha256", "row_count", "fields", "monotonic_timestamps", "schema_valid")
if any(key not in manifest for key in required):
raise ValueError("Dataset manifest is missing required artifact declarations.")
if (
not isinstance(manifest["artifact"], str)
or not isinstance(manifest["sha256"], str)
or len(manifest["sha256"]) != 64
or manifest["row_count"] != DATASET_RECORD_COUNT
or not isinstance(manifest["fields"], list)
or not manifest["monotonic_timestamps"]
or not manifest["schema_valid"]
):
raise ValueError("Dataset manifest declarations are invalid.")
def validate_selection_source(
cohort_manifest: dict[str, Any], selection: dict[str, Any], supplied_path: Path
) -> str:
"""Bind the unhashed declared source report to all 20 frozen member records."""
if cohort_manifest.get("source_report") != str(supplied_path):
raise ValueError("Supplied selection source does not match cohort source_report.")
selected = selection.get("selected")
members = cohort_manifest.get("members")
if not isinstance(selected, list) or not isinstance(members, list) or len(members) != 20:
raise ValueError("Selection source or frozen members are incomplete.")
for member, source in zip(members, selected[:20], strict=True):
if (
not isinstance(source, dict)
or source.get("selection") != member.get("old_selection_metrics")
or source.get("final") != member.get("old_final_metrics")
or source.get("selection", {}).get("id") != member.get("source_strategy_id")
or source.get("selection", {}).get("name") != member.get("source_name")
):
raise ValueError("Selection source does not reproduce the frozen member order.")
return file_digest(supplied_path)
def _timestamp(value: str) -> datetime:
raw = float(value)
if raw > 100_000_000_000:
raw /= 1_000
return datetime.fromtimestamp(raw, tz=UTC)
def load_csv(path: Path, timestamp_field: str) -> tuple[list[dict[str, Any]], list[str]]:
with path.open(newline="", encoding="utf-8") as handle:
reader = csv.DictReader(handle)
headers = reader.fieldnames or []
if timestamp_field not in headers:
raise ValueError(f"Dataset CSV is missing timestamp field {timestamp_field!r}.")
rows = []
for raw in reader:
row: dict[str, Any] = dict(raw)
row[timestamp_field] = _timestamp(raw[timestamp_field])
for field in ("open", "high", "low", "close", "volume"):
if field in row:
row[field] = float(row[field])
rows.append(row)
return rows, headers
def validate_rows(rows: list[dict[str, Any]], headers: list[str], manifest: dict[str, Any]) -> None:
if len(rows) != DATASET_RECORD_COUNT:
raise ValueError(f"Cohort001 requires exactly {DATASET_RECORD_COUNT} bars.")
expected_headers = manifest.get(
"fields", manifest.get("columns", manifest.get("schema", manifest.get("dataset_schema")))
)
if isinstance(expected_headers, dict):
expected_headers = list(expected_headers)
if expected_headers != headers:
raise ValueError("Dataset CSV schema differs from the frozen dataset manifest.")
timestamp_field = manifest.get("timestamp_field", "timestamp")
timestamps = [row[timestamp_field] for row in rows]
interval = timestamps[1] - timestamps[0]
if interval.total_seconds() <= 0 or any(
right - left != interval for left, right in zip(timestamps, timestamps[1:], strict=False)
):
raise ValueError("Dataset timestamp continuity does not verify.")
canonical_rows = [
{
key: value.isoformat() if isinstance(value, datetime) else value
for key, value in row.items()
}
for row in rows
]
expected = manifest.get("canonical_rows_sha256")
if expected is not None and expected != digest(canonical_rows):
raise ValueError("Dataset canonical row hash does not verify.")
def reconstruct_folds(rows: list[dict[str, Any]], timestamp_field: str) -> list[dict[str, Any]]:
block = TRAIN_BARS + EMBARGO_BARS + TEST_BARS
starts = [index * (len(rows) - block) // (FOLD_COUNT - 1) for index in range(FOLD_COUNT)]
folds = []
for number, start in enumerate(starts, start=1):
train_end = start + TRAIN_BARS
test_start = train_end + EMBARGO_BARS
test_end = test_start + TEST_BARS
fold = {"fold": f"Fold {number}", "provenance": PROVENANCE}
for name, begin, end in (
("train", start, train_end),
("embargo", train_end, test_start),
("test", test_start, test_end),
):
fold[name] = {
"start_index": begin,
"end_index": end - 1,
"bars": end - begin,
"start_at": rows[begin][timestamp_field].isoformat(),
"end_at": rows[end - 1][timestamp_field].isoformat(),
"sha256": digest(rows[begin:end]),
"status": "EXCLUDED" if name == "embargo" else "ACTIVE",
}
folds.append(fold)
return folds

44
control_plane/trading_studio/indicators/__init__.py Normal file
View file

@ -0,0 +1,44 @@
"""Versioned indicator registry contracts. Formula implementations are deliberately absent."""
from .engine import (
IndicatorEngine,
RefusalCode,
RefusalState,
compute_hs22_state,
evaluate_indicator,
)
from .feature_catalog import (
FEATURE_CATALOG_V1,
CausalClassification,
FeatureResearchDefinition,
FeatureSupportState,
require_validated_features,
)
from .registry import (
HISTORICAL_REGISTRY,
HS22_COHORT001_V1,
HS22_HISTORICAL_COMPLETE_V1,
HS22_REQUIRED_V1,
historical_artifact_digest,
)
from .schema import HS22State, parse_hs22
__all__ = [
"HISTORICAL_REGISTRY",
"HS22_COHORT001_V1",
"HS22_REQUIRED_V1",
"HS22_HISTORICAL_COMPLETE_V1",
"FEATURE_CATALOG_V1",
"CausalClassification",
"FeatureResearchDefinition",
"FeatureSupportState",
"HS22State",
"IndicatorEngine",
"RefusalCode",
"RefusalState",
"historical_artifact_digest",
"compute_hs22_state",
"evaluate_indicator",
"parse_hs22",
"require_validated_features",
]

307
control_plane/trading_studio/indicators/batch01_native_parity.py Normal file
View file

@ -0,0 +1,307 @@
"""Offline exact-parity gate for the frozen Batch01 band/channel oracle."""
from __future__ import annotations
import csv
import hashlib
import json
from pathlib import Path
from typing import Any
import numpy as np
from .historical_band_channel import OBSERVED_BAND_CHANNEL_PARAMS, evaluate_band_channel
from .parity_harness_v1 import (
ComparisonRecord,
FeatureVersion,
ParityStatus,
compare_array,
compare_float64_ulp,
recompute_coverage,
)
ROOT = Path(__file__).resolve().parents[3]
BATCH01_REQUEST = ROOT / "batch01_oracle_request.json"
BATCH01_RESULT = ROOT / "batch01_oracle_result.json"
BATCH01_OUTPUTS = ROOT / "batch01_oracle_outputs.npz"
CANONICAL_CSV = ROOT / "hs22_oracle_v1" / "binance_btcusdt_spot_2m_180d.csv"
BOLLINGER_RESOLUTION = ROOT / "batch01_bollinger_parity_resolution_v1.json"
BOLLINGER_ROOT_REPORT = ROOT / "batch01_native_mismatch_report.json"
_DIGESTS = {
"batch01_oracle_request.json": (
"7ca0c8578dd81c3ae60898d83e6e73f9bf8889a39a873b451c7cdaf036cbaf24"
),
"batch01_oracle_result.json": (
"7c1d5dbdcb183aa2d872baa45c4d02fe2d59cccabe9022f8797237b3776c8e96"
),
"batch01_oracle_outputs.npz": (
"e1f2268b634e50eb65b9b5175ea369e52bf0a2039b18049030849948a45fd149"
),
"hs22_oracle_v1/binance_btcusdt_spot_2m_180d.csv": (
"7fb56bb05b0edc2348407765cd693efadf334307c25877dfb52acfe4221e5d00"
),
}
class FrozenBatch01ArtifactError(ValueError):
"""Raised when a parity input is not the pinned local oracle artifact."""
def _digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _canonical_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")
def _array_digest(values: np.ndarray) -> str:
values = np.ascontiguousarray(values)
header = _canonical_bytes({"dtype": values.dtype.str, "shape": values.shape})
return hashlib.sha256(header + b"\0" + values.tobytes()).hexdigest()
def batch01_native_parity_manifest() -> dict[str, object]:
"""Return the reproducible exact-comparison contract for all 97 variants."""
return {
"schema_version": 1,
"artifact": "BATCH01_NATIVE_PARITY_MANIFEST_V1",
"engine_revision": "code-5056feb",
"frozen_inputs": {
name: {"path": name, "sha256": digest} for name, digest in _DIGESTS.items()
},
"input_columns": ["close", "high", "low", "volume"],
"variant_count": 97,
"comparison": {"dtype": "exact", "shape": "exact", "nan_mask": "exact", "values": "exact"},
"documented_exception": {
"path": BOLLINGER_RESOLUTION.name,
"scope": "float64 one-ULP cases listed for indicator IDs 17 and 18 only",
"global_tolerance": "forbidden",
},
"derived_state_transitions": {
"indicator_ids": [19, 28],
"state": "1 when close is greater than output, otherwise -1; warmup NaN is 0",
"transitions": "state differs from the preceding bar",
},
}
def batch01_native_parity_manifest_bytes() -> bytes:
return _canonical_bytes(batch01_native_parity_manifest()) + b"\n"
def write_batch01_native_parity_manifest(path: Path) -> None:
path.write_bytes(batch01_native_parity_manifest_bytes())
def _require_frozen_artifacts() -> tuple[dict[str, Any], dict[str, Any]]:
paths = {
"batch01_oracle_request.json": BATCH01_REQUEST,
"batch01_oracle_result.json": BATCH01_RESULT,
"batch01_oracle_outputs.npz": BATCH01_OUTPUTS,
"hs22_oracle_v1/binance_btcusdt_spot_2m_180d.csv": CANONICAL_CSV,
}
for name, path in paths.items():
if _digest(path) != _DIGESTS[name]:
raise FrozenBatch01ArtifactError(f"{name} digest does not match the frozen artifact")
request = json.loads(BATCH01_REQUEST.read_text(encoding="utf-8"))
result = json.loads(BATCH01_RESULT.read_text(encoding="utf-8"))
if request.get("artifact") != "HISTORICAL_FEATURE_ORACLE_V1_REQUEST":
raise FrozenBatch01ArtifactError("invalid Batch01 request artifact")
if result.get("artifact") != "HISTORICAL_FEATURE_ORACLE_V1_RESULT":
raise FrozenBatch01ArtifactError("invalid Batch01 result artifact")
if result.get("request_sha256") != _DIGESTS["batch01_oracle_request.json"]:
raise FrozenBatch01ArtifactError("result is not linked to the frozen request")
csv_digest = _DIGESTS["hs22_oracle_v1/binance_btcusdt_spot_2m_180d.csv"]
if result.get("input_csv_sha256") != csv_digest:
raise FrozenBatch01ArtifactError("result is not linked to the canonical CSV")
if result.get("outputs_npz", {}).get("sha256") != _DIGESTS["batch01_oracle_outputs.npz"]:
raise FrozenBatch01ArtifactError("result is not linked to the frozen NPZ")
return request, result
def _bollinger_resolution() -> dict[str, tuple[dict[str, Any], ...]]:
resolution = json.loads(BOLLINGER_RESOLUTION.read_text(encoding="utf-8"))
report = json.loads(BOLLINGER_ROOT_REPORT.read_text(encoding="utf-8"))
source = resolution.get("source_report", {})
exception = resolution.get("exception", {})
if (
resolution.get("artifact") != "BATCH01_BOLLINGER_PARITY_RESOLUTION_V1"
or resolution.get("schema_version") != 1
or source.get("artifact") != report.get("artifact")
or source.get("classification") != report.get("classification")
or source.get("failed_value_count") != report.get("failed_value_count")
or exception != {
"indicator_ids": [17, 18],
"dtype": "float64",
"max_ulps": 1,
"unlisted_mismatch_policy": "fail",
"all_other_indicators_policy": "exact",
"shape_policy": "exact",
"nan_mask_policy": "exact",
}
):
raise FrozenBatch01ArtifactError("invalid Bollinger parity resolution")
report_cases = {
(item["request_id"], int(detail["index"])): {
"indicator_id": item["indicator_id"],
"signed_ulp": detail["ulp"]["signed"],
"threshold_sensitivity": {
key: detail["threshold_sensitivity"][key]
for key in (
"oracle_close_above_band",
"native_close_above_band",
"decision_changed",
"close_to_oracle_band_ulps",
"close_to_native_band_ulps",
)
},
}
for item in report["variants"]
for detail in item["mismatches"]
}
accepted = resolution.get("accepted_cases")
if not isinstance(accepted, list) or len(accepted) != 8:
raise FrozenBatch01ArtifactError("resolution must list all eight accepted cases")
resolution_cases = {
(item["request_id"], int(item["index"])): {
"indicator_id": item["indicator_id"],
"signed_ulp": item["signed_ulp"],
"threshold_sensitivity": item["threshold_sensitivity"],
}
for item in accepted
}
if resolution_cases != report_cases or any(
int(item["indicator_id"]) not in {17, 18}
or item["threshold_sensitivity"]["decision_changed"]
for item in accepted
):
raise FrozenBatch01ArtifactError("resolution does not match the root-cause evidence")
grouped: dict[str, list[dict[str, Any]]] = {}
for item in accepted:
grouped.setdefault(item["request_id"], []).append(item)
return {request_id: tuple(cases) for request_id, cases in grouped.items()}
def _load_ohlcv() -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
with CANONICAL_CSV.open(newline="", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
return tuple(
np.asarray([float(row[name]) for row in rows], dtype=np.float64)
for name in ("close", "high", "low", "volume")
) # type: ignore[return-value]
def _state_and_transitions(close: np.ndarray, output: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
state = np.zeros(output.shape, dtype=np.int8)
valid = ~np.isnan(output)
state[valid] = np.where(close[valid] > output[valid], 1, -1)
transitions = np.zeros(output.shape, dtype=np.bool_)
transitions[1:] = state[1:] != state[:-1]
return state, transitions
def _record_artifact(record: ComparisonRecord) -> dict[str, object]:
return record.artifact()
def batch01_native_acceptance_manifest(report: dict[str, object]) -> dict[str, object]:
"""Summarize the resolved Batch01 gate and recompute its submitted coverage."""
variants = [item for item in report["records"] if ":" not in item["name"]]
feature_versions = [
FeatureVersion(
feature_version=item["name"],
primitive_key=item["name"],
engineering_family="band_channel",
usage_slots=1,
status=ParityStatus(item["status"]),
)
for item in variants
]
accepted = [
item for item in variants if item["reason"] == "accepted_documented_float64_ulp_drift"
]
return {
"schema_version": 1,
"artifact": "BATCH01_NATIVE_ACCEPTANCE_MANIFEST_V1",
"parity_manifest": batch01_native_parity_manifest(),
"status": "accepted" if report["passed"] else "rejected",
"accepted_documented_ulp_variant_count": len(accepted),
"accepted_documented_ulp_value_count": sum(item["value_mismatches"] for item in accepted),
"coverage": recompute_coverage(feature_versions),
}
def write_batch01_native_acceptance_manifest(path: Path) -> None:
report = evaluate_batch01_native_parity()
path.write_bytes(_canonical_bytes(batch01_native_acceptance_manifest(report)) + b"\n")
def evaluate_batch01_native_parity() -> dict[str, object]:
"""Evaluate native Batch01 functions against the immutable oracle NPZ exactly."""
request, result = _require_frozen_artifacts()
accepted_cases = _bollinger_resolution()
requests = request["requests"]
result_requests = {item["request_id"]: item for item in result["requests"]}
requested_params = {
(int(item["indicator_id"]), int(item["period"]), float(item["p1"])) for item in requests
}
if requested_params != OBSERVED_BAND_CHANNEL_PARAMS or len(requests) != 97:
raise FrozenBatch01ArtifactError("request variants do not match the Batch01 native surface")
if set(result_requests) != {item["request_id"] for item in requests}:
raise FrozenBatch01ArtifactError("result request records do not match the frozen request")
close, high, low, volume = _load_ohlcv()
records: list[ComparisonRecord] = []
with np.load(BATCH01_OUTPUTS) as expected_outputs:
if set(expected_outputs.files) != set(result_requests):
raise FrozenBatch01ArtifactError("NPZ output keys do not match result request records")
for item in requests:
request_id = item["request_id"]
expected = expected_outputs[request_id]
metadata = result_requests[request_id]
if expected.dtype.str != metadata["dtype"] or list(expected.shape) != metadata["shape"]:
raise FrozenBatch01ArtifactError(f"NPZ metadata mismatch for {request_id}")
if _array_digest(expected) != metadata["sha256"]:
raise FrozenBatch01ArtifactError(f"NPZ value digest mismatch for {request_id}")
actual = evaluate_band_channel(
int(item["indicator_id"]),
close,
high,
low,
volume,
int(item["period"]),
float(item["p1"]),
)
indicator_id = int(item["indicator_id"])
cases = accepted_cases.get(request_id, ())
records.append(
compare_float64_ulp(
request_id,
expected,
actual,
max_ulps=1,
allowed_indexes=frozenset(int(case["index"]) for case in cases),
)
if indicator_id in {17, 18}
else compare_array(request_id, expected, actual)
)
if indicator_id in {19, 28}:
expected_state, expected_transitions = _state_and_transitions(close, expected)
actual_state, actual_transitions = _state_and_transitions(close, actual)
records.append(compare_array(f"{request_id}:state", expected_state, actual_state))
records.append(
compare_array(
f"{request_id}:transitions", expected_transitions, actual_transitions
)
)
failures = [record for record in records if not record.passed]
report = {
"manifest": batch01_native_parity_manifest(),
"records": [_record_artifact(record) for record in records],
"passed": not failures,
"failure_count": len(failures),
"failures": [_record_artifact(record) for record in failures],
}
report["acceptance_manifest"] = batch01_native_acceptance_manifest(report)
return report

65
control_plane/trading_studio/indicators/batch01_oracle_manifest.py Normal file
View file

@ -0,0 +1,65 @@
"""Deterministic external-oracle requests for the observed Batch01 surface."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from .historical_band_channel import OBSERVED_BAND_CHANNEL_PARAMS
BATCH01_ORACLE_REQUEST_MANIFEST_V1_PATH = Path(__file__).with_name(
"batch01_oracle_request_manifest_v1.json"
)
_ENGINE_REVISION = "code-5056feb"
_SEMANTIC_FIELDS = {
"engine_revision": _ENGINE_REVISION,
"input_columns": ["close", "high", "low", "volume"],
"output_dtype": "float64",
"window_policy": "continuous_full_history",
}
def _canonical_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")
def batch01_oracle_request_manifest() -> dict[str, object]:
"""Build exact observed requests without evaluating any historical formula."""
requests = []
for indicator_id, period, p1 in sorted(OBSERVED_BAND_CHANNEL_PARAMS):
semantic = {
**_SEMANTIC_FIELDS,
"indicator_id": indicator_id,
"period": period,
"p1": p1,
}
requests.append(
{
"request_id": f"batch01_{indicator_id}_{period}_{p1:g}",
"indicator_id": indicator_id,
"period": period,
"p1": p1,
"semantic_fingerprint": hashlib.sha256(_canonical_bytes(semantic)).hexdigest(),
}
)
assert len(requests) == 97
return {
"schema_version": 1,
"artifact": "HISTORICAL_FEATURE_ORACLE_V1_REQUEST",
"engine_revision": _ENGINE_REVISION,
"input_columns": _SEMANTIC_FIELDS["input_columns"],
"output_dtype": _SEMANTIC_FIELDS["output_dtype"],
"window_policy": _SEMANTIC_FIELDS["window_policy"],
"requests": requests,
}
def batch01_oracle_request_manifest_bytes() -> bytes:
return _canonical_bytes(batch01_oracle_request_manifest()) + b"\n"
def write_batch01_oracle_request_manifest(
path: Path = BATCH01_ORACLE_REQUEST_MANIFEST_V1_PATH,
) -> None:
path.write_bytes(batch01_oracle_request_manifest_bytes())

264
control_plane/trading_studio/indicators/definitions.py Normal file
View file

@ -0,0 +1,264 @@
"""Typed, data-only definitions for the recovered HS22 indicator inventory."""
from __future__ import annotations
from dataclasses import asdict, dataclass
from enum import StrEnum
from itertools import product
class Role(StrEnum):
LEVEL = "level"
OSC = "osc"
TREND = "trend"
FILTER = "filter"
class ImplementationStatus(StrEnum):
UNIMPLEMENTED = "unimplemented"
ALIAS = "alias"
UNREGISTERED = "unregistered"
@dataclass(frozen=True, slots=True)
class IndicatorDefinition:
indicator_id: int
name: str
role: Role | None
status: ImplementationStatus = ImplementationStatus.UNIMPLEMENTED
alias_of: int | None = None
behavior: str = "refuse_execution"
@dataclass(frozen=True, slots=True)
class IndicatorVariant:
indicator_id: int
period: int
p1: float
role: Role
def asdict(self) -> dict[str, object]:
return asdict(self)
# Recovered from the historical pool definition. This is deliberately data, not
# a dependency on or import of the reference runtime.
_SPECS: tuple[tuple[int, str, Role, tuple[int, ...], tuple[float, ...]], ...] = (
*(
(i, name, Role.LEVEL, tuple(range(start, stop, step)), params)
for i, name, start, stop, step, params in (
(0, "SMA", 5, 51, 3, (0.0,)),
(1, "EMA", 5, 51, 3, (0.0,)),
(2, "WMA", 5, 51, 3, (0.0,)),
(3, "HMA", 6, 51, 2, (0.0,)),
(4, "DEMA", 5, 41, 3, (0.0,)),
(5, "TEMA", 5, 36, 3, (0.0,)),
(6, "KAMA", 8, 36, 3, (0.0,)),
(7, "EHLERS_SS", 6, 31, 2, (0.0,)),
(8, "MCGINLEY", 8, 36, 3, (0.0,)),
(9, "JMA", 8, 36, 4, (-50.0, 0.0, 50.0, 100.0)),
(10, "T3", 8, 31, 3, (0.6, 0.7, 0.8, 0.9)),
(11, "ALMA", 8, 31, 3, (4.0, 6.0, 8.0)),
(12, "ZLEMA", 5, 41, 3, (0.0,)),
(13, "VIDYA", 8, 36, 4, (0.0,)),
(14, "FRAMA", 10, 41, 4, (0.0,)),
(15, "LSMA", 8, 41, 3, (0.0,)),
(16, "SWMA", 5, 31, 3, (0.0,)),
(17, "BB_UPPER", 14, 26, 2, (1.5, 2.0, 2.5)),
(18, "BB_LOWER", 14, 26, 2, (1.5, 2.0, 2.5)),
(19, "SUPERTREND", 8, 16, 2, (2.0, 3.0, 4.0)),
(20, "DONCHIAN_UPPER", 10, 30, 4, (0.0,)),
(21, "DONCHIAN_LOWER", 10, 30, 4, (0.0,)),
(22, "DONCHIAN_MID", 10, 30, 4, (0.0,)),
(23, "KELTNER_UPPER", 14, 26, 2, (1.0, 1.5, 2.0)),
(24, "KELTNER_LOWER", 14, 26, 2, (1.0, 1.5, 2.0)),
)
),
(25, "KELTNER_MID", Role.LEVEL, tuple(range(14, 26, 2)), (0.0,)),
(26, "ICHIMOKU_TENKAN", Role.LEVEL, tuple(range(7, 13, 2)), (0.0,)),
(27, "ICHIMOKU_KIJUN", Role.LEVEL, tuple(range(20, 30, 2)), (0.0,)),
(28, "PSAR", Role.LEVEL, (1,), (0.01, 0.02, 0.03)),
)
# The remaining recovered records use compact rows: id, name, role, periods, p1 values.
_COMPACT_SPECS = (
(30, "RSI", "osc", range(6, 28, 2)),
(31, "STOCH_K", "osc", range(5, 22, 2)),
(32, "STOCH_D", "osc", range(5, 22, 2)),
(33, "CCI", "osc", range(10, 30, 2)),
(34, "WILLIAMS_R", "osc", range(5, 22, 2)),
(35, "ROC", "osc", range(5, 21, 2)),
(36, "CMO", "osc", range(8, 22, 2)),
(37, "TRIX", "osc", range(8, 22, 2)),
(38, "PPO", "osc", range(16, 30, 2)),
(39, "MACD_HIST", "osc", range(20, 32, 2)),
(40, "MFI", "osc", range(10, 22, 2)),
(41, "DPO", "osc", range(10, 26, 2)),
(42, "PCTILE_RANK", "osc", range(14, 51, 6)),
(43, "ZSCORE", "osc", range(14, 51, 6)),
(44, "MINMAX", "osc", range(14, 51, 6)),
*(
(i, name, "osc", periods)
for i, name, periods in (
(90, "MICRO_BODY_RATIO", range(5, 21, 3)),
(91, "MICRO_UPPER_WICK", range(5, 21, 3)),
(92, "MICRO_LOWER_WICK", range(5, 21, 3)),
(93, "MICRO_BUY_PRESSURE", range(5, 21, 3)),
(94, "MICRO_SELL_PRESSURE", range(5, 21, 3)),
(95, "MICRO_STOP_HUNT", range(10, 21, 5)),
(96, "MICRO_FVG", (1,)),
(97, "MICRO_LIQ_SWEEP", range(5, 21, 4)),
(98, "TREND_BREAKOUT", range(10, 31, 5)),
(99, "TREND_PULLBACK", range(10, 31, 5)),
(100, "MOM_VELOCITY", range(5, 21, 3)),
(101, "MOM_ACCELERATION", range(5, 21, 3)),
(102, "MOM_COMPOSITE", range(12, 31, 4)),
(103, "MOM_NORMALIZED_ROC", range(5, 21, 3)),
(104, "CROSS_PRICE_VS_EMA", range(8, 31, 3)),
(105, "CROSS_EMA_SPREAD", range(14, 41, 4)),
(106, "CROSS_MOM_X_VOL", range(8, 21, 3)),
(107, "CROSS_VOL_ADJ_MOM", range(8, 21, 3)),
(108, "NORM_RETURN_ZSCORE", range(14, 51, 6)),
(109, "TIME_BARS_SINCE_HIGH", range(10, 51, 10)),
(110, "TIME_BARS_SINCE_LOW", range(10, 51, 10)),
(116, "REGIME_SKEWNESS", range(20, 61, 10)),
(52, "VOL_ATR_ZSCORE", range(10, 31, 5)),
(59, "VOL_RANGE_PCTILE", range(14, 31, 4)),
(60, "VOL_TR_MOMENTUM", range(10, 31, 5)),
)
),
*(
(i, name, "trend", periods)
for i, name, periods in (
(70, "ADX", range(10, 22, 2)),
(71, "AROON_UP", range(10, 30, 4)),
(72, "AROON_DOWN", range(10, 30, 4)),
(73, "LINREG_SLOPE", range(8, 31, 3)),
(74, "LINREG_R2", range(8, 31, 3)),
(75, "EFFICIENCY", range(8, 31, 3)),
(76, "ANGLE", range(8, 31, 3)),
(77, "DURATION", (1,)),
(78, "HURST", range(20, 61, 10)),
(80, "FRACTAL_DIM", range(20, 61, 10)),
(81, "TREND_PERSIST", range(10, 31, 5)),
(83, "MICRO_TREND_STRUCT", range(5, 21, 3)),
)
),
)
def _variants() -> tuple[IndicatorVariant, ...]:
specs = list(_SPECS)
specs.extend((i, n, Role(r), tuple(p), (0.0,)) for i, n, r, p in _COMPACT_SPECS)
specs.extend(
(
(79, "VARIANCE_RATIO", Role.TREND, tuple(range(20, 61, 10)), (2.0, 4.0, 8.0)),
(82, "AUTOCORR", Role.TREND, tuple(range(20, 51, 10)), (1.0, 3.0, 5.0)),
)
)
specs.extend(
(i, n, Role.FILTER, tuple(p), (0.0,))
for i, n, p in (
(50, "ATR", range(8, 22, 2)),
(51, "ATR_PCT", range(8, 22, 2)),
(53, "VOL_COMPRESSION", range(14, 31, 4)),
(54, "VOL_OF_VOL", range(14, 31, 4)),
(55, "VOL_PARKINSON", range(10, 31, 5)),
(56, "VOL_GARMAN_KLASS", range(10, 31, 5)),
(57, "VOL_ROGERS_SATCHELL", range(10, 31, 5)),
(58, "VOL_REALIZED", range(10, 31, 5)),
(61, "NORM_VOL_ZSCORE", range(14, 51, 6)),
(63, "MICRO_VOL_DELTA", range(5, 21, 3)),
(111, "ENTROPY_SHANNON", range(20, 61, 10)),
(112, "KURTOSIS", range(20, 61, 10)),
(113, "HALFLIFE", range(30, 61, 10)),
(115, "MA_COMPRESSION", range(10, 31, 5)),
(118, "TIME_HOUR_COS", (1,)),
(119, "TIME_SESSION_VOL", (1,)),
)
)
specs.extend(
(
(62, "MICRO_VOL_CLUSTERING", Role.TREND, tuple(range(10, 31, 5)), (0.0,)),
(117, "TIME_HOUR_SIN", Role.TREND, (1,), (0.0,)),
)
)
specs.append((114, "DC_EVENTS", Role.FILTER, tuple(range(20, 51, 10)), (10.0, 20.0, 50.0)))
specs.extend(
(
i,
n,
Role.LEVEL if role == "level" else Role.OSC if role == "osc" else Role.FILTER,
tuple(p),
params,
)
for i, n, role, p, params in (
(120, "PIVOT_CLASSIC", "level", range(30, 361, 30), (0.0,)),
(121, "PIVOT_R1", "level", range(30, 361, 30), (0.0,)),
(122, "PIVOT_S1", "level", range(30, 361, 30), (0.0,)),
(123, "PIVOT_DISTANCE", "osc", range(30, 361, 30), (0.0,)),
(124, "PREV_PERIOD_HIGH", "level", range(30, 361, 30), (0.0,)),
(125, "PREV_PERIOD_LOW", "level", range(30, 361, 30), (0.0,)),
(126, "PREV_PERIOD_CLOSE", "level", range(30, 361, 30), (0.0,)),
(127, "ROLLING_MEDIAN", "level", range(10, 41, 5), (0.0,)),
(128, "LINREG_CHAN_UPPER", "level", range(14, 31, 4), (1.5, 2.0, 2.5)),
(129, "LINREG_CHAN_LOWER", "level", range(14, 31, 4), (1.5, 2.0, 2.5)),
(130, "QUANTILE_UPPER", "level", range(14, 41, 6), (0.0,)),
(131, "QUANTILE_LOWER", "level", range(14, 41, 6), (0.0,)),
(135, "FRESH_BREAKOUT", "osc", range(10, 31, 5), (0.0,)),
(136, "FAILED_BREAKOUT", "osc", range(10, 31, 5), (0.0,)),
(137, "FIRST_PULLBACK", "osc", range(10, 31, 5), (0.0,)),
(138, "VOL_EXPANSION", "osc", range(10, 21, 5), (0.0,)),
(139, "INSIDE_BAR", "osc", (1,), (0.0,)),
(140, "COMPRESS_RELEASE", "osc", range(10, 21, 5), (0.0,)),
(141, "SWEEP_RECLAIM", "osc", range(10, 31, 5), (0.0,)),
(142, "CLOSE_IN_RANGE", "osc", (1,), (0.0,)),
(143, "CLOSE_IN_ROLLING_RANGE", "osc", range(10, 31, 5), (0.0,)),
(144, "DIST_RECENT_HIGH", "osc", range(10, 31, 5), (0.0,)),
(145, "DIST_RECENT_LOW", "osc", range(10, 31, 5), (0.0,)),
(146, "CHANNEL_POSITION", "osc", range(14, 31, 4), (0.0,)),
(147, "OC_VS_PRIOR", "osc", range(10, 31, 5), (0.0,)),
(150, "CHOP_INDEX", "filter", range(10, 31, 5), (0.0,)),
(151, "CANDLE_OVERLAP", "filter", range(10, 31, 5), (0.0,)),
(152, "NOISE_RATIO", "filter", range(10, 31, 5), (0.0,)),
(153, "WICK_INSTABILITY", "filter", range(10, 31, 5), (0.0,)),
(154, "FALSE_BREAK_FREQ", "filter", range(10, 31, 10), (0.0,)),
(155, "SPREAD_PROXY", "filter", range(10, 31, 5), (0.0,)),
(156, "DIR_CLEAN", "filter", range(10, 31, 5), (0.0,)),
(157, "REVERSAL_FREQ", "filter", range(10, 31, 5), (0.0,)),
(158, "MEDIAN_EXCURSION", "filter", range(10, 31, 5), (0.0,)),
(160, "RSI_SLOPE", "osc", range(8, 22, 2), (0.0,)),
(161, "RSI_DIST_50", "osc", range(8, 22, 2), (0.0,)),
(162, "RSI_DIVERGENCE", "osc", range(10, 22, 2), (5.0, 10.0)),
(163, "MACD_SLOPE", "osc", range(20, 30, 2), (0.0,)),
(164, "MACD_DIVERGENCE", "osc", range(20, 30, 2), (0.0,)),
(165, "TIME_SINCE_OB", "osc", range(8, 22, 2), (0.0,)),
(166, "TIME_SINCE_OS", "osc", range(8, 22, 2), (0.0,)),
(167, "EXHAUSTION", "osc", range(10, 22, 2), (0.0,)),
(168, "LHLL_SCORE", "trend", range(5, 21, 3), (0.0,)),
)
)
return tuple(
IndicatorVariant(i, p, q, r) for i, _, r, ps, qs in specs for p, q in product(ps, qs)
)
HISTORICAL_VARIANTS = _variants()
assert len(HISTORICAL_VARIANTS) == 1107
_names = {i: n for i, n, *_ in _SPECS}
_names.update({i: n for i, n, *_ in _COMPACT_SPECS})
_names.update(
{
variant.indicator_id: _names.get(variant.indicator_id, f"ID_{variant.indicator_id}")
for variant in HISTORICAL_VARIANTS
}
)
HISTORICAL_DEFINITIONS = tuple(
IndicatorDefinition(
i,
_names.get(i, f"RESERVED_{i}"),
next((v.role for v in HISTORICAL_VARIANTS if v.indicator_id == i), None),
)
for i in sorted({v.indicator_id for v in HISTORICAL_VARIANTS})
)

131
control_plane/trading_studio/indicators/engine.py Normal file
View file

@ -0,0 +1,131 @@
"""Formula-free execution boundary. It refuses rather than silently falling back."""
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
from typing import Any
import numpy as np
from .definitions import IndicatorVariant
from .feature_catalog import FeatureUnavailableError, require_validated_features
from .registry import HISTORICAL_REGISTRY, HS22_REQUIRED_V1, IndicatorRegistry
from .schema import HS22State, parse_hs22
class RefusalCode(StrEnum):
FORMULA_UNAVAILABLE = "formula_unavailable"
UNKNOWN_INDICATOR = "unknown_indicator"
UNSUPPORTED_VARIANT = "unsupported_variant"
UNREGISTERED = "unregistered"
@dataclass(frozen=True, slots=True)
class RefusalState:
code: RefusalCode
message: str
all_nan_behavior: bool = False
class IndicatorEngine:
def __init__(self, registry: IndicatorRegistry = HISTORICAL_REGISTRY) -> None:
self.registry = registry
def resolve(self, variant: IndicatorVariant) -> RefusalState:
definition = self.registry.definition(variant.indicator_id)
if definition is None:
return RefusalState(
RefusalCode.UNKNOWN_INDICATOR, "unknown ID returns all-NaN; no fallback", True
)
if definition.status.value == "unregistered":
return RefusalState(
RefusalCode.UNREGISTERED, "indicator is not registered for execution", True
)
if self.registry.variant(variant.indicator_id, variant.period, variant.p1) is None:
return RefusalState(
RefusalCode.UNSUPPORTED_VARIANT, "variant is outside the registry", False
)
return RefusalState(
RefusalCode.FORMULA_UNAVAILABLE, "indicator formulas are not implemented", False
)
def compute(self, variant: IndicatorVariant, length: int) -> RefusalState:
# This API intentionally does not emit a numeric fallback array.
if length < 0:
raise ValueError("length must be non-negative")
return self.resolve(variant)
def _arrays(close: Any, high: Any, low: Any, volume: Any) -> tuple[np.ndarray, ...]:
arrays = tuple(np.asarray(item, dtype=np.float64) for item in (close, high, low, volume))
if not arrays[0].ndim == 1 or any(item.shape != arrays[0].shape for item in arrays[1:]):
raise ValueError("HS22 OHLCV inputs must be equally sized one-dimensional arrays")
return arrays
def evaluate_indicator(
variant: IndicatorVariant, close: Any, high: Any, low: Any, volume: Any
) -> np.ndarray:
"""Evaluate exactly one frozen Cohort001 role triple; all others fail closed."""
try:
require_validated_features([variant])
except FeatureUnavailableError as error:
raise ValueError(str(error)) from error
from .historical_formulae import compute_indicator
close, high, low, volume = _arrays(close, high, low, volume)
return compute_indicator(
variant.indicator_id, close, high, low, volume, variant.period, variant.p1
)
def signal_color(values: Any) -> np.ndarray:
values = np.asarray(values, dtype=np.float64)
colors = np.zeros(len(values), dtype=np.int8)
previous = 0
for index in range(1, len(values)):
if np.isnan(values[index]) or np.isnan(values[index - 1]):
previous = 0
elif values[index] > values[index - 1]:
previous = 1
elif values[index] < values[index - 1]:
previous = -1
colors[index] = previous
return colors
def compute_hs22_state(
state: HS22State | object, close: Any, high: Any, low: Any, volume: Any
) -> dict[str, np.ndarray]:
"""Port of historical ``paper_replay.compute_combo_state`` without a reference import."""
if not isinstance(state, HS22State):
state = parse_hs22(state, HS22_REQUIRED_V1)
close, high, low, volume = _arrays(close, high, low, volume)
trend = evaluate_indicator(state.trend, close, high, low, volume)
signal = evaluate_indicator(state.signal, close, high, low, volume)
trigger = evaluate_indicator(state.trigger, close, high, low, volume)
confirm = evaluate_indicator(state.confirm, close, high, low, volume)
vol = evaluate_indicator(state.volatility, close, high, low, volume)
return {"trend": trend, "signal": signal, "trigger": trigger, "confirm": confirm, "vol": vol,
"trend_color": signal_color(trend), "signal_color": signal_color(signal)}
def analyze_corpus_coverage(
path: str, registry: IndicatorRegistry = HISTORICAL_REGISTRY
) -> dict[str, object]:
"""Inspect a parquet corpus when an optional parquet reader is installed."""
try:
import pyarrow.parquet as parquet # type: ignore[import-not-found]
except ImportError:
return {"available": False, "reason": "pyarrow is not installed"}
table = parquet.read_table(path)
fields = set(table.column_names)
required = {"open", "high", "low", "close", "volume"}
return {
"available": True,
"rows": table.num_rows,
"fields": sorted(fields),
"required_fields_present": required <= fields,
"registered_variants": len(registry.variants),
}

184
control_plane/trading_studio/indicators/feature_catalog.py Normal file
View file

@ -0,0 +1,184 @@
"""Source-only FEATURE_CATALOG_V1 and safety audit for historical features."""
from __future__ import annotations
import hashlib
import json
from dataclasses import asdict, dataclass
from enum import StrEnum
from pathlib import Path
from .definitions import IndicatorVariant
from .registry import HISTORICAL_REGISTRY, HISTORICAL_VARIANTS, HS22_REQUIRED_V1
FEATURE_CATALOG_V1_PATH = Path(__file__).with_name("feature_catalog_v1.json")
FEATURE_SAFETY_AUDIT_V1_PATH = Path(__file__).with_name("feature_safety_audit_v1.json")
class FeatureSupportState(StrEnum):
VALIDATED = "validated"
SOURCE_IMPLEMENTED_UNVALIDATED = "source_implemented_unvalidated"
ALIAS_REFUSED = "alias_refused"
UNREGISTERED = "unregistered"
class CausalClassification(StrEnum):
TRAILING_WINDOW_CAUSAL = "trailing_window_causal"
INDEX_POSITION_CAUSAL = "index_position_causal"
NOT_EXECUTABLE = "not_executable"
class FeatureUnavailableError(ValueError):
"""Raised when a requested historical feature lacks validated coverage."""
@dataclass(frozen=True, slots=True)
class FeatureResearchDefinition:
indicator_id: int
name: str
period: int
p1: float
role: str
support_state: FeatureSupportState
causal_classification: CausalClassification
source_semantics: str
@property
def key(self) -> tuple[int, int, float]:
return self.indicator_id, self.period, self.p1
def artifact(self) -> dict[str, object]:
return asdict(self)
# These are the 80 distinct formula inputs covered by the frozen Cohort001
# qualification corpus. A triple may appear in more than one role slot.
VALIDATED_FEATURE_KEYS = frozenset(
(item.indicator_id, item.period, item.p1) for item in HS22_REQUIRED_V1.variants
)
assert len(VALIDATED_FEATURE_KEYS) == 80
_INDEX_POSITION_IDS = frozenset({117, 118})
def _source_semantics(variant: IndicatorVariant) -> tuple[CausalClassification, str]:
if variant.indicator_id in _INDEX_POSITION_IDS:
return (
CausalClassification.INDEX_POSITION_CAUSAL,
"source derives a deterministic cycle from the row index; it does not read future rows",
)
return (
CausalClassification.TRAILING_WINDOW_CAUSAL,
"source reads the current row and preceding rows only; "
"completed-bar availability remains required",
)
def feature_definition(variant: IndicatorVariant) -> FeatureResearchDefinition:
definition = HISTORICAL_REGISTRY.definition(variant.indicator_id)
if definition is None:
return FeatureResearchDefinition(
variant.indicator_id, "UNKNOWN", variant.period, variant.p1, variant.role.value,
FeatureSupportState.UNREGISTERED, CausalClassification.NOT_EXECUTABLE,
"not present in the Artifex historical registry",
)
if definition.status.value == "alias":
return FeatureResearchDefinition(
variant.indicator_id, definition.name, variant.period, variant.p1, variant.role.value,
FeatureSupportState.ALIAS_REFUSED, CausalClassification.NOT_EXECUTABLE,
"historical alias semantics are intentionally not executable",
)
causal, semantics = _source_semantics(variant)
state = (
FeatureSupportState.VALIDATED
if (variant.indicator_id, variant.period, variant.p1) in VALIDATED_FEATURE_KEYS
else FeatureSupportState.SOURCE_IMPLEMENTED_UNVALIDATED
)
return FeatureResearchDefinition(
variant.indicator_id, definition.name, variant.period, variant.p1, variant.role.value,
state, causal, semantics,
)
FEATURE_CATALOG_V1 = tuple(feature_definition(variant) for variant in HISTORICAL_VARIANTS)
assert len(FEATURE_CATALOG_V1) == 1107
def require_validated_features(
variants: tuple[IndicatorVariant, ...] | list[IndicatorVariant],
) -> None:
"""Reject the complete request if any member is outside validated coverage."""
unavailable = []
for variant in variants:
feature = feature_definition(variant)
if feature.support_state != FeatureSupportState.VALIDATED:
unavailable.append(feature)
if unavailable:
requested = ", ".join(
f"{item.indicator_id}:{item.period}:{item.p1}:{item.role}" for item in unavailable
)
raise FeatureUnavailableError(
"FEATURE_CATALOG_V1 fails closed: unvalidated or non-executable features requested: "
+ requested
)
def feature_catalog_artifact() -> dict[str, object]:
return {
"schema_version": 1,
"catalog": "FEATURE_CATALOG_V1",
"source": "Artifex historical registry; source-semantics audit only",
"evaluation": "not evaluated via reference runtime; no strategy search",
"implementation_coverage": {
"validated_distinct_formula_variants": len(VALIDATED_FEATURE_KEYS),
"historical_registered_variants": len(HISTORICAL_VARIANTS),
"coverage": "80 of 1107",
},
"features": [item.artifact() for item in FEATURE_CATALOG_V1],
}
def feature_safety_audit_artifact() -> dict[str, object]:
catalog = feature_catalog_artifact()
return {
"schema_version": 1,
"audit": "FEATURE_SAFETY_AUDIT_V1",
"source": catalog["source"],
"evaluation": catalog["evaluation"],
"implementation_coverage": catalog["implementation_coverage"],
"support_states": [state.value for state in FeatureSupportState],
"causal_classifications": [classification.value for classification in CausalClassification],
"fail_closed_policy": (
"Any requested non-validated historical variant rejects the full request; "
"no fallback or baseline expansion."
),
"findings": [item.artifact() for item in FEATURE_CATALOG_V1],
}
def _artifact_bytes(payload: dict[str, object]) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
def feature_catalog_artifact_bytes() -> bytes:
return _artifact_bytes(feature_catalog_artifact())
def feature_safety_audit_artifact_bytes() -> bytes:
return _artifact_bytes(feature_safety_audit_artifact())
def feature_catalog_digest() -> str:
return hashlib.sha256(feature_catalog_artifact_bytes()).hexdigest()
def feature_safety_audit_digest() -> str:
return hashlib.sha256(feature_safety_audit_artifact_bytes()).hexdigest()
def write_feature_artifacts(
catalog_path: Path = FEATURE_CATALOG_V1_PATH,
audit_path: Path = FEATURE_SAFETY_AUDIT_V1_PATH,
) -> None:
catalog_path.write_bytes(feature_catalog_artifact_bytes() + b"\n")
audit_path.write_bytes(feature_safety_audit_artifact_bytes() + b"\n")

1
control_plane/trading_studio/indicators/feature_catalog_v1.json Normal file
View file

File diff suppressed because one or more lines are too long

1
control_plane/trading_studio/indicators/feature_safety_audit_v1.json Normal file
View file

File diff suppressed because one or more lines are too long

267
control_plane/trading_studio/indicators/historical_band_channel.py Normal file
View file

@ -0,0 +1,267 @@
"""Native Batch01 port of observed historical band/channel formulas.
This module is intentionally limited to the exact parameter triples observed in
the frozen engineering map. Oracle parity remains the gate for production
feature-catalog validation.
"""
from __future__ import annotations
import math
from collections.abc import Callable
from itertools import product
import numpy as np
NativeEvaluator = Callable[[np.ndarray, np.ndarray, np.ndarray, np.ndarray, int, float], np.ndarray]
def _sma(values: np.ndarray, period: int) -> np.ndarray:
out = np.full(len(values), np.nan)
if len(values) < period:
return out
total = sum(values[:period])
out[period - 1] = total / period
for index in range(period, len(values)):
total += values[index] - values[index - period]
out[index] = total / period
return out
def _ema(values: np.ndarray, period: int) -> np.ndarray:
out = np.full(len(values), np.nan)
if len(values) < period:
return out
alpha = 2.0 / (period + 1)
out[period - 1] = sum(values[:period]) / period
for index in range(period, len(values)):
out[index] = alpha * values[index] + (1 - alpha) * out[index - 1]
return out
def _rma(values: np.ndarray, period: int) -> np.ndarray:
out = np.full(len(values), np.nan)
total = 0.0
count = 0
start = -1
for index, value in enumerate(values):
if not np.isnan(value):
total += value
count += 1
if count == period:
start = index
break
if start < 0:
return out
out[start] = total / period
alpha = 1.0 / period
for index in range(start + 1, len(values)):
out[index] = (
out[index - 1]
if np.isnan(values[index])
else alpha * values[index] + (1 - alpha) * out[index - 1]
)
return out
def _true_range(high: np.ndarray, low: np.ndarray, close: np.ndarray) -> np.ndarray:
out = np.empty(len(close))
out[0] = high[0] - low[0]
for index in range(1, len(close)):
out[index] = max(
high[index] - low[index],
abs(high[index] - close[index - 1]),
abs(low[index] - close[index - 1]),
)
return out
def _atr(close: np.ndarray, high: np.ndarray, low: np.ndarray, period: int) -> np.ndarray:
return _rma(_true_range(high, low, close), period)
def bb_upper(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
multiplier = p1 if p1 > 0 else 2.0
average = _sma(close, period)
out = np.full(len(close), np.nan)
for index in range(period - 1, len(close)):
variance = sum(
(close[item] - average[index]) ** 2 for item in range(index - period + 1, index + 1)
)
out[index] = average[index] + multiplier * math.sqrt(variance / (period - 1))
return out
def bb_lower(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
multiplier = p1 if p1 > 0 else 2.0
average = _sma(close, period)
out = np.full(len(close), np.nan)
for index in range(period - 1, len(close)):
variance = sum(
(close[item] - average[index]) ** 2 for item in range(index - period + 1, index + 1)
)
out[index] = average[index] - multiplier * math.sqrt(variance / (period - 1))
return out
def supertrend(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
multiplier = p1 if p1 > 0 else 3.0
atr = _rma(_true_range(high, low, close), period)
out = np.full(len(close), np.nan)
upper = np.full(len(close), np.nan)
lower = np.full(len(close), np.nan)
direction = np.ones(len(close))
for index in range(period, len(close)):
if np.isnan(atr[index]):
continue
midpoint = (high[index] + low[index]) / 2.0
basic_upper = midpoint + multiplier * atr[index]
basic_lower = midpoint - multiplier * atr[index]
if index == period:
upper[index], lower[index] = basic_upper, basic_lower
out[index] = lower[index] if close[index] > lower[index] else upper[index]
direction[index] = 1.0 if close[index] > out[index] else -1.0
continue
upper[index] = (
min(basic_upper, upper[index - 1])
if not np.isnan(upper[index - 1]) and close[index - 1] <= upper[index - 1]
else basic_upper
)
lower[index] = (
max(basic_lower, lower[index - 1])
if not np.isnan(lower[index - 1]) and close[index - 1] >= lower[index - 1]
else basic_lower
)
if direction[index - 1] == 1.0:
out[index] = lower[index] if close[index] >= lower[index] else upper[index]
else:
out[index] = upper[index] if close[index] <= upper[index] else lower[index]
direction[index] = 1.0 if close[index] > out[index] else -1.0
return out
def donch_upper(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
out = np.full(len(high), np.nan)
for index in range(period - 1, len(high)):
out[index] = max(high[index - period + 1 : index + 1])
return out
def donch_lower(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
out = np.full(len(low), np.nan)
for index in range(period - 1, len(low)):
out[index] = min(low[index - period + 1 : index + 1])
return out
def kelt_upper(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
multiplier = p1 if p1 > 0 else 1.5
basis = _ema(close, period)
atr = _atr(close, high, low, period)
out = np.full(len(close), np.nan)
valid = ~np.isnan(basis) & ~np.isnan(atr)
out[valid] = basis[valid] + multiplier * atr[valid]
return out
def kelt_lower(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
multiplier = p1 if p1 > 0 else 1.5
basis = _ema(close, period)
atr = _atr(close, high, low, period)
out = np.full(len(close), np.nan)
valid = ~np.isnan(basis) & ~np.isnan(atr)
out[valid] = basis[valid] - multiplier * atr[valid]
return out
def psar(
close: np.ndarray, high: np.ndarray, low: np.ndarray, volume: np.ndarray, period: int, p1: float
) -> np.ndarray:
acceleration_step = p1 if p1 > 0 else 0.02
out = np.full(len(close), np.nan)
if len(close) < 2:
return out
bullish, acceleration, acceleration_maximum = True, 0.02, 0.2
extreme, sar = high[0], low[0]
for index in range(1, len(close)):
previous_sar = sar
sar = previous_sar + acceleration * (extreme - previous_sar)
if bullish:
sar = min(sar, low[index - 1])
if index >= 2:
sar = min(sar, low[index - 2])
if low[index] < sar:
bullish, sar, extreme, acceleration = False, extreme, low[index], 0.02
elif high[index] > extreme:
extreme = high[index]
acceleration = min(acceleration + acceleration_step, acceleration_maximum)
else:
sar = max(sar, high[index - 1])
if index >= 2:
sar = max(sar, high[index - 2])
if high[index] > sar:
bullish, sar, extreme, acceleration = True, extreme, high[index], 0.02
elif low[index] < extreme:
extreme = low[index]
acceleration = min(acceleration + acceleration_step, acceleration_maximum)
out[index] = sar
return out
_BOLLINGER = tuple(product((14, 16, 18, 20, 22, 24), (1.5, 2.0, 2.5)))
_KELTNER = tuple(product((14, 16, 18, 20, 22, 24), (1.0, 1.5, 2.0)))
OBSERVED_BAND_CHANNEL_PARAMS = frozenset(
(indicator_id, period, p1)
for indicator_id, params in (
(17, _BOLLINGER),
(18, _BOLLINGER),
(19, product((8, 10, 12, 14), (2.0, 3.0, 4.0))),
(20, ((period, 0.0) for period in (10, 14, 18, 22, 26))),
(21, ((period, 0.0) for period in (10, 14, 18, 22, 26))),
(23, _KELTNER),
(24, _KELTNER),
(28, ((1, p1) for p1 in (0.01, 0.02, 0.03))),
)
for period, p1 in params
)
NATIVE_EVALUATORS: dict[int, NativeEvaluator] = {
17: bb_upper,
18: bb_lower,
19: supertrend,
20: donch_upper,
21: donch_lower,
23: kelt_upper,
24: kelt_lower,
28: psar,
}
def evaluate_band_channel(
indicator_id: int,
close: np.ndarray,
high: np.ndarray,
low: np.ndarray,
volume: np.ndarray,
period: int,
p1: float,
) -> np.ndarray:
"""Evaluate one exact Batch01 observed primitive, refusing grid expansion."""
key = indicator_id, period, p1
if key not in OBSERVED_BAND_CHANNEL_PARAMS:
raise ValueError(f"unsupported Batch01 band_channel variant: {indicator_id}:{period}:{p1}")
return NATIVE_EVALUATORS[indicator_id](close, high, low, volume, period, p1)

2391
control_plane/trading_studio/indicators/historical_formulae.py Normal file
View file

File diff suppressed because it is too large Load diff

1
control_plane/trading_studio/indicators/hs22_historical_registry_v1.json Normal file
View file

File diff suppressed because one or more lines are too long

242
control_plane/trading_studio/indicators/parity_harness_v1.py Normal file
View file

@ -0,0 +1,242 @@
"""FEATURE_PARITY_HARNESS_V1 contracts; formula ports are deliberately external."""
from __future__ import annotations
import hashlib
import json
from collections.abc import Callable, Iterable, Mapping
from dataclasses import asdict, dataclass, replace
from enum import StrEnum
from pathlib import Path
from typing import Any
import numpy as np
ROOT = Path(__file__).resolve().parents[3]
FROZEN_ENGINEERING_MAP_V1 = ROOT / "archaeology" / "engineering_family_map_v1.json"
FROZEN_ENGINEERING_MAP_V1_SHA256 = (
"314a87ca468dd80e6f8bf1eaaa3e6350771ad5f53ee4f3f80db7c8949b2de1ab"
)
class ParityStatus(StrEnum):
PASS = "pass"
FAIL = "fail"
NOT_IMPLEMENTED = "not_implemented"
@dataclass(frozen=True, slots=True)
class FeatureVersion:
"""One immutable implementation candidate in a parity run.
Coverage is intentionally calculated from the versions submitted to one
run, never from the historical registry or an implied implementation pool.
"""
feature_version: str
primitive_key: str
engineering_family: str
usage_slots: int
evaluator: Callable[..., np.ndarray] | None = None
status: ParityStatus = ParityStatus.NOT_IMPLEMENTED
@dataclass(frozen=True, slots=True)
class ComparisonRecord:
name: str
status: ParityStatus
expected_dtype: str
actual_dtype: str
expected_shape: tuple[int, ...]
actual_shape: tuple[int, ...]
nan_mask_mismatches: int
value_mismatches: int
reason: str | None = None
@property
def passed(self) -> bool:
return self.status == ParityStatus.PASS
def artifact(self) -> dict[str, object]:
return asdict(self)
class FrozenArtifactError(ValueError):
pass
def _digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def load_frozen_engineering_map(path: Path = FROZEN_ENGINEERING_MAP_V1) -> dict[str, object]:
"""Load only the pinned engineering map; reject substituted archaeology."""
if _digest(path) != FROZEN_ENGINEERING_MAP_V1_SHA256:
raise FrozenArtifactError(
"ENGINEERING_FAMILY_MAP_V1 digest does not match the frozen artifact"
)
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("artifact") != "ENGINEERING_FAMILY_MAP_V1" or payload.get("schema_version") != 1:
raise FrozenArtifactError("invalid ENGINEERING_FAMILY_MAP_V1 manifest")
return payload
def parity_manifest() -> dict[str, object]:
"""Return the reproducible run contract without claiming any formula parity."""
engineering_map = load_frozen_engineering_map()
return {
"schema_version": 1,
"artifact": "FEATURE_PARITY_HARNESS_V1_MANIFEST",
"frozen_inputs": {
"engineering_family_map": {
"path": "archaeology/engineering_family_map_v1.json",
"sha256": FROZEN_ENGINEERING_MAP_V1_SHA256,
"primitive_count": engineering_map["counts"]["primitives"],
}
},
"oracle_windows": {"execution": "spark_offline", "network_access": "forbidden"},
"comparison": {
"dtype": "exact",
"shape": "exact",
"nan_mask": "exact",
"state_keys": "exact",
"tolerance_by_engineering_family": {},
"unknown_family_policy": "fail_closed_no_tolerance",
},
"coverage": "recomputed solely from submitted FeatureVersions with status=pass",
"scope": "architecture only; no new formula ports",
}
def enumerate_native_evaluators(
feature_versions: Iterable[FeatureVersion],
) -> tuple[FeatureVersion, ...]:
"""Return parameterized native evaluators and reject duplicate version IDs."""
versions = tuple(feature_versions)
ids = [item.feature_version for item in versions]
if len(ids) != len(set(ids)):
raise ValueError("FeatureVersion identifiers must be unique")
return tuple(item for item in versions if item.evaluator is not None)
def compare_array(name: str, expected: Any, actual: Any) -> ComparisonRecord:
"""Compare a feature output without numerical tolerance or dtype coercion."""
expected_array, actual_array = np.asarray(expected), np.asarray(actual)
dtype_match = expected_array.dtype == actual_array.dtype
shape_match = expected_array.shape == actual_array.shape
if not shape_match:
return ComparisonRecord(
name, ParityStatus.FAIL, str(expected_array.dtype), str(actual_array.dtype),
expected_array.shape, actual_array.shape, 0, 0, "shape_mismatch",
)
expected_nan = (
np.isnan(expected_array)
if np.issubdtype(expected_array.dtype, np.inexact)
else np.zeros(expected_array.shape, bool)
)
actual_nan = (
np.isnan(actual_array)
if np.issubdtype(actual_array.dtype, np.inexact)
else np.zeros(actual_array.shape, bool)
)
nan_mismatches = int(np.count_nonzero(expected_nan != actual_nan))
values_match = dtype_match and np.array_equal(expected_array, actual_array, equal_nan=True)
matching_values = (expected_array == actual_array) | (expected_nan & actual_nan)
value_mismatches = (
0
if values_match
else int(expected_array.size - np.count_nonzero(matching_values))
)
return ComparisonRecord(
name, ParityStatus.PASS if values_match else ParityStatus.FAIL,
str(expected_array.dtype),
str(actual_array.dtype),
expected_array.shape,
actual_array.shape,
nan_mismatches,
value_mismatches,
None if values_match else "dtype_or_value_mismatch",
)
def compare_float64_ulp(
name: str,
expected: Any,
actual: Any,
*,
max_ulps: int,
allowed_indexes: frozenset[int],
) -> ComparisonRecord:
"""Accept only explicitly listed finite float64 differences within a ULP bound."""
exact = compare_array(name, expected, actual)
if exact.passed:
return exact
expected_array, actual_array = np.asarray(expected), np.asarray(actual)
if (
expected_array.dtype != np.dtype(np.float64)
or actual_array.dtype != np.dtype(np.float64)
or expected_array.shape != actual_array.shape
or exact.nan_mask_mismatches
):
return exact
mismatches = np.flatnonzero(
~((expected_array == actual_array) | (np.isnan(expected_array) & np.isnan(actual_array)))
)
if (
not len(mismatches)
or not np.all(np.isfinite(expected_array[mismatches]))
or not np.all(np.isfinite(actual_array[mismatches]))
or any(int(index) not in allowed_indexes for index in mismatches)
):
return exact
for index in mismatches:
expected_bits = int(np.asarray(expected_array[index]).view(np.uint64))
actual_bits = int(np.asarray(actual_array[index]).view(np.uint64))
expected_ordered = _ordered_float64_bits(expected_bits)
actual_ordered = _ordered_float64_bits(actual_bits)
if abs(expected_ordered - actual_ordered) > max_ulps:
return exact
return replace(exact, status=ParityStatus.PASS, reason="accepted_documented_float64_ulp_drift")
def _ordered_float64_bits(bits: int) -> int:
return ~bits & ((1 << 64) - 1) if bits >> 63 else bits | (1 << 63)
def compare_state(
expected: Mapping[str, Any], actual: Mapping[str, Any],
) -> tuple[ComparisonRecord, ...]:
"""Produce auditable exact records for every state key, including missing keys."""
records: list[ComparisonRecord] = []
for name in sorted(set(expected) | set(actual)):
if name not in expected or name not in actual:
missing = expected.get(name, actual.get(name))
array = np.asarray(missing)
records.append(ComparisonRecord(
name, ParityStatus.FAIL, str(array.dtype) if name in expected else "<missing>",
str(array.dtype) if name in actual else "<missing>",
array.shape if name in expected else (),
array.shape if name in actual else (),
0,
0,
"state_key_mismatch",
))
else:
records.append(compare_array(name, expected[name], actual[name]))
return tuple(records)
def recompute_coverage(feature_versions: Iterable[FeatureVersion]) -> dict[str, object]:
"""Compute coverage only over the explicitly submitted FeatureVersions."""
versions = tuple(feature_versions)
total = sum(item.usage_slots for item in versions)
passed = tuple(item for item in versions if item.status == ParityStatus.PASS)
passed_slots = sum(item.usage_slots for item in passed)
return {
"submitted_feature_versions": len(versions),
"passed_feature_versions": len(passed),
"submitted_usage_slots": total,
"passed_usage_slots": passed_slots,
"coverage_percent": 0.0 if total == 0 else passed_slots * 100 / total,
"feature_versions": [item.feature_version for item in passed],
}

193
control_plane/trading_studio/indicators/registry.py Normal file
View file

@ -0,0 +1,193 @@
"""Deterministic registry/universe construction and artifact serialization."""
from __future__ import annotations
import hashlib
import json
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
from .definitions import (
HISTORICAL_DEFINITIONS,
HISTORICAL_VARIANTS,
ImplementationStatus,
IndicatorDefinition,
IndicatorVariant,
Role,
)
HISTORICAL_ARTIFACT_PATH = Path(__file__).with_name("hs22_historical_registry_v1.json")
@dataclass(frozen=True, slots=True)
class IndicatorUniverse:
name: str
variants: tuple[IndicatorVariant, ...]
namespace: str = "historical"
def by_role(self, role: Role | str) -> tuple[IndicatorVariant, ...]:
return tuple(variant for variant in self.variants if variant.role == Role(role))
def contains(self, variant: IndicatorVariant) -> bool:
return variant in self.variants
@dataclass(frozen=True, slots=True)
class IndicatorRegistry:
definitions: tuple[IndicatorDefinition, ...]
variants: tuple[IndicatorVariant, ...]
def definition(self, indicator_id: int) -> IndicatorDefinition | None:
return next((item for item in self.definitions if item.indicator_id == indicator_id), None)
def variant(self, indicator_id: int, period: int, p1: float = 0.0) -> IndicatorVariant | None:
return next(
(
item
for item in self.variants
if (item.indicator_id, item.period, item.p1) == (indicator_id, period, p1)
),
None,
)
def role_pool(self, role: Role | str) -> tuple[IndicatorVariant, ...]:
return HS22_HISTORICAL_COMPLETE_V1.by_role(role)
def _all_definitions() -> tuple[IndicatorDefinition, ...]:
# Historical aliases remain metadata only. Formula execution is unavailable.
existing = {item.indicator_id: item for item in HISTORICAL_DEFINITIONS}
aliases = {25: 1, 26: None, 27: None}
for indicator_id, target in aliases.items():
base = existing[indicator_id]
existing[indicator_id] = IndicatorDefinition(
indicator_id,
base.name,
base.role,
ImplementationStatus.ALIAS,
target,
"historical_alias; execution_refused",
)
definitions = tuple(sorted(existing.values(), key=lambda item: item.indicator_id))
assert len(definitions) == 145
return definitions
HISTORICAL_REGISTRY = IndicatorRegistry(_all_definitions(), HISTORICAL_VARIANTS)
HS22_HISTORICAL_COMPLETE_V1 = IndicatorUniverse("HS22_HISTORICAL_COMPLETE_V1", HISTORICAL_VARIANTS)
EXPERIMENTAL_UNIVERSE = IndicatorUniverse("HS22_EXPERIMENTAL_V1", (), "experimental")
UNREGISTERED_INDICATORS = {
"MOM_JERK": "unregistered; unknown_dispatch_returns_all_nan; no_fallback",
}
def cohort001_universe(manifest: object | None = None) -> IndicatorUniverse:
"""Build the derived universe only from canonical manifest variants.
A missing manifest intentionally produces an empty placeholder; it never
substitutes the broad historical pool.
"""
variants: list[IndicatorVariant] = []
if isinstance(manifest, dict):
rows = manifest.get("variants", ())
else:
rows = ()
for row in rows:
if not isinstance(row, dict):
continue
try:
candidate = IndicatorVariant(
int(row["indicator_id"]),
int(row["period"]),
float(row.get("p1", 0.0)),
Role(row["role"]),
)
except (KeyError, TypeError, ValueError):
continue
if candidate in HISTORICAL_VARIANTS:
variants.append(candidate)
return IndicatorUniverse("HS22_COHORT001_V1", tuple(variants), "derived")
HS22_COHORT001_V1 = cohort001_universe()
# This is the complete, role-sensitive Cohort001 execution surface recovered
# from the frozen oracle. It is product data, not an oracle dependency.
_HS22_REQUIRED_TRIPLES = (
(17, 16, 2.0, "signal"), (18, 16, 2.0, "signal"), (18, 22, 2.5, "signal"), (18, 24, 2.5, "signal"),
(19, 8, 4.0, "signal"), (19, 10, 2.0, "signal"), (19, 10, 4.0, "signal"), (19, 14, 3.0, "signal"),
(21, 10, 0.0, "trigger"), (21, 14, 0.0, "trigger"), (21, 18, 0.0, "signal"), (21, 22, 0.0, "trigger"),
(24, 14, 1.0, "signal"), (30, 14, 0.0, "confirm"), (31, 5, 0.0, "confirm"), (31, 7, 0.0, "confirm"),
(31, 21, 0.0, "confirm"), (33, 16, 0.0, "confirm"), (33, 20, 0.0, "confirm"), (33, 22, 0.0, "confirm"),
(33, 24, 0.0, "confirm"), (36, 10, 0.0, "confirm"), (36, 18, 0.0, "confirm"), (36, 20, 0.0, "confirm"),
(39, 20, 0.0, "vol"), (40, 12, 0.0, "vol"), (40, 14, 0.0, "vol"), (41, 12, 0.0, "vol"),
(41, 16, 0.0, "vol"), (41, 22, 0.0, "vol"), (42, 20, 0.0, "vol"), (42, 44, 0.0, "confirm"),
(50, 8, 0.0, "trend"), (53, 18, 0.0, "vol"), (53, 30, 0.0, "vol"), (54, 22, 0.0, "trend"),
(54, 30, 0.0, "trend"), (55, 10, 0.0, "vol"), (57, 10, 0.0, "trend"), (58, 30, 0.0, "trend"),
(61, 14, 0.0, "trend"), (63, 11, 0.0, "trend"), (70, 18, 0.0, "trend"), (70, 20, 0.0, "trend"),
(71, 14, 0.0, "trend"), (71, 18, 0.0, "trend"), (72, 18, 0.0, "trend"), (73, 20, 0.0, "trend"),
(75, 26, 0.0, "trend"), (76, 23, 0.0, "trend"), (77, 1, 0.0, "vol"), (79, 30, 2.0, "trigger"),
(79, 30, 8.0, "trend"), (79, 40, 8.0, "trigger"), (79, 50, 2.0, "trigger"), (79, 50, 8.0, "trigger"),
(79, 60, 2.0, "trend"), (79, 60, 2.0, "trigger"), (81, 25, 0.0, "trigger"), (82, 30, 3.0, "trigger"),
(82, 40, 1.0, "trigger"), (91, 8, 0.0, "signal"), (91, 11, 0.0, "signal"), (92, 5, 0.0, "signal"),
(92, 17, 0.0, "signal"), (101, 8, 0.0, "trend"), (106, 8, 0.0, "confirm"), (106, 14, 0.0, "confirm"),
(112, 30, 0.0, "vol"), (113, 30, 0.0, "vol"), (114, 20, 10.0, "vol"), (114, 30, 50.0, "vol"),
(122, 60, 0.0, "trigger"), (122, 90, 0.0, "trigger"), (122, 150, 0.0, "trigger"), (122, 180, 0.0, "trigger"),
(122, 210, 0.0, "trigger"), (124, 60, 0.0, "signal"), (129, 22, 2.0, "signal"), (129, 30, 2.0, "signal"),
(152, 15, 0.0, "vol"),
)
HS22_REQUIRED_V1 = IndicatorUniverse(
"HS22_COHORT001_REQUIRED_V1",
tuple(
IndicatorVariant(
indicator_id,
period,
p1,
{"trend": Role.TREND, "signal": Role.OSC, "trigger": Role.LEVEL,
"confirm": Role.OSC, "vol": Role.FILTER}[slot],
)
for indicator_id, period, p1, slot in _HS22_REQUIRED_TRIPLES
),
"derived",
)
def historical_artifact() -> dict[str, object]:
role_counts = Counter(item.role.value for item in HISTORICAL_VARIANTS)
return {
"schema_version": 1,
"registry": "HS22_HISTORICAL_COMPLETE_V1",
"definition_count": len(HISTORICAL_REGISTRY.definitions),
"variant_count": len(HISTORICAL_VARIANTS),
"role_counts": dict(sorted(role_counts.items())),
"definitions": [
{
"indicator_id": item.indicator_id,
"name": item.name,
"role": item.role.value if item.role else None,
"status": item.status.value,
"alias_of": item.alias_of,
"behavior": item.behavior,
}
for item in HISTORICAL_REGISTRY.definitions
],
"variants": [item.asdict() for item in HISTORICAL_VARIANTS],
"unknown_behavior": "all_nan; no_fallback",
"unregistered": UNREGISTERED_INDICATORS,
}
def historical_artifact_bytes() -> bytes:
return json.dumps(
historical_artifact(), sort_keys=True, separators=(",", ":"), allow_nan=False
).encode()
def historical_artifact_digest() -> str:
return hashlib.sha256(historical_artifact_bytes()).hexdigest()
def write_historical_artifact(path: Path = HISTORICAL_ARTIFACT_PATH) -> None:
path.write_bytes(historical_artifact_bytes() + b"\n")

76
control_plane/trading_studio/indicators/schema.py Normal file
View file

@ -0,0 +1,76 @@
"""Strict schema parser and state container for a five-role HS22 selection."""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from .definitions import IndicatorVariant, Role
from .registry import HS22_HISTORICAL_COMPLETE_V1, IndicatorUniverse
_ROLE_ORDER = (Role.TREND, Role.OSC, Role.LEVEL, Role.OSC, Role.FILTER)
_ROLE_NAMES = ("trend", "signal", "trigger", "confirm", "volatility")
class HS22SchemaError(ValueError):
pass
@dataclass(frozen=True, slots=True)
class HS22State:
trend: IndicatorVariant
signal: IndicatorVariant
trigger: IndicatorVariant
confirm: IndicatorVariant
volatility: IndicatorVariant
tail: tuple[float, ...] = ()
def variants(self) -> tuple[IndicatorVariant, ...]:
return (self.trend, self.signal, self.trigger, self.confirm, self.volatility)
def _parse_variant(
value: object, expected_role: Role, universe: IndicatorUniverse
) -> IndicatorVariant:
if isinstance(value, Mapping):
try:
variant = IndicatorVariant(
int(value["indicator_id"]),
int(value["period"]),
float(value.get("p1", 0.0)),
Role(value["role"]),
)
except (KeyError, TypeError, ValueError) as error:
raise HS22SchemaError("invalid HS22 variant") from error
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 3:
try:
variant = IndicatorVariant(int(value[0]), int(value[1]), float(value[2]), expected_role)
except (TypeError, ValueError) as error:
raise HS22SchemaError("invalid positional HS22 variant") from error
else:
raise HS22SchemaError("HS22 variant must be an object or [id, period, p1]")
if variant.role != expected_role:
raise HS22SchemaError(f"HS22 role must be {expected_role.value}")
if not universe.contains(variant):
raise HS22SchemaError("unsupported HS22 variant; no fallback is permitted")
return variant
def parse_hs22(
value: object, universe: IndicatorUniverse = HS22_HISTORICAL_COMPLETE_V1
) -> HS22State:
if isinstance(value, Mapping):
items = tuple(
_parse_variant(value.get(name), role, universe)
for name, role in zip(_ROLE_NAMES, _ROLE_ORDER, strict=True)
)
tail = tuple(float(number) for number in value.get("tail", ()))
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 22:
items = tuple(
_parse_variant(value[index : index + 3], role, universe)
for index, role in zip(range(0, 15, 3), _ROLE_ORDER, strict=True)
)
tail = tuple(float(number) for number in value[15:])
else:
raise HS22SchemaError("HS22 must be a role mapping or exactly 22 positional values")
return HS22State(*items, tail=tail)

192
control_plane/trading_studio/management/commands/canary_hyperscalper_cohort_001.py Normal file
View file

@ -0,0 +1,192 @@
from __future__ import annotations
import json
from pathlib import Path
import numpy as np
from django.core.management.base import BaseCommand, CommandError
from control_plane.trading_studio.cohort_materialization import (
COHORT_MANIFEST,
DATASET_CSV,
DATASET_MANIFEST,
file_digest,
load_csv,
reconstruct_folds,
validate_dataset_manifest,
validate_manifest,
validate_rows,
)
from control_plane.trading_studio.indicators import compute_hs22_state
from control_plane.trading_studio.management.commands.materialize_hyperscalper_cohort_001 import (
DATASET_SHA256,
DATASET_VERSION_ID,
)
from control_plane.trading_studio.management.commands.materialize_hyperscalper_cohort_001 import (
Command as MaterializeCommand,
)
from control_plane.trading_studio.models import TradingCohort
from control_plane.trading_studio.qualification import (
Bar,
CalibrationMethod,
calibrate_train_only,
qualification_scenario_catalog,
)
from control_plane.trading_studio.services import TradingStudioService
class Command(BaseCommand):
help = "Run the native HS22 Fold 1 baseline canary for frozen Cohort001."
def add_arguments(self, parser):
parser.add_argument("artifact_root")
parser.add_argument("--cohort-id", required=True)
def handle(self, *args, **options):
root = Path(options["artifact_root"])
try:
cohort_manifest = self._json(root / COHORT_MANIFEST)
dataset_manifest = self._json(root / DATASET_MANIFEST)
validate_manifest(cohort_manifest, cohort=True)
validate_dataset_manifest(dataset_manifest)
dataset_path = root / DATASET_CSV
if dataset_manifest["sha256"] != file_digest(dataset_path):
raise ValueError(
"Dataset CSV file hash does not match the frozen dataset manifest."
)
rows, headers = load_csv(
dataset_path, dataset_manifest.get("timestamp_field", "timestamp")
)
validate_rows(rows, headers, dataset_manifest)
cohort = TradingCohort.objects.get(pk=options["cohort_id"])
memberships = list(cohort.memberships.order_by("ordinal"))
self._validate_reconstruction(
cohort,
memberships,
cohort_manifest,
dataset_manifest,
rows,
)
membership = memberships[0]
fold = cohort.policy_snapshot["reconstruction_v1"]["folds"][0]
state = self._native_state(rows, membership.strategy_version.genome)
train = self._bars(rows, state, fold["train"])
replay = self._bars(rows, state, fold["test"])
calibration = calibrate_train_only(
train,
method=CalibrationMethod.RAW,
fold=fold["fold"],
train_start=train[0].at,
train_end=rows[fold["embargo"]["start_index"]]["timestamp"],
source_record_ids=(str(membership.id),),
source_combo=json.dumps(
membership.strategy_version.genome["combo"],
separators=(",", ":"),
),
)
run = TradingStudioService().run_qualification_canary(
strategy_version=membership.strategy_version,
dataset_version=cohort.dataset_version,
fold=fold["fold"],
scenario=qualification_scenario_catalog()["BASELINE"],
runner_name="canary_hyperscalper_cohort_001",
bars=replay,
calibration=calibration,
)
except (
KeyError,
TradingCohort.DoesNotExist,
OSError,
ValueError,
json.JSONDecodeError,
) as error:
raise CommandError(str(error)) from error
self.stdout.write(json.dumps({"qualification_run_id": str(run.id), "mode": "CANARY_ONLY"}))
@staticmethod
def _json(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
@staticmethod
def _validate_reconstruction(cohort, memberships, cohort_manifest, dataset_manifest, rows):
reconstruction = cohort.policy_snapshot.get("reconstruction_v1")
if not isinstance(reconstruction, dict):
raise ValueError("Cohort has no validated reconstruction_v1 policy.")
if (
cohort.dataset_version_id != DATASET_VERSION_ID
or dataset_manifest["sha256"] != DATASET_SHA256
or reconstruction.get("cohort_manifest", {}).get("manifest_sha256")
!= cohort_manifest["manifest_sha256"]
or reconstruction.get("dataset_manifest", {}).get("sha256")
!= dataset_manifest["sha256"]
or reconstruction.get("provenance") != "RECONSTRUCTED_FROM_FROZEN_SPEC_V1"
or reconstruction.get("folds")
!= reconstruct_folds(rows, dataset_manifest.get("timestamp_field", "timestamp"))
):
raise ValueError("Cohort reconstruction_v1 does not match the validated artifact.")
members = cohort_manifest["members"]
if len(memberships) != 20 or [member.ordinal for member in memberships] != list(
range(1, 21)
):
raise ValueError("Cohort does not contain the required 20 ordered memberships.")
for membership, member in zip(memberships, members, strict=True):
if (
membership.id != MaterializeCommand._member_id(member)
or membership.strategy_version.fingerprint
!= MaterializeCommand._fingerprint(member)
or membership.metadata.get("selection_position") != member["rank"]
):
raise ValueError("Cohort membership does not match the frozen semantic member.")
@staticmethod
def _native_state(rows, genome):
try:
combo = genome["combo"]
except (KeyError, TypeError) as error:
raise ValueError("First frozen strategy genome has no HS22 combo.") from error
if not isinstance(combo, (list, tuple)) or len(combo) != 22:
raise ValueError("First frozen strategy genome has no 22-field HS22 combo.")
close, high, low, volume = (
np.asarray([row[field] for row in rows], dtype=np.float64)
for field in ("close", "high", "low", "volume")
)
state = compute_hs22_state(combo, close, high, low, volume)
valid = (
np.isfinite(state["trigger"])
& np.isfinite(state["confirm"])
& np.isfinite(state["vol"])
)
volatility = state["vol"][np.isfinite(state["vol"])]
if not len(volatility):
raise ValueError("Native HS22 state contains no finite volatility values.")
threshold = np.sort(volatility)[
min(int(float(combo[21]) / 100 * len(volatility)), len(volatility) - 1)
]
valid &= state["vol"] >= threshold
signal = state["signal_color"]
trend = state["trend_color"]
flips = (signal != np.roll(signal, 1)) & (signal != 0)
long = valid & flips & (signal == 1) & (trend == 1) & (close > state["trigger"])
long &= (float(combo[17]) <= state["confirm"]) & (state["confirm"] <= float(combo[18]))
short = valid & flips & (signal == -1) & (trend == -1) & (close < state["trigger"])
short &= (float(combo[19]) <= state["confirm"]) & (state["confirm"] <= float(combo[20]))
state["decision"] = long.astype(np.int8) - short.astype(np.int8)
return state
@staticmethod
def _bars(rows, state, window):
start, end = window["start_index"], window["end_index"] + 1
names = ("trend", "signal", "trigger", "confirm", "vol", "trend_color", "signal_color")
return [
Bar(
at=row["timestamp"],
open=row["open"],
high=row["high"],
low=row["low"],
close=row["close"],
signal=int(state["decision"][index]) or None,
context=tuple(float(state[name][index]) for name in names),
reference=f"hs22-native:{index}",
)
for index, row in enumerate(rows[start:end], start=start)
]

395
control_plane/trading_studio/management/commands/cohort_qualification_hyperscalper_001.py Normal file
View file

@ -0,0 +1,395 @@
from __future__ import annotations
import json
from collections import Counter
from dataclasses import asdict, fields
from pathlib import Path
from typing import Any
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from control_plane.trading_studio.cohort_materialization import (
COHORT_MANIFEST,
DATASET_CSV,
DATASET_MANIFEST,
digest,
file_digest,
load_csv,
validate_dataset_manifest,
validate_manifest,
validate_rows,
)
from control_plane.trading_studio.management.commands.canary_hyperscalper_cohort_001 import (
Command as CanaryCommand,
)
from control_plane.trading_studio.management.commands.specimen_hyperscalper_cohort_001 import (
Command as SpecimenCommand,
SCENARIO_NAMES,
)
from control_plane.trading_studio.models import QualificationReplayRun, TradingCohort
from control_plane.trading_studio.qualification import (
CalibrationMethod,
DEQPrimitives,
calibrate_train_only,
qualification_scenario_catalog,
)
from control_plane.trading_studio.services import TradingStudioService
RUNNER_NAME = "cohort_qualification_hyperscalper_001:v1"
RUNNER_PREFIX = "cohort_qualification_hyperscalper_001"
CONTRACT = "Cohort001-qualification-v1"
SPECIMEN_CONTRACT = "Cohort001-first-member-specimen-v1"
class Command(BaseCommand):
help = "Qualify frozen HyperScalper Cohort001 members 2 through 20 with persisted HS22 replays."
def add_arguments(self, parser):
parser.add_argument("artifact_root")
parser.add_argument("--cohort-id", required=True)
parser.add_argument("--specimen-report-path", required=True)
parser.add_argument("--specimen-report-sha", required=True)
parser.add_argument("--report-path", default="qualification_cohort_001_v1.json")
parser.add_argument("--indicator-expansion-path", default="indicator_expansion_brief.md")
def handle(self, *args, **options):
root = Path(options["artifact_root"])
specimen_path = Path(options["specimen_report_path"])
report_path = Path(options["report_path"])
brief_path = Path(options["indicator_expansion_path"])
try:
cohort_manifest = self._json(root / COHORT_MANIFEST)
dataset_manifest = self._json(root / DATASET_MANIFEST)
validate_manifest(cohort_manifest, cohort=True)
validate_dataset_manifest(dataset_manifest)
dataset_path = root / DATASET_CSV
if dataset_manifest["sha256"] != file_digest(dataset_path):
raise ValueError("Dataset CSV file hash does not match the frozen dataset manifest.")
rows, headers = load_csv(
dataset_path, dataset_manifest.get("timestamp_field", "timestamp")
)
validate_rows(rows, headers, dataset_manifest)
specimen_sha256 = file_digest(specimen_path)
expected_specimen_sha256 = options["specimen_report_sha"]
if (
len(expected_specimen_sha256) != 64
or any(character not in "0123456789abcdef" for character in expected_specimen_sha256)
or specimen_sha256 != expected_specimen_sha256
):
raise ValueError("Specimen report SHA does not match the externally frozen SHA.")
specimen = self._json(specimen_path)
catalog = qualification_scenario_catalog()
if tuple(catalog) != SCENARIO_NAMES or len(catalog) != 13:
raise ValueError(
"QualificationReplayV1 must provide the frozen 13-scenario catalog."
)
config_sha256 = digest({name: asdict(spec) for name, spec in catalog.items()})
self._validate_specimen(specimen)
protocol_identity = {
"cohort_manifest_sha256": cohort_manifest["manifest_sha256"],
"dataset_sha256": dataset_manifest["sha256"],
"qualification_scenario_catalog_sha256": config_sha256,
"specimen_report_sha256": specimen_sha256,
"runner_name": RUNNER_NAME,
}
cohort = TradingCohort.objects.get(pk=options["cohort_id"])
memberships = list(cohort.memberships.select_related("strategy_version").order_by("ordinal"))
CanaryCommand._validate_reconstruction(
cohort, memberships, cohort_manifest, dataset_manifest, rows
)
self._validate_runner_versions(memberships, cohort.dataset_version_id)
except (
KeyError,
TradingCohort.DoesNotExist,
OSError,
ValueError,
json.JSONDecodeError,
) as error:
raise CommandError(str(error)) from error
member_reports = []
try:
# A cohort qualification is all-or-nothing; reruns reuse the same immutable identity.
with transaction.atomic():
service = TradingStudioService()
for membership in memberships[1:]:
state = CanaryCommand._native_state(
rows, membership.strategy_version.genome
)
folds = []
for fold in cohort.policy_snapshot["reconstruction_v1"]["folds"]:
train = CanaryCommand._bars(rows, state, fold["train"])
replay = CanaryCommand._bars(rows, state, fold["test"])
calibration = self._calibration(membership, fold, rows, train)
scenarios = []
for name in SCENARIO_NAMES:
run, reused = self._run_or_reuse(
service,
membership,
cohort,
fold,
catalog[name],
replay,
calibration,
protocol_identity,
)
scenario = SpecimenCommand._scenario_report(run)
scenario["persistence"] = "REUSED" if reused else "CREATED"
scenarios.append(scenario)
folds.append({"fold": fold["fold"], "scenarios": scenarios})
member_reports.append(
{
"ordinal": membership.ordinal,
"member_id": str(membership.id),
"strategy_version_id": str(membership.strategy_version_id),
"folds": folds,
}
)
except ValueError as error:
raise CommandError(str(error)) from error
report = self._report(
cohort,
member_reports,
cohort_manifest["manifest_sha256"],
dataset_manifest["sha256"],
config_sha256,
specimen_sha256,
)
brief = self._indicator_brief(report["machine"]["indicator_expansion"])
try:
for path, payload in ((report_path, json.dumps(report, sort_keys=True, indent=2)),):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(payload, encoding="utf-8")
brief_path.parent.mkdir(parents=True, exist_ok=True)
brief_path.write_text(brief, encoding="utf-8")
except OSError as error:
raise CommandError(str(error)) from error
self.stdout.write(json.dumps(report, sort_keys=True, ensure_ascii=True))
@staticmethod
def _json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
@staticmethod
def _validate_specimen(report: dict[str, Any]) -> None:
machine = report.get("machine", {})
if (
machine.get("contract") != SPECIMEN_CONTRACT
or machine.get("member_ordinal") != 1
or machine.get("fold_count") != 4
or machine.get("scenario_count_per_fold") != 13
):
raise ValueError(
"Specimen report is not the frozen Cohort001 ordinal-one 4x13 report."
)
folds = machine.get("folds")
if not isinstance(folds, list) or len(folds) != 4:
raise ValueError("Legacy specimen report is missing its four fold records.")
required_scenario = {
"qualification_run_id",
"scenario",
"summary",
"ledger",
"ledger_reconciliation",
"deq",
"rescue",
}
required_ledger = {
"gross_pnl",
"net_pnl",
"entry_commission",
"exit_commission",
"entry_slippage",
"exit_slippage",
"funding",
"other_cost",
"deq",
}
for fold in folds:
scenarios = fold.get("scenarios") if isinstance(fold, dict) else None
if not isinstance(fold, dict) or not fold.get("fold") or not isinstance(scenarios, list):
raise ValueError("Legacy specimen report fold fields are incomplete.")
if len(scenarios) != 13 or [item.get("scenario") for item in scenarios] != list(
SCENARIO_NAMES
):
raise ValueError("Legacy specimen report scenario catalog does not match the frozen 13.")
for scenario in scenarios:
ledger = scenario.get("ledger") if isinstance(scenario, dict) else None
if not isinstance(scenario, dict) or required_scenario - set(scenario):
raise ValueError("Legacy specimen report scenario fields are incomplete.")
if not isinstance(scenario["summary"], dict) or not isinstance(ledger, list):
raise ValueError("Legacy specimen report scenario summary or ledger is invalid.")
if any(not isinstance(row, dict) or required_ledger - set(row) for row in ledger):
raise ValueError("Legacy specimen report ledger fields are incomplete.")
@staticmethod
def _validate_runner_versions(memberships, dataset_version_id) -> None:
strategy_ids = [member.strategy_version_id for member in memberships[1:]]
versions = QualificationReplayRun.objects.filter(
strategy_version_id__in=strategy_ids,
dataset_version_id=dataset_version_id,
runner_name__startswith=RUNNER_PREFIX,
).exclude(runner_name=RUNNER_NAME)
if versions.exists():
raise ValueError("Qualification cohort refuses mixed runner versions.")
@staticmethod
def _calibration(membership, fold, rows, train):
return calibrate_train_only(
train,
method=CalibrationMethod.RAW,
fold=fold["fold"],
train_start=train[0].at,
train_end=rows[fold["embargo"]["start_index"]]["timestamp"],
source_record_ids=(str(membership.id),),
source_combo=json.dumps(
membership.strategy_version.genome["combo"], separators=(",", ":")
),
allow_empty_raw=True,
)
@staticmethod
def _run_or_reuse(
service, membership, cohort, fold, scenario, replay, calibration, protocol_identity
):
existing = QualificationReplayRun.objects.filter(
strategy_version=membership.strategy_version,
dataset_version=cohort.dataset_version,
fold=fold["fold"],
scenario_name=scenario.name,
runner_name=RUNNER_NAME,
input_hash=calibration.input_hash,
).first()
expected_configuration = TradingStudioService.qualification_configuration(
scenario,
protocol_identity=protocol_identity,
require_protocol_identity=True,
)
if existing:
if (
existing.replay_mode != "CANARY_ONLY"
or existing.calibration
!= TradingStudioService._json_safe(asdict(calibration))
or existing.configuration != expected_configuration
):
raise ValueError("Persisted qualification identity has incompatible protocol inputs.")
return existing, True
return (
service.run_qualification_canary(
strategy_version=membership.strategy_version,
dataset_version=cohort.dataset_version,
fold=fold["fold"],
scenario=scenario,
runner_name=RUNNER_NAME,
bars=replay,
calibration=calibration,
protocol_identity=protocol_identity,
require_protocol_identity=True,
),
False,
)
@staticmethod
def _report(
cohort, members, manifest_sha256, dataset_sha256, config_sha256, specimen_sha256
):
scenarios = [
scenario
for member in members
for fold in member["folds"]
for scenario in fold["scenarios"]
]
expansion = Command._indicator_expansion(scenarios)
machine = {
"contract": CONTRACT,
"cohort_id": str(cohort.id),
"member_ordinals": [member["ordinal"] for member in members],
"fold_count": 4,
"scenario_count_per_fold": 13,
"expected_persisted_runs": len(members) * 4 * 13,
"protocol_identity": {
"cohort_manifest_sha256": manifest_sha256,
"dataset_sha256": dataset_sha256,
"qualification_scenario_catalog_sha256": config_sha256,
"specimen_report_sha256": specimen_sha256,
"runner_name": RUNNER_NAME,
},
"members": members,
"indicator_expansion": expansion,
}
return {
"machine": machine,
"human": {
"scope": "Cohort001 ordinals 2 through 20; ordinal 1 remains the frozen specimen.",
"persisted_runs": len(scenarios),
"ledger_reconciled": all(
scenario["ledger_reconciliation"]["consistent"] for scenario in scenarios
),
"runner_version": RUNNER_NAME,
"indicator_expansion_status": expansion["status"],
},
}
@staticmethod
def _indicator_expansion(scenarios):
ledger = [row for scenario in scenarios for row in scenario["ledger"]]
exit_classes = Counter(row["exit_reason"] for row in ledger)
direction_classes = Counter(
"LONG" if row["take_profit_price"] > row["entry_execution_price"] else "SHORT"
for row in ledger
)
pnl_classes = Counter(
"WIN" if row["net_pnl"] > 0 else "LOSS" if row["net_pnl"] < 0 else "BREAKEVEN"
for row in ledger
)
deq = {}
for field in fields(DEQPrimitives):
values = [
row["deq"][field.name]
for row in ledger
if row["deq"][field.name] is not None
]
payload = {"available_count": len(values), "missing_count": len(ledger) - len(values)}
if values and isinstance(values[0], bool):
payload["classes"] = {"true": sum(values), "false": len(values) - sum(values)}
elif values:
payload["distribution"] = {
"min": min(values), "max": max(values), "mean": sum(values) / len(values)
}
else:
payload["status"] = "INSUFFICIENT"
deq[field.name] = payload
return {
"status": "AVAILABLE" if ledger else "INSUFFICIENT",
"ledger_trade_count": len(ledger),
"classes": {
"exit_reason": dict(sorted(exit_classes.items())),
"direction": dict(sorted(direction_classes.items())),
"net_pnl": dict(sorted(pnl_classes.items())),
},
"deq_distributions": deq,
}
@staticmethod
def _indicator_brief(expansion):
lines = [
"# Indicator Expansion Brief",
"",
f"Status: {expansion['status']}",
f"Persisted ledger trades: {expansion['ledger_trade_count']}",
"",
"## Required classes",
]
for name, values in expansion["classes"].items():
lines.append(f"- {name}: {json.dumps(values, sort_keys=True)}")
lines.extend(["", "## DEQ distributions"])
for name, value in expansion["deq_distributions"].items():
lines.append(f"- {name}: {json.dumps(value, sort_keys=True)}")
if expansion["status"] == "INSUFFICIENT":
lines.extend(
["", "No ledger observations were emitted; distributions are insufficient."]
)
return "\n".join(lines) + "\n"

221
control_plane/trading_studio/management/commands/materialize_hyperscalper_cohort_001.py Normal file
View file

@ -0,0 +1,221 @@
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from uuid import NAMESPACE_URL, UUID, uuid5
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from control_plane.trading_studio.cohort_materialization import (
COHORT_MANIFEST,
DATASET_CSV,
DATASET_MANIFEST,
PROVENANCE,
file_digest,
load_csv,
reconstruct_folds,
validate_dataset_manifest,
validate_manifest,
validate_rows,
validate_selection_source,
)
from control_plane.trading_studio.models import (
CohortMembership,
MarketDatasetVersion,
Strategy,
StrategyVersion,
TradingCohort,
)
DATASET_VERSION_ID = UUID("1f224db9-cbb8-4aec-8575-98c4b0279a83")
DATASET_SHA256 = "7fb56bb05b0edc2348407765cd693efadf334307c25877dfb52acfe4221e5d00"
UUID_NAMESPACE = uuid5(NAMESPACE_URL, "artifex:hyperscalper-revalidation-001")
class Command(BaseCommand):
help = "Recover frozen HyperScalper Cohort 001 from its CSV artifact."
def add_arguments(self, parser):
parser.add_argument("artifact_root")
parser.add_argument("--selection-source", required=True)
parser.add_argument("--cohort-id", required=True, type=UUID)
def handle(self, *args, **options):
root = Path(options["artifact_root"])
selection_path = Path(options["selection_source"])
try:
cohort_manifest = self._json(root / COHORT_MANIFEST)
dataset_manifest = self._json(root / DATASET_MANIFEST)
validate_manifest(cohort_manifest, cohort=True)
validate_dataset_manifest(dataset_manifest)
if dataset_manifest["sha256"] != DATASET_SHA256:
raise ValueError(
"Dataset manifest SHA does not match the authorized frozen artifact."
)
selection = self._json(selection_path)
selection_sha256 = validate_selection_source(cohort_manifest, selection, selection_path)
rows, headers = load_csv(
root / DATASET_CSV, dataset_manifest.get("timestamp_field", "timestamp")
)
if dataset_manifest["sha256"] != file_digest(root / DATASET_CSV):
raise ValueError(
"Dataset CSV file hash does not match the frozen dataset manifest."
)
validate_rows(rows, headers, dataset_manifest)
folds = reconstruct_folds(rows, dataset_manifest.get("timestamp_field", "timestamp"))
strategies = cohort_manifest["members"]
if len(strategies) != 20 or any(len(member["combo"]) != 22 for member in strategies):
raise ValueError(
"Frozen cohort must contain 20 ordered members with 22-field combos."
)
if not isinstance(selection, dict):
raise ValueError("Selection source must be a JSON object.")
except (KeyError, OSError, ValueError, json.JSONDecodeError) as error:
raise CommandError(str(error)) from error
snapshot = {
"cohort_manifest": cohort_manifest,
"dataset_manifest": dataset_manifest,
"folds": folds,
"provenance": PROVENANCE,
"artifact_refs": {
"cohort_manifest": f"artifact://{COHORT_MANIFEST}",
"dataset_manifest": f"artifact://{DATASET_MANIFEST}",
"dataset": f"artifact://{DATASET_CSV}",
"selection_source": cohort_manifest["source_report"],
"selection_source_observed_sha256": selection_sha256,
},
}
with transaction.atomic():
version = self._dataset(root, dataset_manifest, rows)
project = version.dataset.trading_project
try:
cohort = TradingCohort.objects.get(pk=options["cohort_id"])
except TradingCohort.DoesNotExist as error:
raise CommandError("The authorized pre-existing cohort does not exist.") from error
if (
cohort.dataset_version_id != version.id
or self._historic_manifest_sha(cohort.policy_snapshot)
!= cohort_manifest["manifest_sha256"]
):
raise CommandError(
"Existing cohort frozen core identity conflicts with the artifact."
)
policy = dict(cohort.policy_snapshot)
if policy.get("reconstruction_v1") != snapshot:
policy["reconstruction_v1"] = snapshot
cohort.policy_snapshot = policy
cohort.save(update_fields=["policy_snapshot", "updated_at"])
for ordinal, item in enumerate(strategies, start=1):
strategy, version = self._strategy(project, item)
member_id = self._member_id(item)
member, member_created = CohortMembership.objects.get_or_create(
id=member_id,
defaults={
"cohort": cohort,
"strategy_version": version,
"ordinal": ordinal,
"member_id": member_id,
"provenance": PROVENANCE,
"metadata": {
"selection_position": item["rank"],
"source_strategy_id": item["source_strategy_id"],
"base_family": item["base_family"],
"combo": item["combo"],
},
},
)
if not member_created and (
member.cohort_id != cohort.id
or member.strategy_version_id != version.id
or member.ordinal != ordinal
):
raise CommandError("Frozen CohortMembership ID exists with different content.")
self.stdout.write(json.dumps({"cohort_id": str(cohort.id), "reconstructed": True}))
@staticmethod
def _json(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
@staticmethod
def _dataset(root: Path, manifest: dict, rows: list[dict]):
try:
version = MarketDatasetVersion.objects.select_related("dataset__trading_project").get(
pk=DATASET_VERSION_ID
)
except MarketDatasetVersion.DoesNotExist as error:
raise CommandError(
"The authorized pre-existing dataset version does not exist."
) from error
if (
version.dataset.trading_project.slug != "crypto-hyperscalper"
or version.content_hash != manifest["sha256"]
or version.record_count != len(rows)
or Path(version.reference).resolve() != (root / DATASET_CSV).resolve()
or version.fields != manifest["fields"]
or version.start_at != rows[0]["timestamp"]
or version.end_at != rows[-1]["timestamp"]
or int(rows[0]["timestamp"].timestamp()) != manifest["first_timestamp"]
or int(rows[-1]["timestamp"].timestamp()) != manifest["last_timestamp"]
):
raise CommandError(
"Existing dataset version does not match the frozen artifact identity."
)
return version
@staticmethod
def _historic_manifest_sha(policy: dict) -> str | None:
return (
policy.get("manifest_sha256")
or policy.get("cohort_manifest", {}).get("manifest_sha256")
or policy.get("frozen_manifest", {}).get("manifest_sha256")
)
@staticmethod
def _strategy(project, item):
fingerprint = Command._fingerprint(item)
strategy, _ = Strategy.objects.get_or_create(
id=uuid5(UUID_NAMESPACE, f"strategy:{fingerprint}"),
defaults={
"trading_project": project,
"name": item["source_name"],
"strategy_class": "HS22",
},
)
version_defaults = {
"strategy": strategy,
"version": "frozen-v1",
"genome": {"base_family": item["base_family"], "combo": item["combo"]},
"fingerprint": fingerprint,
}
version, created = StrategyVersion.objects.get_or_create(
id=uuid5(UUID_NAMESPACE, f"strategy-version:{fingerprint}"),
defaults=version_defaults,
)
if created:
version.immutable = True
version.save(update_fields=["immutable", "updated_at"])
elif not version.immutable or any(
getattr(version, field) != value for field, value in version_defaults.items()
):
raise CommandError("Frozen strategy version exists with different immutable identity.")
if strategy.trading_project_id != project.id or version.strategy_id != strategy.id:
raise CommandError("Frozen strategy ID exists with different content.")
return strategy, version
@staticmethod
def _fingerprint(member: dict) -> str:
projection = {
"selection_position": member["rank"],
"source_strategy_id": member["source_strategy_id"],
"base_family": member["base_family"],
"combo": member["combo"],
}
payload = json.dumps(projection, sort_keys=True, ensure_ascii=True, separators=(",", ":"))
return hashlib.sha256(payload.encode()).hexdigest()
@staticmethod
def _member_id(member: dict) -> UUID:
return uuid5(UUID_NAMESPACE, f"membership:{Command._fingerprint(member)}")

236
control_plane/trading_studio/management/commands/specimen_hyperscalper_cohort_001.py Normal file
View file

@ -0,0 +1,236 @@
from __future__ import annotations
import json
from dataclasses import fields
from pathlib import Path
from typing import Any
from django.core.management.base import BaseCommand, CommandError
from control_plane.trading_studio.cohort_materialization import (
COHORT_MANIFEST,
DATASET_CSV,
DATASET_MANIFEST,
file_digest,
load_csv,
validate_dataset_manifest,
validate_manifest,
validate_rows,
)
from control_plane.trading_studio.management.commands.canary_hyperscalper_cohort_001 import (
Command as CanaryCommand,
)
from control_plane.trading_studio.models import TradingCohort
from control_plane.trading_studio.qualification import (
CalibrationMethod,
DEQPrimitives,
calibrate_train_only,
qualification_scenario_catalog,
)
from control_plane.trading_studio.services import TradingStudioService
SCENARIO_NAMES = (
"BASELINE",
"SLIPPAGE_ADVERSE_2BP",
"TOTAL_COST_6BP",
"TOTAL_COST_10BP",
"TOTAL_COST_15BP",
"PESSIMISTIC_SAME_BAR",
"ENTRY_DELAY_PLUS_ONE_BAR",
"WEEKDAYS_ONLY",
"ALL_DAYS",
"STOP_WIDTH_100",
"STOP_WIDTH_75",
"STOP_WIDTH_50",
"STOP_WIDTH_25",
)
RUNNER_NAME = "specimen_hyperscalper_cohort_001"
class Command(BaseCommand):
help = "Persist and report the first Cohort001 member across all reconstructed folds."
def add_arguments(self, parser):
parser.add_argument("artifact_root")
parser.add_argument("--cohort-id", required=True)
parser.add_argument("--report-path", required=True)
def handle(self, *args, **options):
root = Path(options["artifact_root"])
report_path = Path(options["report_path"])
try:
cohort_manifest = self._json(root / COHORT_MANIFEST)
dataset_manifest = self._json(root / DATASET_MANIFEST)
validate_manifest(cohort_manifest, cohort=True)
validate_dataset_manifest(dataset_manifest)
dataset_path = root / DATASET_CSV
if dataset_manifest["sha256"] != file_digest(dataset_path):
raise ValueError(
"Dataset CSV file hash does not match the frozen dataset manifest."
)
rows, headers = load_csv(
dataset_path, dataset_manifest.get("timestamp_field", "timestamp")
)
validate_rows(rows, headers, dataset_manifest)
cohort = TradingCohort.objects.get(pk=options["cohort_id"])
memberships = list(cohort.memberships.order_by("ordinal"))
CanaryCommand._validate_reconstruction(
cohort, memberships, cohort_manifest, dataset_manifest, rows
)
membership = memberships[0] # Specimen scope is deliberately limited to rank one.
state = CanaryCommand._native_state(rows, membership.strategy_version.genome)
catalog = qualification_scenario_catalog()
if tuple(catalog) != SCENARIO_NAMES:
raise ValueError(
"QualificationReplayV1 scenario catalog differs from specimen contract."
)
fold_reports = []
service = TradingStudioService()
for fold in cohort.policy_snapshot["reconstruction_v1"]["folds"]:
train = CanaryCommand._bars(rows, state, fold["train"])
replay = CanaryCommand._bars(rows, state, fold["test"])
calibration = calibrate_train_only(
train,
method=CalibrationMethod.RAW,
fold=fold["fold"],
train_start=train[0].at,
train_end=rows[fold["embargo"]["start_index"]]["timestamp"],
source_record_ids=(str(membership.id),),
source_combo=json.dumps(
membership.strategy_version.genome["combo"], separators=(",", ":")
),
)
scenarios = []
for name in SCENARIO_NAMES:
run = service.run_qualification_canary(
strategy_version=membership.strategy_version,
dataset_version=cohort.dataset_version,
fold=fold["fold"],
scenario=catalog[name],
runner_name=RUNNER_NAME,
bars=replay,
calibration=calibration,
)
scenarios.append(self._scenario_report(run))
fold_reports.append({"fold": fold["fold"], "scenarios": scenarios})
report = self._report(cohort, membership, fold_reports)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(
json.dumps(report, sort_keys=True, indent=2, ensure_ascii=True), encoding="utf-8"
)
except (
KeyError,
TradingCohort.DoesNotExist,
OSError,
ValueError,
json.JSONDecodeError,
) as error:
raise CommandError(str(error)) from error
self.stdout.write(json.dumps(report, sort_keys=True, ensure_ascii=True))
@staticmethod
def _json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
@staticmethod
def _scenario_report(run) -> dict[str, Any]:
ledger = [row.result for row in run.ledger_rows.order_by("sequence")]
reconciliation = Command._reconcile(run.summary, ledger)
return {
"qualification_run_id": str(run.id),
"scenario": run.scenario_name,
"summary": run.summary,
"ledger": ledger,
"ledger_reconciliation": reconciliation,
"deq": Command._deq_aggregate(ledger),
"rescue": run.summary["rescue"],
}
@staticmethod
def _reconcile(summary: dict[str, Any], ledger: list[dict[str, Any]]) -> dict[str, Any]:
sums = {
"gross_pnl": sum(row["gross_pnl"] for row in ledger),
"net_pnl": sum(row["net_pnl"] for row in ledger),
"commissions": sum(row["entry_commission"] + row["exit_commission"] for row in ledger),
"slippage": sum(row["entry_slippage"] + row["exit_slippage"] for row in ledger),
"funding": sum(row["funding"] for row in ledger),
"other_cost": sum(row["other_cost"] for row in ledger),
}
rescue = summary["rescue"]
rows_consistent = all(
abs(
row["net_pnl"]
- (
row["gross_pnl"]
- row["entry_commission"]
- row["exit_commission"]
- row["funding"]
- row["other_cost"]
)
)
<= 1e-9
for row in ledger
)
consistent = (
rows_consistent
and summary["ledger_count"] == len(ledger)
and all(
abs(summary[name] - value) <= 1e-9
for name, value in sums.items()
if name in summary
)
and all(abs(rescue[name] - value) <= 1e-9 for name, value in sums.items())
)
return {
"consistent": consistent,
"row_pnl_consistent": rows_consistent,
"ledger_count": len(ledger),
"totals": sums,
}
@staticmethod
def _deq_aggregate(ledger: list[dict[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {"trade_count": len(ledger), "fields": {}}
for field in fields(DEQPrimitives):
name = field.name
values = [row["deq"][name] for row in ledger if row["deq"][name] is not None]
if values and isinstance(values[0], bool):
aggregate = {"true_count": sum(values), "false_count": len(values) - sum(values)}
else:
aggregate = {"mean": sum(values) / len(values) if values else None}
result["fields"][name] = {"available_count": len(values), **aggregate}
return result
@staticmethod
def _report(cohort, membership, folds: list[dict[str, Any]]) -> dict[str, Any]:
criteria = cohort.policy_snapshot.get("edge_vs_monetization_criteria")
edge_status = "INCOMPLETE" if isinstance(criteria, dict) and criteria else "INSUFFICIENT"
machine = {
"contract": "Cohort001-first-member-specimen-v1",
"cohort_id": str(cohort.id),
"member_ordinal": membership.ordinal,
"member_id": str(membership.id),
"strategy_version_id": str(membership.strategy_version_id),
"dataset_version_id": str(cohort.dataset_version_id),
"fold_count": len(folds),
"scenario_count_per_fold": len(SCENARIO_NAMES),
"folds": folds,
"edge_vs_monetization": {"status": edge_status, "criteria": criteria or None},
}
return {
"machine": machine,
"human": {
"scope": (
"Cohort001 member ordinal 1 only; Cohort002 and all other members excluded."
),
"folds_persisted": len(folds),
"scenarios_per_fold": len(SCENARIO_NAMES),
"ledger_reconciled": all(
scenario["ledger_reconciliation"]["consistent"]
for fold in folds for scenario in fold["scenarios"]
),
"edge_vs_monetization": edge_status,
},
}

24
control_plane/trading_studio/management/commands/tradingstudio_generate_batch01_native_acceptance_manifest.py Normal file
View file

@ -0,0 +1,24 @@
"""Write the resolved Batch01 native acceptance manifest and coverage."""
from __future__ import annotations
from pathlib import Path
from django.core.management.base import BaseCommand
from control_plane.trading_studio.indicators.batch01_native_parity import (
write_batch01_native_acceptance_manifest,
)
class Command(BaseCommand):
help = "Evaluate Batch01 and write its scoped-ULP acceptance manifest."
def add_arguments(self, parser):
parser.add_argument("--output", type=Path, required=True)
def handle(self, *args, **options):
output = options["output"]
output.parent.mkdir(parents=True, exist_ok=True)
write_batch01_native_acceptance_manifest(output)
self.stdout.write(str(output))

24
control_plane/trading_studio/management/commands/tradingstudio_generate_batch01_native_parity_manifest.py Normal file
View file

@ -0,0 +1,24 @@
"""Write the immutable local parity contract for Batch01 native evaluators."""
from __future__ import annotations
from pathlib import Path
from django.core.management.base import BaseCommand
from control_plane.trading_studio.indicators.batch01_native_parity import (
write_batch01_native_parity_manifest,
)
class Command(BaseCommand):
help = "Write the frozen Batch01 native parity manifest."
def add_arguments(self, parser):
parser.add_argument("--output", type=Path, required=True)
def handle(self, *args, **options):
output = options["output"]
output.parent.mkdir(parents=True, exist_ok=True)
write_batch01_native_parity_manifest(output)
self.stdout.write(str(output))

24
control_plane/trading_studio/management/commands/tradingstudio_generate_batch01_oracle_manifest.py Normal file
View file

@ -0,0 +1,24 @@
"""Write the Batch01 request manifest for an external historical oracle."""
from __future__ import annotations
from pathlib import Path
from django.core.management.base import BaseCommand
from control_plane.trading_studio.indicators.batch01_oracle_manifest import (
write_batch01_oracle_request_manifest,
)
class Command(BaseCommand):
help = "Write 97 exact Batch01 historical-oracle requests; never evaluates formulas."
def add_arguments(self, parser):
parser.add_argument("--output", type=Path, required=True)
def handle(self, *args, **options):
output = options["output"]
output.parent.mkdir(parents=True, exist_ok=True)
write_batch01_oracle_request_manifest(output)
self.stdout.write(str(output))

42
control_plane/trading_studio/management/commands/tradingstudio_generate_feature_catalog.py Normal file
View file

@ -0,0 +1,42 @@
from __future__ import annotations
import json
from pathlib import Path
from django.core.management.base import BaseCommand
from control_plane.trading_studio.indicators.feature_catalog import (
FEATURE_CATALOG_V1_PATH,
FEATURE_SAFETY_AUDIT_V1_PATH,
feature_catalog_digest,
feature_safety_audit_digest,
write_feature_artifacts,
)
class Command(BaseCommand):
help = "Generate deterministic FEATURE_CATALOG_V1 and FEATURE_SAFETY_AUDIT_V1 artifacts."
def add_arguments(self, parser):
parser.add_argument("--output-dir", type=Path)
def handle(self, *args, **options):
output_dir = options["output_dir"]
catalog_path = (
output_dir / FEATURE_CATALOG_V1_PATH.name if output_dir else FEATURE_CATALOG_V1_PATH
)
audit_path = (
output_dir / FEATURE_SAFETY_AUDIT_V1_PATH.name
if output_dir
else FEATURE_SAFETY_AUDIT_V1_PATH
)
if output_dir:
output_dir.mkdir(parents=True, exist_ok=True)
write_feature_artifacts(catalog_path, audit_path)
self.stdout.write(json.dumps({
"catalog": str(catalog_path),
"catalog_sha256": feature_catalog_digest(),
"audit": str(audit_path),
"audit_sha256": feature_safety_audit_digest(),
"coverage": "80 validated of 1107 historical variants",
}, sort_keys=True))

70
control_plane/trading_studio/management/commands/tradingstudio_generate_oracle_windows.py Normal file
View file

@ -0,0 +1,70 @@
"""Generate offline Spark input windows for an external historical oracle."""
from __future__ import annotations
import json
from pathlib import Path
from django.core.management.base import BaseCommand, CommandError
from control_plane.trading_studio.indicators.parity_harness_v1 import parity_manifest
class Command(BaseCommand):
help = "Materialize deterministic offline Spark oracle windows; it never evaluates formulas."
def add_arguments(self, parser):
parser.add_argument("--input-csv", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--window-size", type=int, default=4096)
parser.add_argument("--spark-master", default="local[*]")
parser.add_argument("--order-column", default="timestamp")
def handle(self, *args, **options):
if options["window_size"] <= 0:
raise CommandError("--window-size must be positive")
try:
from pyspark.sql import SparkSession, Window
from pyspark.sql.functions import col, countDistinct, row_number
except ImportError as error:
raise CommandError("pyspark is required; this command has no local fallback") from error
source = options["input_csv"]
if not source.is_file():
raise CommandError("--input-csv must name a local file")
output_dir = options["output_dir"]
output_dir.mkdir(parents=True, exist_ok=True)
spark = (
SparkSession.builder.master(options["spark_master"])
.appName("artifex-oracle-windows")
.getOrCreate()
)
try:
frame = spark.read.option("header", True).option("inferSchema", True).csv(str(source))
order_column = options["order_column"]
if order_column not in frame.columns:
raise CommandError("--order-column must exist in --input-csv")
row_count = frame.count()
distinct_count = frame.select(countDistinct(col(order_column))).first()[0]
if row_count != distinct_count:
raise CommandError("--order-column must be unique for reproducible Spark windows")
ordered = Window.orderBy(col(order_column))
row_index = row_number().over(ordered) - 1
window_id = (row_index / options["window_size"]).cast("long")
windowed = frame.withColumn("oracle_window_id", window_id)
windowed.write.mode("errorifexists").parquet(str(output_dir / "windows.parquet"))
finally:
spark.stop()
(output_dir / "oracle_window_manifest_v1.json").write_text(
json.dumps(
{
**parity_manifest(),
"input_csv": str(source),
"window_size": options["window_size"],
"order_column": options["order_column"],
},
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)
self.stdout.write(str(output_dir / "oracle_window_manifest_v1.json"))

102
control_plane/trading_studio/migrations/0002_qualification_replay_v1.py Normal file
View file

@ -0,0 +1,102 @@
# Generated manually for QualificationReplayV1.
import uuid
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("trading_studio", "0001_initial")]
operations = [
migrations.CreateModel(
name="QualificationReplayRun",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4, editable=False, primary_key=True, serialize=False
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("fold", models.CharField(max_length=120)),
("scenario_name", models.CharField(max_length=160)),
("runner_name", models.CharField(max_length=160)),
("replay_mode", models.CharField(default="CANARY_ONLY", max_length=32)),
("input_hash", models.CharField(max_length=128)),
("calibration", models.JSONField(default=dict)),
("configuration", models.JSONField(default=dict)),
("summary", models.JSONField(default=dict)),
(
"dataset_version",
models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="qualification_runs",
to="trading_studio.marketdatasetversion",
),
),
(
"strategy_version",
models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
related_name="qualification_runs",
to="trading_studio.strategyversion",
),
),
],
),
migrations.CreateModel(
name="QualificationReplayLedger",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4, editable=False, primary_key=True, serialize=False
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("sequence", models.PositiveIntegerField()),
("identity", models.CharField(max_length=512)),
("result", models.JSONField(default=dict)),
(
"qualification_run",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="ledger_rows",
to="trading_studio.qualificationreplayrun",
),
),
],
),
migrations.AddConstraint(
model_name="qualificationreplayrun",
constraint=models.UniqueConstraint(
fields=(
"strategy_version",
"dataset_version",
"fold",
"scenario_name",
"runner_name",
"input_hash",
),
name="unique_qualification_replay_identity",
),
),
migrations.AddConstraint(
model_name="qualificationreplayledger",
constraint=models.UniqueConstraint(
fields=("qualification_run", "sequence"),
name="unique_qualification_ledger_sequence",
),
),
migrations.AddConstraint(
model_name="qualificationreplayledger",
constraint=models.UniqueConstraint(
fields=("qualification_run", "identity"),
name="unique_qualification_ledger_identity",
),
),
]

31
control_plane/trading_studio/migrations/0003_cohort_membership.py Normal file
View file

@ -0,0 +1,31 @@
# Generated manually for frozen cohort membership.
import uuid
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("trading_studio", "0002_qualification_replay_v1")]
operations = [
migrations.CreateModel(
name="CohortMembership",
fields=[
("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("ordinal", models.PositiveIntegerField()),
("member_id", models.UUIDField(unique=True)),
("provenance", models.CharField(max_length=80)),
("metadata", models.JSONField(default=dict)),
("cohort", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name="memberships", to="trading_studio.tradingcohort")),
("strategy_version", models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name="cohort_memberships", to="trading_studio.strategyversion")),
],
),
migrations.AddConstraint(
model_name="cohortmembership",
constraint=models.UniqueConstraint(fields=("cohort", "ordinal"), name="unique_cohort_member_ordinal"),
),
]

58
control_plane/trading_studio/models.py
View file

@ -237,6 +237,28 @@ class TradingCohort(TimestampedModel):
report = models.ForeignKey("TradingResearchReport", on_delete=models.SET_NULL, null=True, blank=True, related_name="cohorts")
class CohortMembership(TimestampedModel):
"""An ordered, immutable strategy membership imported from a frozen cohort spec."""
cohort = models.ForeignKey(TradingCohort, on_delete=models.CASCADE, related_name="memberships")
strategy_version = models.ForeignKey(StrategyVersion, on_delete=models.PROTECT, related_name="cohort_memberships")
ordinal = models.PositiveIntegerField()
member_id = models.UUIDField(unique=True)
provenance = models.CharField(max_length=80)
metadata = models.JSONField(default=dict)
class Meta:
constraints = [models.UniqueConstraint(fields=["cohort", "ordinal"], name="unique_cohort_member_ordinal")]
def save(self, *args, **kwargs):
if self.pk and type(self).objects.filter(pk=self.pk).exists():
original = type(self).objects.get(pk=self.pk)
fields = ["cohort_id", "strategy_version_id", "ordinal", "member_id", "provenance", "metadata"]
if any(getattr(self, field) != getattr(original, field) for field in fields):
raise ValueError("CohortMembership is immutable after frozen-spec materialization.")
super().save(*args, **kwargs)
class StrategyExperiment(TimestampedModel):
cohort = models.ForeignKey(TradingCohort, on_delete=models.CASCADE, related_name="experiments")
strategy_version = models.ForeignKey(StrategyVersion, on_delete=models.PROTECT, related_name="experiments")
@ -266,6 +288,42 @@ class BacktestRun(TimestampedModel):
failure_details = models.TextField(blank=True)
class QualificationReplayRun(TimestampedModel):
"""Persisted CANARY_ONLY qualification replay and its immutable inputs."""
strategy_version = models.ForeignKey(StrategyVersion, on_delete=models.PROTECT, related_name="qualification_runs")
dataset_version = models.ForeignKey(MarketDatasetVersion, on_delete=models.PROTECT, related_name="qualification_runs")
fold = models.CharField(max_length=120)
scenario_name = models.CharField(max_length=160)
runner_name = models.CharField(max_length=160)
replay_mode = models.CharField(max_length=32, default="CANARY_ONLY")
input_hash = models.CharField(max_length=128)
calibration = models.JSONField(default=dict)
configuration = models.JSONField(default=dict)
summary = models.JSONField(default=dict)
class Meta:
constraints = [models.UniqueConstraint(
fields=["strategy_version", "dataset_version", "fold", "scenario_name", "runner_name", "input_hash"],
name="unique_qualification_replay_identity",
)]
class QualificationReplayLedger(TimestampedModel):
"""Append-only replay result. Payload preserves every directly emitted ledger field."""
qualification_run = models.ForeignKey(QualificationReplayRun, on_delete=models.CASCADE, related_name="ledger_rows")
sequence = models.PositiveIntegerField()
identity = models.CharField(max_length=512)
result = models.JSONField(default=dict)
class Meta:
constraints = [
models.UniqueConstraint(fields=["qualification_run", "sequence"], name="unique_qualification_ledger_sequence"),
models.UniqueConstraint(fields=["qualification_run", "identity"], name="unique_qualification_ledger_identity"),
]
class WalkForwardRun(TimestampedModel):
experiment = models.ForeignKey(StrategyExperiment, on_delete=models.PROTECT, related_name="walk_forwards")
dataset_version = models.ForeignKey(MarketDatasetVersion, on_delete=models.PROTECT, related_name="walk_forward_runs")

684
control_plane/trading_studio/qualification.py Normal file
View file

@ -0,0 +1,684 @@
"""Deterministic QualificationReplayV1 with no external or reference-data dependency."""
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from datetime import datetime
from enum import StrEnum
from hashlib import sha256
from json import dumps
class ReplayMode(StrEnum):
CANARY_ONLY = "CANARY_ONLY"
class CalibrationMethod(StrEnum):
RAW = "RAW"
PERCENTILE = "PERCENTILE"
class SameBarPolicy(StrEnum):
PESSIMISTIC_SL = "PESSIMISTIC_SL"
class EventType(StrEnum):
SIGNAL = "SIGNAL"
ENTRY_QUEUED = "ENTRY_QUEUED"
ENTRY = "ENTRY"
EXIT = "EXIT"
COOLDOWN_START = "COOLDOWN_START"
COOLDOWN_END = "COOLDOWN_END"
ENTRY_SKIPPED = "ENTRY_SKIPPED"
class ReplayState(StrEnum):
FLAT = "FLAT"
ENTRY_QUEUED = "ENTRY_QUEUED"
OPEN = "OPEN"
COOLDOWN = "COOLDOWN"
@dataclass(frozen=True)
class CostModel:
version: str
entry_commission_bps: float = 0.0
exit_commission_bps: float = 0.0
entry_slippage_bps: float = 0.0
exit_slippage_bps: float = 0.0
funding_bps_per_bar: float = 0.0
entry_other_cost: float = 0.0
exit_other_cost: float = 0.0
def __post_init__(self) -> None:
if not self.version:
raise ValueError("Cost model version is required.")
if (
min(
self.entry_commission_bps,
self.exit_commission_bps,
self.entry_slippage_bps,
self.exit_slippage_bps,
self.entry_other_cost,
self.exit_other_cost,
)
< 0
):
raise ValueError("Commission, slippage, and other costs cannot be negative.")
@property
def nominal_round_trip_bps(self) -> float:
"""Known entry/exit friction, excluding time-dependent funding and fixed costs."""
return (
self.entry_commission_bps
+ self.exit_commission_bps
+ self.entry_slippage_bps
+ self.exit_slippage_bps
)
@dataclass(frozen=True)
class ScenarioSpec:
name: str
stop_loss_bps: float
take_profit_bps: float
entry_delay_bars: int = 1
cooldown_bars: int = 0
max_holding_bars: int = 1
allow_weekend_entries: bool = False
same_bar_policy: SameBarPolicy = SameBarPolicy.PESSIMISTIC_SL
cost_model: CostModel = CostModel(version="qualification-v1")
def __post_init__(self) -> None:
if not self.name:
raise ValueError("Every qualification scenario must be named.")
if min(self.stop_loss_bps, self.take_profit_bps) <= 0:
raise ValueError("TP and SL widths must be positive.")
if self.entry_delay_bars < 1 or self.cooldown_bars < 0 or self.max_holding_bars < 1:
raise ValueError("Invalid entry delay, cooldown, or holding period.")
def qualification_scenario_catalog() -> dict[str, ScenarioSpec]:
"""The fixed, named QualificationReplayV1 scenario catalog."""
def total_cost(name: str, total_bps: float) -> ScenarioSpec:
component = total_bps / 4
return ScenarioSpec(
name,
100,
100,
cost_model=CostModel(
version=f"{name.lower()}-v1",
entry_commission_bps=component,
exit_commission_bps=component,
entry_slippage_bps=component,
exit_slippage_bps=component,
),
)
baseline = ScenarioSpec("BASELINE", 100, 100)
return {
"BASELINE": baseline,
"SLIPPAGE_ADVERSE_2BP": ScenarioSpec(
"SLIPPAGE_ADVERSE_2BP",
100,
100,
cost_model=CostModel(
version="slippage-adverse-2bp-v1",
entry_slippage_bps=1,
exit_slippage_bps=1,
),
),
"TOTAL_COST_6BP": total_cost("TOTAL_COST_6BP", 6),
"TOTAL_COST_10BP": total_cost("TOTAL_COST_10BP", 10),
"TOTAL_COST_15BP": total_cost("TOTAL_COST_15BP", 15),
"PESSIMISTIC_SAME_BAR": ScenarioSpec("PESSIMISTIC_SAME_BAR", 100, 100),
"ENTRY_DELAY_PLUS_ONE_BAR": ScenarioSpec(
"ENTRY_DELAY_PLUS_ONE_BAR", 100, 100, entry_delay_bars=2
),
"WEEKDAYS_ONLY": ScenarioSpec("WEEKDAYS_ONLY", 100, 100, allow_weekend_entries=False),
"ALL_DAYS": ScenarioSpec("ALL_DAYS", 100, 100, allow_weekend_entries=True),
"STOP_WIDTH_100": ScenarioSpec("STOP_WIDTH_100", 100, 100),
"STOP_WIDTH_75": ScenarioSpec("STOP_WIDTH_75", 75, 100),
"STOP_WIDTH_50": ScenarioSpec("STOP_WIDTH_50", 50, 100),
"STOP_WIDTH_25": ScenarioSpec("STOP_WIDTH_25", 25, 100),
}
@dataclass(frozen=True)
class Bar:
at: datetime
open: float
high: float
low: float
close: float
signal: float | None = None
context: tuple[float, ...] = ()
reference: str = ""
def __post_init__(self) -> None:
if min(self.open, self.high, self.low, self.close) <= 0:
raise ValueError("OHLC prices must be positive.")
if self.high < max(self.open, self.close) or self.low > min(self.open, self.close):
raise ValueError("OHLC values are inconsistent.")
@dataclass(frozen=True)
class Calibration:
method: CalibrationMethod
threshold: float
percentile: float | None
source_record_ids: tuple[str, ...]
source_combo: str
fold: str
input_hash: str
train_start: datetime
train_end: datetime
sample_count: int
@dataclass(frozen=True)
class ReplayEvent:
sequence: int
event_type: EventType
state: ReplayState
at: datetime
bar_index: int
detail: str = ""
@dataclass(frozen=True)
class DEQPrimitives:
return_1_bars_bps: float | None
return_2_bars_bps: float | None
return_3_bars_bps: float | None
return_5_bars_bps: float | None
return_10_bars_bps: float | None
mfe_bps: float
mae_bps: float
time_to_positive_bars: int | None
time_to_25_bps_bars: int | None
time_to_50_bps_bars: int | None
max_adverse_before_positive_bps: float
winner_negative_first: bool
recovery_bars: int | None
@dataclass(frozen=True)
class RescuePrimitives:
activated: bool
trigger_drawdown: float
max_drawdown: float
max_drawdown_bps: float
peak_equity: float
final_equity: float
trade_count: int
win_count: int
loss_count: int
win_rate: float
gross_profit: float
gross_loss: float
profit_factor: float | None
gross_pnl: float
commissions: float
slippage: float
funding: float
other_cost: float
net_pnl: float
average_trade_pnl: float
max_losing_streak: int
recovered: bool
recovery_bars: int | None
RescueMetrics = RescuePrimitives
@dataclass(frozen=True)
class LedgerRow:
identity: str
strategy: str
dataset: str
fold: str
scenario: str
runner: str
signal_at: datetime
signal_bar: int
signal_reference: str
entry_at: datetime
entry_bar: int
entry_reference_price: float
entry_execution_price: float
exit_at: datetime
exit_bar: int
exit_reference_price: float
exit_execution_price: float
take_profit_price: float
stop_loss_price: float
quantity: float
gross_pnl: float
entry_commission: float
exit_commission: float
entry_slippage: float
exit_slippage: float
funding: float
other_cost: float
net_pnl: float
holding_bars: int
cooldown_bars: int
same_bar_policy: SameBarPolicy
exit_reason: str
deq: DEQPrimitives
@dataclass(frozen=True)
class ReplayAggregate:
mode: ReplayMode
strategy: str
dataset: str
fold: str
scenario: ScenarioSpec
runner: str
calibration: Calibration
events: tuple[ReplayEvent, ...]
ledger: tuple[LedgerRow, ...]
gross_pnl: float
commissions: float
slippage: float
funding: float
other_cost: float
net_pnl: float
rescue: RescueMetrics
def calibrate_train_only(
bars: Iterable[Bar],
*,
method: CalibrationMethod,
fold: str,
train_start: datetime,
train_end: datetime,
source_record_ids: Iterable[str],
source_combo: str,
percentile: float | None = None,
allow_empty_raw: bool = False,
) -> Calibration:
"""Calibrate from continuous signal/context records strictly inside the train window."""
records = tuple(bars)
train_records = [bar for bar in records if train_start <= bar.at < train_end]
train = [bar for bar in records if train_start <= bar.at < train_end and bar.signal is not None]
source_ids = tuple(source_record_ids)
raw_empty_allowed = method is CalibrationMethod.RAW and allow_empty_raw
if (
not train_records
or (not train and not raw_empty_allowed)
or not source_ids
or not fold
or not source_combo
):
raise ValueError(
"Train-only calibration requires train records, sources, fold, and source combo."
)
values = sorted(abs(bar.signal) for bar in train if bar.signal is not None)
if method is CalibrationMethod.RAW:
threshold = sum(values) / len(values) if values else 0.0
selected_percentile = None
else:
if percentile is None or not 0 <= percentile <= 100:
raise ValueError("PERCENTILE calibration requires a percentile from 0 through 100.")
position = round((len(values) - 1) * percentile / 100)
threshold = values[position]
selected_percentile = percentile
payload = [
{
"at": bar.at.isoformat(),
"signal": bar.signal,
"context": bar.context,
"reference": bar.reference,
}
for bar in (train_records if raw_empty_allowed else train)
]
input_hash = sha256(dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
return Calibration(
method,
threshold,
selected_percentile,
source_ids,
source_combo,
fold,
input_hash,
train_start,
train_end,
len(train),
)
class QualificationReplayV1:
mode = ReplayMode.CANARY_ONLY
def run(
self,
bars: Iterable[Bar],
*,
strategy: str,
dataset: str,
fold: str,
scenario: ScenarioSpec,
runner: str,
calibration: Calibration,
initial_equity: float = 1.0,
) -> ReplayAggregate:
sequence = tuple(bars)
if not strategy or not dataset or not fold or not runner:
raise ValueError("Strategy, dataset, fold, and runner are required.")
if calibration.fold != fold or initial_equity <= 0:
raise ValueError("Calibration fold must match and initial equity must be positive.")
if any(left.at >= right.at for left, right in zip(sequence, sequence[1:], strict=False)):
raise ValueError("Bars must be strictly chronological.")
events: list[ReplayEvent] = []
ledger: list[LedgerRow] = []
state = ReplayState.FLAT
queued: tuple[Bar, int] | None = None
position: tuple[Bar, int, float, float, float] | None = None
cooldown_remaining = 0
def emit(
event_type: EventType, state_value: ReplayState, index: int, detail: str = ""
) -> None:
events.append(
ReplayEvent(len(events), event_type, state_value, sequence[index].at, index, detail)
)
for index, bar in enumerate(sequence):
if state is ReplayState.COOLDOWN:
cooldown_remaining -= 1
if cooldown_remaining == 0:
state = ReplayState.FLAT
emit(EventType.COOLDOWN_END, state, index)
else:
continue
if state is ReplayState.ENTRY_QUEUED and queued is not None and index == queued[1]:
signal_bar, _ = queued
if not scenario.allow_weekend_entries and bar.at.weekday() >= 5:
emit(EventType.ENTRY_SKIPPED, ReplayState.FLAT, index, "weekend")
queued = None
state = ReplayState.FLAT
else:
direction = 1 if signal_bar.signal and signal_bar.signal > 0 else -1
entry_reference = bar.open
entry_execution = self._apply_slippage(
entry_reference, direction, scenario.cost_model.entry_slippage_bps
)
stop = entry_execution * (1 - direction * scenario.stop_loss_bps / 10_000)
target = entry_execution * (1 + direction * scenario.take_profit_bps / 10_000)
position = (signal_bar, index, entry_execution, stop, target)
queued = None
state = ReplayState.OPEN
emit(EventType.ENTRY, state, index)
if state is ReplayState.OPEN and position is not None:
signal_bar, entry_index, entry_execution, stop, target = position
exit_reason, exit_reference = self._exit_for_bar(
bar, entry_execution, stop, target, index - entry_index + 1, scenario
)
if exit_reason:
direction = 1 if signal_bar.signal and signal_bar.signal > 0 else -1
row = self._ledger_row(
sequence,
strategy,
dataset,
fold,
scenario,
runner,
signal_bar,
entry_index,
index,
entry_execution,
stop,
target,
exit_reference,
direction,
)
ledger.append(row)
emit(EventType.EXIT, ReplayState.FLAT, index, exit_reason)
position = None
if scenario.cooldown_bars:
state = ReplayState.COOLDOWN
cooldown_remaining = scenario.cooldown_bars
emit(EventType.COOLDOWN_START, state, index)
else:
state = ReplayState.FLAT
if (
state is ReplayState.FLAT
and bar.at >= calibration.train_end
and bar.signal is not None
):
if abs(
bar.signal
) >= calibration.threshold and index + scenario.entry_delay_bars < len(sequence):
emit(EventType.SIGNAL, state, index)
queued = (bar, index + scenario.entry_delay_bars)
state = ReplayState.ENTRY_QUEUED
emit(EventType.ENTRY_QUEUED, state, index, f"bar={queued[1]}")
commissions = sum(row.entry_commission + row.exit_commission for row in ledger)
slippage = sum(row.entry_slippage + row.exit_slippage for row in ledger)
funding = sum(row.funding for row in ledger)
other = sum(row.other_cost for row in ledger)
gross = sum(row.gross_pnl for row in ledger)
net = sum(row.net_pnl for row in ledger)
rescue = self._rescue(tuple(ledger), initial_equity)
return ReplayAggregate(
self.mode,
strategy,
dataset,
fold,
scenario,
runner,
calibration,
tuple(events),
tuple(ledger),
gross,
commissions,
slippage,
funding,
other,
net,
rescue,
)
@staticmethod
def _apply_slippage(price: float, direction: int, bps: float) -> float:
return price * (1 + direction * bps / 10_000)
@staticmethod
def _exit_for_bar(
bar: Bar, entry: float, stop: float, target: float, held: int, scenario: ScenarioSpec
) -> tuple[str, float]:
long = target > entry
stopped = bar.low <= stop if long else bar.high >= stop
target_hit = bar.high >= target if long else bar.low <= target
if stopped and (scenario.same_bar_policy is SameBarPolicy.PESSIMISTIC_SL or not target_hit):
return "SL", min(bar.open, stop) if long else max(bar.open, stop)
if target_hit:
return "TP", target
if held >= scenario.max_holding_bars:
return "TIME", bar.close
return "", 0.0
def _ledger_row(
self,
bars: tuple[Bar, ...],
strategy: str,
dataset: str,
fold: str,
scenario: ScenarioSpec,
runner: str,
signal: Bar,
entry_index: int,
exit_index: int,
entry: float,
stop: float,
target: float,
exit_reference: float,
direction: int,
) -> LedgerRow:
cost = scenario.cost_model
exit_execution = self._apply_slippage(exit_reference, -direction, cost.exit_slippage_bps)
entry_commission = entry * cost.entry_commission_bps / 10_000
exit_commission = exit_execution * cost.exit_commission_bps / 10_000
entry_slippage = abs(entry - bars[entry_index].open)
exit_slippage = abs(exit_execution - exit_reference)
holding = exit_index - entry_index + 1
funding = entry * holding * cost.funding_bps_per_bar / 10_000
other = cost.entry_other_cost + cost.exit_other_cost
gross = (exit_execution - entry) * direction
net = gross - entry_commission - exit_commission - funding - other
deq = self._deq(bars, entry_index, entry, direction)
identity = f"{strategy}:{dataset}:{fold}:{scenario.name}:{runner}:{signal.at.isoformat()}"
return LedgerRow(
identity,
strategy,
dataset,
fold,
scenario.name,
runner,
signal.at,
bars.index(signal),
signal.reference,
bars[entry_index].at,
entry_index,
bars[entry_index].open,
entry,
bars[exit_index].at,
exit_index,
exit_reference,
exit_execution,
target,
stop,
1.0,
gross,
entry_commission,
exit_commission,
entry_slippage,
exit_slippage,
funding,
other,
net,
holding,
scenario.cooldown_bars,
scenario.same_bar_policy,
self._exit_for_bar(bars[exit_index], entry, stop, target, holding, scenario)[0],
deq,
)
@staticmethod
def _deq(
bars: tuple[Bar, ...],
entry_index: int,
entry: float,
direction: int,
) -> DEQPrimitives:
path = bars[entry_index:]
highs = [direction * (bar.high - entry) / entry * 10_000 for bar in path]
lows = [direction * (bar.low - entry) / entry * 10_000 for bar in path]
favorable = highs if direction > 0 else lows
adverse = lows if direction > 0 else highs
positive_at = next((index for index, value in enumerate(favorable) if value > 0), None)
before_positive = adverse[:positive_at] if positive_at is not None else adverse
recovery = next(
(index for index, bar in enumerate(path) if direction * (bar.close - entry) >= 0), None
)
def forward(horizon: int) -> float | None:
if entry_index + horizon >= len(bars):
return None
return direction * (bars[entry_index + horizon].close - entry) / entry * 10_000
return DEQPrimitives(
forward(1),
forward(2),
forward(3),
forward(5),
forward(10),
max(favorable),
min(adverse),
positive_at,
next((index for index, value in enumerate(favorable) if value >= 25), None),
next((index for index, value in enumerate(favorable) if value >= 50), None),
min(before_positive, default=0.0),
path[-1].close * direction > entry * direction and min(adverse) < 0,
recovery,
)
@staticmethod
def _rescue(ledger: tuple[LedgerRow, ...], initial_equity: float) -> RescuePrimitives:
equity = initial_equity
peak = equity
max_drawdown = 0.0
max_losing_streak = losing_streak = 0
recovery_bars: int | None = 0
recovery_target = initial_equity
gross_profit = gross_loss = 0.0
for index, row in enumerate(ledger, start=1):
equity += row.net_pnl
gross_profit += max(row.net_pnl, 0)
gross_loss += min(row.net_pnl, 0)
losing_streak = losing_streak + 1 if row.net_pnl < 0 else 0
max_losing_streak = max(max_losing_streak, losing_streak)
peak = max(peak, equity)
drawdown = peak - equity
if drawdown > max_drawdown:
max_drawdown = drawdown
recovery_target = peak
recovery_bars = None
elif recovery_bars is None and equity >= recovery_target:
recovery_bars = index
wins = sum(row.net_pnl > 0 for row in ledger)
losses = sum(row.net_pnl < 0 for row in ledger)
commissions = sum(row.entry_commission + row.exit_commission for row in ledger)
slippage = sum(row.entry_slippage + row.exit_slippage for row in ledger)
funding = sum(row.funding for row in ledger)
other_cost = sum(row.other_cost for row in ledger)
gross_pnl = sum(row.gross_pnl for row in ledger)
net_pnl = sum(row.net_pnl for row in ledger)
return RescuePrimitives(
activated=equity <= 0,
trigger_drawdown=initial_equity,
max_drawdown=max_drawdown,
max_drawdown_bps=max_drawdown / initial_equity * 10_000,
peak_equity=peak,
final_equity=equity,
trade_count=len(ledger),
win_count=wins,
loss_count=losses,
win_rate=wins / len(ledger) if ledger else 0,
gross_profit=gross_profit,
gross_loss=gross_loss,
profit_factor=gross_profit / abs(gross_loss) if gross_loss else None,
gross_pnl=gross_pnl,
commissions=commissions,
slippage=slippage,
funding=funding,
other_cost=other_cost,
net_pnl=net_pnl,
average_trade_pnl=net_pnl / len(ledger) if ledger else 0.0,
max_losing_streak=max_losing_streak,
recovered=recovery_bars is not None,
recovery_bars=recovery_bars,
)
def ledger_payload(row: LedgerRow) -> dict[str, object]:
"""JSON-safe immutable representation for the Django ledger model."""
return asdict(
row,
dict_factory=lambda values: {
key: value.isoformat() if isinstance(value, datetime) else value
for key, value in values
},
)

79
control_plane/trading_studio/services.py
View file

@ -2,6 +2,7 @@ from __future__ import annotations
import hashlib
import json
from dataclasses import asdict
from datetime import datetime
from decimal import Decimal
from typing import Any
@ -14,11 +15,12 @@ from control_plane.trading_studio.models import (
AllocationTier, BacktestRun, DataKind, EvidenceStatus, ExperimentStatus,
FailureType, FeatureDefinition, FeatureSetVersion, LiveStrategyRun,
MarketDataset, MarketDatasetVersion, RunStatus, ShadowRun, SplitKind,
Strategy, StrategyCapitalAllocation, StrategyEvaluation, StrategyExperiment,
QualificationReplayLedger, QualificationReplayRun, Strategy, StrategyCapitalAllocation, StrategyEvaluation, StrategyExperiment,
StrategyPromotionDecision, StrategyStage, StrategyVersion, TradingCohort,
TradingProject, TradingProjectStatus, TradingResearchReport,
)
from control_plane.trading_studio.profiles import CryptoTradingProfile, ExistingSystemProfile, HyperScalperProfile, TradingProjectProfile
from control_plane.trading_studio.qualification import Bar, Calibration, QualificationReplayV1, ScenarioSpec, ledger_payload
class TradingStudioService:
@ -125,6 +127,71 @@ class TradingStudioService:
version.save(update_fields=["holdout_exposure_count", "immutable", "updated_at"])
return run
@transaction.atomic
def run_qualification_canary(
self, *, strategy_version: StrategyVersion, dataset_version: MarketDatasetVersion, fold: str,
scenario: ScenarioSpec, runner_name: str, bars: list[Bar], calibration: Calibration,
protocol_identity: dict[str, Any] | None = None,
require_protocol_identity: bool = False,
) -> QualificationReplayRun:
"""Persist a supplied Cohort001 replay; never synthesize unavailable canary records."""
missing = [
name
for name, value in (
("strategy_version", strategy_version),
("dataset_version", dataset_version),
("fold", fold),
("bars", bars),
("calibration", calibration),
)
if not value
]
if missing:
raise ValueError(
"CANARY_ONLY refuses to run: missing materialized " + ", ".join(missing) + "."
)
result = QualificationReplayV1().run(
bars, strategy=str(strategy_version.id), dataset=str(dataset_version.id), fold=fold,
scenario=scenario, runner=runner_name, calibration=calibration,
)
configuration = self.qualification_configuration(
scenario,
protocol_identity=protocol_identity,
require_protocol_identity=require_protocol_identity,
)
run = QualificationReplayRun.objects.create(
strategy_version=strategy_version, dataset_version=dataset_version, fold=fold,
scenario_name=scenario.name, runner_name=runner_name, replay_mode=result.mode,
input_hash=calibration.input_hash, calibration=self._json_safe(asdict(calibration)),
configuration=configuration, summary={
"gross_pnl": result.gross_pnl, "net_pnl": result.net_pnl, "event_count": len(result.events),
"ledger_count": len(result.ledger), "rescue": self._json_safe(asdict(result.rescue)),
},
)
QualificationReplayLedger.objects.bulk_create([
QualificationReplayLedger(qualification_run=run, sequence=index, identity=row.identity,
result=self._json_safe(ledger_payload(row)))
for index, row in enumerate(result.ledger)
])
return run
@staticmethod
def qualification_configuration(
scenario: ScenarioSpec,
*,
protocol_identity: dict[str, Any] | None = None,
require_protocol_identity: bool = False,
) -> dict[str, Any]:
"""Build the immutable scenario snapshot used for persisted replay identity."""
if require_protocol_identity and not protocol_identity:
raise ValueError("Batch qualification requires a protocol identity snapshot.")
configuration = TradingStudioService._json_safe(asdict(scenario))
if protocol_identity is not None:
if not isinstance(protocol_identity, dict) or not protocol_identity:
raise ValueError("Protocol identity must be a non-empty object.")
configuration["protocol_identity"] = TradingStudioService._json_safe(protocol_identity)
return configuration
def judge_backtest(self, experiment: StrategyExperiment, backtest: BacktestRun, *, policy: dict[str, Any]) -> StrategyEvaluation:
metrics = backtest.metrics
failure = ""
@ -187,3 +254,13 @@ class TradingStudioService:
@staticmethod
def _hash(value: Any) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest()
@staticmethod
def _json_safe(value: Any) -> Any:
if isinstance(value, datetime):
return value.isoformat()
if isinstance(value, dict):
return {key: TradingStudioService._json_safe(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [TradingStudioService._json_safe(item) for item in value]
return value

94
generate_full_historical_oracle_request.py Normal file
View file

@ -0,0 +1,94 @@
#!/usr/bin/env python3
"""Build the exact 711-feature code-5056feb oracle request without Django."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
ENGINE_REVISION = "code-5056feb"
INPUT_COLUMNS = ["close", "high", "low", "volume"]
def canonical(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def validate_usage_summary(path: Path) -> None:
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("artifact") != "HISTORICAL_FEATURE_USAGE_SUMMARY_V1":
raise ValueError("--usage-summary is not HISTORICAL_FEATURE_USAGE_SUMMARY_V1")
if payload.get("counts", {}).get("role_independent_primitives") != 711:
raise ValueError("usage summary does not attest to 711 role-independent primitives")
def triples_from_engineering(path: Path) -> set[tuple[int, int, float]]:
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("artifact") != "ENGINEERING_FAMILY_MAP_V1":
raise ValueError("--engineering-map is not ENGINEERING_FAMILY_MAP_V1")
return {
(int(row["indicator_id"]), int(row["period"]), float(row["p1"]))
for row in payload["primitives"]
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--usage-summary", type=Path, required=True, help="historical_feature_usage_v1_summary.json")
parser.add_argument("--engineering-map", type=Path, required=True, help="engineering_family_map_v1.json")
parser.add_argument("--input-csv", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--deq-targets", type=Path, help="optional JSON metadata copied into the request")
args = parser.parse_args()
validate_usage_summary(args.usage_summary)
engineering = triples_from_engineering(args.engineering_map)
if len(engineering) != 711:
raise ValueError(f"engineering map contains {len(engineering)} rather than 711 primitives")
if not args.input_csv.is_file():
raise ValueError("--input-csv must be a file")
semantic_base = {
"engine_revision": ENGINE_REVISION,
"input_columns": INPUT_COLUMNS,
"output_dtype": "float64",
"window_policy": "continuous_full_history",
}
requests = []
for indicator_id, period, p1 in sorted(engineering):
semantic = {**semantic_base, "indicator_id": indicator_id, "period": period, "p1": p1}
requests.append(
{
"request_id": f"hs22_{indicator_id}_{period}_{p1:g}",
"indicator_id": indicator_id,
"period": period,
"p1": p1,
"semantic_fingerprint": hashlib.sha256(canonical(semantic)).hexdigest(),
}
)
payload: dict[str, object] = {
"schema_version": 1,
"artifact": "HISTORICAL_FEATURE_ORACLE_V1_REQUEST",
**semantic_base,
"input_csv_sha256": sha256(args.input_csv),
"source_artifacts": {
"usage_summary": {"path": args.usage_summary.name, "sha256": sha256(args.usage_summary)},
"engineering_map": {"path": args.engineering_map.name, "sha256": sha256(args.engineering_map)},
},
"requests": requests,
}
if args.deq_targets:
payload["deq_targets"] = json.loads(args.deq_targets.read_text(encoding="utf-8"))
payload["source_artifacts"]["deq_targets"] = {"path": args.deq_targets.name, "sha256": sha256(args.deq_targets)}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_bytes(canonical(payload) + b"\n")
print(args.output)
if __name__ == "__main__":
main()

214
historical_feature_oracle_v1.py Normal file
View file

@ -0,0 +1,214 @@
"""Standalone runner for recovered historical ``hyperscalper.fast_engine``.
This file intentionally has no Artifex imports. Copy it with a request JSON,
CSV, and the recovered ``code-5056feb/src`` tree to run an external oracle.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib
import json
import sys
from pathlib import Path
from typing import Any
import numpy as np
ENGINE_REVISION = "code-5056feb"
ARTIFACT = "HISTORICAL_FEATURE_ORACLE_V1_RESULT"
REQUIRED_COLUMNS = ("close", "high", "low", "volume")
def _canonical_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")
def _sha256_bytes(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def _array_sha256(values: np.ndarray) -> str:
contiguous = np.ascontiguousarray(values)
header = _canonical_bytes({"dtype": contiguous.dtype.str, "shape": contiguous.shape})
return _sha256_bytes(header + b"\0" + contiguous.tobytes())
def _load_ohlcv(path: Path, columns: tuple[str, ...]) -> tuple[np.ndarray, ...]:
# NumPy owns parsing so this runner has no CSV/dataframe dependency.
table = np.genfromtxt(path, delimiter=",", names=True, dtype=None, encoding="utf-8")
table = np.atleast_1d(table)
names = set(table.dtype.names or ())
missing = set(columns) - names
if missing:
raise ValueError(f"CSV is missing required columns: {', '.join(sorted(missing))}")
arrays = tuple(np.asarray(table[name], dtype=np.float64) for name in columns)
if not arrays[0].size or any(array.ndim != 1 or array.shape != arrays[0].shape for array in arrays):
raise ValueError("CSV OHLCV columns must be non-empty equally sized vectors")
return arrays
def _load_engine(source: Path):
package = source / "hyperscalper"
engine_file = package / "fast_engine.py"
if not engine_file.is_file() or not (package / "__init__.py").is_file():
raise ValueError("--recovered-source must contain hyperscalper/fast_engine.py from code-5056feb")
if any(
name in {"artifex", "control_plane"}
or name.startswith(("artifex.", "control_plane."))
for name in sys.modules
):
raise RuntimeError("Artifex modules must not be imported by the historical oracle")
sys.path.insert(0, str(source))
engine = importlib.import_module("hyperscalper.fast_engine")
if Path(engine.__file__).resolve() != engine_file.resolve():
raise RuntimeError("refused a hyperscalper.fast_engine outside --recovered-source")
return engine
def _requests(payload: dict[str, Any], known_ids: set[int]) -> list[dict[str, Any]]:
if payload.get("artifact") != "HISTORICAL_FEATURE_ORACLE_V1_REQUEST":
raise ValueError("request artifact must be HISTORICAL_FEATURE_ORACLE_V1_REQUEST")
if payload.get("schema_version") != 1 or payload.get("engine_revision") != ENGINE_REVISION:
raise ValueError("unsupported historical oracle request version or engine revision")
semantic_base = {
"engine_revision": payload["engine_revision"],
"input_columns": payload.get("input_columns"),
"output_dtype": payload.get("output_dtype"),
"window_policy": payload.get("window_policy"),
}
if semantic_base["input_columns"] != list(REQUIRED_COLUMNS) or semantic_base["output_dtype"] != "float64":
raise ValueError("request has unsupported input or output semantics")
rows = payload.get("requests")
if not isinstance(rows, list) or not rows:
raise ValueError("request must contain a non-empty requests list")
ids: set[str] = set()
parsed: list[dict[str, Any]] = []
for row in rows:
if not isinstance(row, dict):
raise ValueError("every request must be an object")
request_id = row.get("request_id")
if not isinstance(request_id, str) or not request_id or request_id in ids:
raise ValueError("request_id must be a unique non-empty string")
indicator_id, period, p1 = int(row["indicator_id"]), int(row["period"]), float(row["p1"])
if indicator_id not in known_ids:
raise ValueError(f"unknown historical indicator ID: {indicator_id}")
if period <= 0 or not np.isfinite(p1):
raise ValueError(f"invalid period or p1 for {request_id}")
fingerprint = row.get("semantic_fingerprint")
expected_fingerprint = _sha256_bytes(
_canonical_bytes(
{**semantic_base, "indicator_id": indicator_id, "period": period, "p1": p1}
)
)
if fingerprint != expected_fingerprint:
raise ValueError(f"semantic fingerprint mismatch for {request_id}")
ids.add(request_id)
parsed.append(
{
"request_id": request_id,
"indicator_id": indicator_id,
"period": period,
"p1": p1,
"semantic_fingerprint": fingerprint,
}
)
return parsed
def _windows(payload: dict[str, Any], length: int) -> list[dict[str, int | str]]:
rows = payload.get("windows", [{"window_id": "full", "start": 0, "stop": length}])
if not isinstance(rows, list) or not rows:
raise ValueError("windows must be a non-empty list")
windows = []
for row in rows:
if not isinstance(row, dict):
raise ValueError("every window must be an object")
window_id, start, stop = str(row.get("window_id", "")), int(row["start"]), int(row["stop"])
if not window_id or start < 0 or stop <= start or stop > length:
raise ValueError(f"invalid output window: {window_id}")
windows.append({"window_id": window_id, "start": start, "stop": stop})
return windows
def run(request_path: Path, csv_path: Path, output_dir: Path, recovered_source: Path) -> Path:
payload = json.loads(request_path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError("request JSON must be an object")
expected_csv_hash = payload.get("input_csv_sha256")
csv_hash = _sha256_bytes(csv_path.read_bytes())
if expected_csv_hash is not None and expected_csv_hash != csv_hash:
raise ValueError("input CSV SHA-256 does not match the request")
columns = tuple(payload.get("input_columns", REQUIRED_COLUMNS))
if columns != REQUIRED_COLUMNS:
raise ValueError("only close, high, low, volume input semantics are supported")
close, high, low, volume = _load_ohlcv(csv_path, columns)
engine = _load_engine(recovered_source)
requests = _requests(payload, {int(row[0]) for row in engine.FAST_POOL})
windows = _windows(payload, len(close))
outputs: dict[str, np.ndarray] = {}
records = []
for row in requests:
values = np.asarray(
engine.compute(
row["indicator_id"], close, high, low, volume, row["period"], row["p1"]
),
dtype=np.float64,
)
# Validate the artifact against a second direct call to the recovered dispatcher.
direct = np.asarray(
engine.compute(
row["indicator_id"], close, high, low, volume, row["period"], row["p1"]
),
dtype=np.float64,
)
if values.shape != close.shape or not np.array_equal(values, direct, equal_nan=True):
raise RuntimeError(f"direct compute self-validation failed for {row['request_id']}")
outputs[row["request_id"]] = values
records.append(
{
**row,
"dtype": values.dtype.str,
"shape": list(values.shape),
"sha256": _array_sha256(values),
"window_sha256": {
window["window_id"]: _array_sha256(values[window["start"] : window["stop"]])
for window in windows
},
}
)
output_dir.mkdir(parents=True, exist_ok=True)
npz_path = output_dir / "historical_feature_oracle_v1_outputs.npz"
np.savez_compressed(npz_path, **outputs)
result_path = output_dir / "historical_feature_oracle_v1_result.json"
result = {
"schema_version": 1,
"artifact": ARTIFACT,
"engine_revision": ENGINE_REVISION,
"request_sha256": _sha256_bytes(request_path.read_bytes()),
"input_csv_sha256": csv_hash,
"engine_sha256": _sha256_bytes(
(recovered_source / "hyperscalper" / "fast_engine.py").read_bytes()
),
"row_count": len(close),
"windows": windows,
"outputs_npz": {"path": npz_path.name, "sha256": _sha256_bytes(npz_path.read_bytes())},
"requests": records,
}
result_path.write_bytes(_canonical_bytes(result) + b"\n")
return result_path
def main() -> None:
parser = argparse.ArgumentParser(description="Run recovered code-5056feb historical features offline.")
parser.add_argument("--request", type=Path, required=True)
parser.add_argument("--input-csv", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--recovered-source", type=Path, required=True)
args = parser.parse_args()
print(run(args.request, args.input_csv, args.output_dir, args.recovered_source))
if __name__ == "__main__":
main()

3
pyproject.toml
View file

@ -8,10 +8,13 @@ dependencies = [
"psycopg[binary]>=3.2,<4.0",
"langgraph>=0.2,<0.3",
"structlog>=24.4,<25.0",
"numpy>=2.0,<3.0",
"numba>=0.60,<1.0",
]
[project.optional-dependencies]
dev = [
"pyarrow>=25.0.1",
"pytest>=8.3,<9.0",
"pytest-django>=4.9,<5.0",
"ruff>=0.8,<1.0",

196
run_historical_oracle_overnight.py Normal file
View file

@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Resumable, Django-free full-history oracle runner for recovered code-5056feb."""
from __future__ import annotations
import argparse
import hashlib
import importlib
import json
import os
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import numpy as np
ENGINE_REVISION = "code-5056feb"
REQUIRED_COLUMNS = ("close", "high", "low", "volume")
def canonical(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8")
def digest_bytes(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def digest_file(path: Path) -> str:
return digest_bytes(path.read_bytes())
def array_digest(values: np.ndarray) -> str:
values = np.ascontiguousarray(values)
return digest_bytes(canonical({"dtype": values.dtype.str, "shape": values.shape}) + b"\0" + values.tobytes())
def atomic_bytes(path: Path, data: bytes) -> None:
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_bytes(data)
os.replace(temporary, path)
def now() -> str:
return datetime.now(timezone.utc).isoformat()
def log(path: Path, event: str, **fields: object) -> None:
atomic_append = canonical({"at": now(), "event": event, **fields}) + b"\n"
with path.open("ab") as handle:
handle.write(atomic_append)
handle.flush()
os.fsync(handle.fileno())
def load_engine(source: Path):
engine_file = source / "hyperscalper" / "fast_engine.py"
if not engine_file.is_file() or not (source / "hyperscalper" / "__init__.py").is_file():
raise ValueError("--recovered-source must contain hyperscalper/fast_engine.py")
sys.path.insert(0, str(source))
engine = importlib.import_module("hyperscalper.fast_engine")
if Path(engine.__file__).resolve() != engine_file.resolve():
raise RuntimeError("refused a fast_engine outside --recovered-source")
return engine
def load_csv(path: Path) -> tuple[np.ndarray, ...]:
table = np.atleast_1d(np.genfromtxt(path, delimiter=",", names=True, dtype=None, encoding="utf-8"))
names = set(table.dtype.names or ())
if set(REQUIRED_COLUMNS) - names:
raise ValueError("CSV must contain close, high, low, volume")
arrays = tuple(np.asarray(table[name], dtype=np.float64) for name in REQUIRED_COLUMNS)
if not arrays[0].size or any(values.shape != arrays[0].shape for values in arrays):
raise ValueError("CSV OHLCV columns must be non-empty, equal-length vectors")
return arrays
def load_request(path: Path, engine: Any) -> list[dict[str, Any]]:
payload = json.loads(path.read_text(encoding="utf-8"))
if payload.get("artifact") != "HISTORICAL_FEATURE_ORACLE_V1_REQUEST" or payload.get("engine_revision") != ENGINE_REVISION:
raise ValueError("unsupported request artifact or engine revision")
if payload.get("input_columns") != list(REQUIRED_COLUMNS) or payload.get("output_dtype") != "float64":
raise ValueError("unsupported input/output semantics")
rows = payload.get("requests")
if not isinstance(rows, list) or len(rows) != 711:
raise ValueError("full historical request must contain exactly 711 features")
known = {int(row[0]) for row in engine.FAST_POOL}
base = {key: payload[key] for key in ("engine_revision", "input_columns", "output_dtype", "window_policy")}
result = []
for row in rows:
item = {"request_id": str(row["request_id"]), "indicator_id": int(row["indicator_id"]), "period": int(row["period"]), "p1": float(row["p1"])}
if item["indicator_id"] not in known or item["period"] <= 0 or not np.isfinite(item["p1"]):
raise ValueError(f"invalid historical request: {item['request_id']}")
expected = digest_bytes(canonical({**base, **{key: item[key] for key in ("indicator_id", "period", "p1")}}))
if row.get("semantic_fingerprint") != expected:
raise ValueError(f"semantic fingerprint mismatch: {item['request_id']}")
result.append(item)
if len({row["request_id"] for row in result}) != len(result):
raise ValueError("request IDs must be unique")
return result
def checkpoint(path: Path, row: dict[str, Any], values: np.ndarray) -> dict[str, object]:
npy = path / f"{row['request_id']}.npy"
meta = path / f"{row['request_id']}.json"
if npy.is_file() and meta.is_file():
saved = json.loads(meta.read_text(encoding="utf-8"))
loaded = np.load(npy, allow_pickle=False)
if saved["sha256"] == array_digest(loaded) and tuple(saved["shape"]) == loaded.shape:
return saved
temporary = npy.with_suffix(".tmp.npy")
np.save(temporary, values, allow_pickle=False)
os.replace(temporary, npy)
saved = {**row, "path": npy.name, "dtype": values.dtype.str, "shape": list(values.shape), "sha256": array_digest(values)}
atomic_bytes(meta, canonical(saved) + b"\n")
return saved
def wrapper_validate(source: Path, engine: Any, arrays: tuple[np.ndarray, ...], combo: list[float]) -> dict[str, object]:
if len(combo) < 15:
raise ValueError("--wrapper-combo requires at least 15 numeric combo values")
wrapper = importlib.import_module("hyperscalper.paper_replay")
close, high, low, volume = arrays
state = wrapper.compute_combo_state(combo, close, high, low, volume)
slots = ("trend", "signal", "trigger", "confirm", "vol")
for slot, offset in zip(slots, range(0, 15, 3), strict=True):
direct = np.asarray(engine.compute(int(combo[offset]), close, high, low, volume, int(combo[offset + 1]), float(combo[offset + 2])), dtype=np.float64)
if slot not in state or not np.array_equal(direct, np.asarray(state[slot], dtype=np.float64), equal_nan=True):
raise RuntimeError(f"paper_replay wrapper mismatch for {slot}")
return {"status": "PASS", "paper_replay_sha256": digest_file(source / "hyperscalper" / "paper_replay.py")}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--request", type=Path, required=True)
parser.add_argument("--input-csv", type=Path, required=True)
parser.add_argument("--recovered-source", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--pilot-count", type=int, default=3)
parser.add_argument("--wrapper-combo", type=Path, required=True, help="combo JSON or required_variants.json; validates paper_replay")
args = parser.parse_args()
if args.pilot_count < 1:
raise ValueError("--pilot-count must be positive")
args.output_dir.mkdir(parents=True, exist_ok=True)
checkpoints = args.output_dir / "checkpoints"
checkpoints.mkdir(exist_ok=True)
event_log = args.output_dir / "events.jsonl"
status = args.output_dir / "final_status.json"
try:
engine = load_engine(args.recovered_source)
arrays = load_csv(args.input_csv)
requests = load_request(args.request, engine)
if json.loads(args.request.read_text(encoding="utf-8")).get("input_csv_sha256") not in (None, digest_file(args.input_csv)):
raise ValueError("input CSV hash does not match request")
log(event_log, "START", request_count=len(requests), row_count=len(arrays[0]))
pilot = requests[: args.pilot_count]
started = time.perf_counter()
for row in pilot:
values = np.asarray(engine.compute(row["indicator_id"], *arrays, row["period"], row["p1"]), dtype=np.float64)
if values.shape != arrays[0].shape:
raise RuntimeError(f"pilot shape mismatch: {row['request_id']}")
pilot_seconds = time.perf_counter() - started
log(event_log, "PILOT_PASS", count=len(pilot), seconds=pilot_seconds, seconds_per_feature=pilot_seconds / len(pilot))
raw = json.loads(args.wrapper_combo.read_text(encoding="utf-8"))
if isinstance(raw, dict) and "combos" in raw:
raw = raw["combos"][0]
combo = raw.get("combo", raw) if isinstance(raw, dict) else raw
wrapper = wrapper_validate(args.recovered_source, engine, arrays, combo)
log(event_log, "WRAPPER_VALIDATION", **wrapper)
records = []
for number, row in enumerate(requests, start=1):
values = np.asarray(engine.compute(row["indicator_id"], *arrays, row["period"], row["p1"]), dtype=np.float64)
if values.shape != arrays[0].shape:
raise RuntimeError(f"shape mismatch: {row['request_id']}")
record = checkpoint(checkpoints, row, values)
records.append(record)
log(event_log, "CHECKPOINT", number=number, request_id=row["request_id"], sha256=record["sha256"])
representative = requests[len(requests) // 2]
first = np.load(checkpoints / f"{representative['request_id']}.npy", allow_pickle=False)
second = np.asarray(engine.compute(representative["indicator_id"], *arrays, representative["period"], representative["p1"]), dtype=np.float64)
if not np.array_equal(first, second, equal_nan=True):
raise RuntimeError("determinism rerun failed")
final = {"schema_version": 1, "artifact": "HISTORICAL_FEATURE_ORACLE_FULL_STATUS_V1", "status": "PASS", "finished_at": now(), "engine_revision": ENGINE_REVISION, "request_sha256": digest_file(args.request), "input_csv_sha256": digest_file(args.input_csv), "engine_sha256": digest_file(args.recovered_source / "hyperscalper" / "fast_engine.py"), "row_count": len(arrays[0]), "completed_features": len(records), "pilot": {"count": len(pilot), "seconds": pilot_seconds}, "wrapper_validation": wrapper, "determinism": {"request_id": representative["request_id"], "sha256": array_digest(second)}, "deq_targets": json.loads(args.request.read_text(encoding="utf-8")).get("deq_targets"), "checkpoints": records}
atomic_bytes(status, canonical(final) + b"\n")
log(event_log, "PASS", completed_features=len(records))
except Exception as error:
failure = {"schema_version": 1, "artifact": "HISTORICAL_FEATURE_ORACLE_FULL_STATUS_V1", "status": "FAIL", "finished_at": now(), "error": f"{type(error).__name__}: {error}"}
atomic_bytes(status, canonical(failure) + b"\n")
log(event_log, "FAIL", error=failure["error"])
raise
if __name__ == "__main__":
main()

17
run_historical_oracle_overnight.sh Normal file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Copy this file and the two Python scripts to Spark; paths are intentionally explicit.
set -euo pipefail
: "${RECOVERED_SOURCE:?set to code-5056feb/src}"
: "${INPUT_CSV:?set to canonical OHLCV CSV}"
ROOT=${ROOT:-"$(pwd)"}
: "${WRAPPER_COMBO:=$ROOT/required_variants.json}"
REQUEST=${REQUEST:-"$ROOT/hs22_full_oracle_request.json"}
OUTPUT_DIR=${OUTPUT_DIR:-"$ROOT/hs22_full_oracle_run"}
request_args=(--usage-summary "$ROOT/historical_feature_usage_v1_summary.json" --engineering-map "$ROOT/engineering_family_map_v1.json" --input-csv "$INPUT_CSV" --output "$REQUEST")
[[ -n "${DEQ_TARGETS:-}" ]] && request_args+=(--deq-targets "$DEQ_TARGETS")
python3 "$ROOT/generate_full_historical_oracle_request.py" "${request_args[@]}"
run_args=(--request "$REQUEST" --input-csv "$INPUT_CSV" --recovered-source "$RECOVERED_SOURCE" --output-dir "$OUTPUT_DIR" --wrapper-combo "$WRAPPER_COMBO")
python3 "$ROOT/run_historical_oracle_overnight.py" "${run_args[@]}"

270
scripts/analyze_historical_feature_usage_v1.py Normal file
View file

@ -0,0 +1,270 @@
"""Build source-only feature-usage and registry-coverage archaeology artifacts."""
from __future__ import annotations
import argparse
import hashlib
import json
from collections import Counter, defaultdict
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
ROLES = ("trend", "signal", "trigger", "confirm", "vol")
FRONTIER_TARGETS = (80, 90, 95, 98, 99, 100)
def canonical_key(indicator_id: int, period: int, p1: float) -> tuple[int, int, float]:
return indicator_id, period, float(p1)
def key_text(key: tuple[int, int, float]) -> str:
return f"{key[0]}:{key[1]}:{key[2]:g}"
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def frontier(counts: Counter[object]) -> list[dict[str, object]]:
total = sum(counts.values())
ordered = sorted(counts.items(), key=lambda item: (-item[1], str(item[0])))
result = []
cumulative = 0
cursor = 0
for target in FRONTIER_TARGETS:
threshold = total * target / 100
while cursor < len(ordered) and cumulative < threshold:
cumulative += ordered[cursor][1]
cursor += 1
result.append(
{
"target_percent": target,
"items_required": cursor,
"cumulative_usage": cumulative,
"cumulative_percent": cumulative * 100 / total if total else 0.0,
}
)
return result
def load_catalog(path: Path) -> dict[tuple[int, int, float], str]:
payload = json.loads(path.read_text(encoding="utf-8"))
return {
canonical_key(item["indicator_id"], item["period"], item["p1"]): item["support_state"]
for item in payload["features"]
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--lineage", type=Path, required=True)
parser.add_argument("--cohort-manifest", type=Path, required=True)
parser.add_argument("--feature-catalog", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
output_dir = args.output_dir
output_dir.mkdir(parents=True, exist_ok=True)
catalog = load_catalog(args.feature_catalog)
cohort = json.loads(args.cohort_manifest.read_text(encoding="utf-8"))
cohort_members = cohort["members"]
cohort_by_family: dict[str, list[dict[str, object]]] = defaultdict(list)
cohort_by_strategy_id: dict[int, str] = {}
for member in cohort_members:
combo = member["combo"]
triples = tuple(
canonical_key(int(combo[index]), int(combo[index + 1]), float(combo[index + 2]))
for index in range(0, 15, 3)
)
family = json.dumps(triples, separators=(",", ":"))
cohort_by_family[family].append(member)
cohort_by_strategy_id[int(member["source_strategy_id"])] = family
id_counts: Counter[int] = Counter()
primitive_counts: Counter[tuple[int, int, float]] = Counter()
role_counts: Counter[str] = Counter()
role_triple_counts: Counter[tuple[str, int, int, float]] = Counter()
family_counts: Counter[str] = Counter()
primitive_roles: dict[tuple[int, int, float], Counter[str]] = defaultdict(Counter)
found_cohort_strategy_ids: set[int] = set()
source = pq.ParquetFile(args.lineage)
rows = 0
for batch in source.iter_batches(columns=["strategy_id", "combo_json"], batch_size=65_536):
for record in batch.to_pylist():
combo = json.loads(record["combo_json"])
if len(combo) < 15:
raise ValueError(f"strategy {record['strategy_id']} has an incomplete primitive combo")
triples = tuple(
canonical_key(int(combo[index]), int(combo[index + 1]), float(combo[index + 2]))
for index in range(0, 15, 3)
)
family = json.dumps(triples, separators=(",", ":"))
strategy_id = int(record["strategy_id"])
expected_family = cohort_by_strategy_id.get(strategy_id)
if expected_family is not None:
if family != expected_family:
raise ValueError(f"cohort strategy {strategy_id} does not match its manifest combo")
found_cohort_strategy_ids.add(strategy_id)
family_counts[family] += 1
rows += 1
for role, primitive in zip(ROLES, triples, strict=True):
indicator_id, period, p1 = primitive
id_counts[indicator_id] += 1
primitive_counts[primitive] += 1
role_counts[role] += 1
role_triple_counts[(role, indicator_id, period, p1)] += 1
primitive_roles[primitive][role] += 1
if rows != source.metadata.num_rows:
raise ValueError("Parquet batch iteration did not cover every lineage row")
if found_cohort_strategy_ids != set(cohort_by_strategy_id):
raise ValueError("not every cohort source_strategy_id is present in the lineage")
cohort_family_counts = {family: len(members) for family, members in cohort_by_family.items()}
cohort_primitive_counts: Counter[tuple[int, int, float]] = Counter()
cohort_role_triple_counts: Counter[tuple[str, int, int, float]] = Counter()
for family, members in cohort_by_family.items():
triples = json.loads(family)
for _member in members:
for role, raw in zip(ROLES, triples, strict=True):
primitive = canonical_key(*raw)
cohort_primitive_counts[primitive] += 1
cohort_role_triple_counts[(role, *primitive)] += 1
usage_rows: list[dict[str, object]] = []
for indicator_id, count in id_counts.items():
usage_rows.append({"entity_type": "indicator_id", "indicator_id": indicator_id, "usage_count": count})
for role, count in role_counts.items():
usage_rows.append({"entity_type": "role", "role": role, "usage_count": count})
for primitive, count in primitive_counts.items():
usage_rows.append(
{
"entity_type": "primitive",
"primitive": key_text(primitive),
"indicator_id": primitive[0],
"period": primitive[1],
"p1": primitive[2],
"usage_count": count,
"cohort_usage_count": cohort_primitive_counts[primitive],
"classification": catalog.get(primitive, "unregistered"),
}
)
for (role, indicator_id, period, p1), count in role_triple_counts.items():
primitive = canonical_key(indicator_id, period, p1)
usage_rows.append(
{
"entity_type": "role_triple",
"role": role,
"primitive": key_text(primitive),
"indicator_id": indicator_id,
"period": period,
"p1": p1,
"usage_count": count,
"cohort_usage_count": cohort_role_triple_counts[(role, indicator_id, period, p1)],
"classification": catalog.get(primitive, "unregistered"),
}
)
for family, count in family_counts.items():
members = cohort_by_family.get(family, [])
usage_rows.append(
{
"entity_type": "base_family",
"base_family_signature": family,
"usage_count": count,
"cohort_usage_count": len(members),
"cohort_base_families": ",".join(str(member["base_family"]) for member in members) or None,
}
)
usage_path = output_dir / "historical_feature_usage_v1.parquet"
pq.write_table(pa.Table.from_pylist(usage_rows), usage_path, compression="zstd")
primitive_registry = []
for primitive, count in sorted(primitive_counts.items(), key=lambda item: (-item[1], item[0])):
classification = catalog.get(primitive, "unregistered")
primitive_registry.append(
{
"primitive": key_text(primitive),
"indicator_id": primitive[0],
"period": primitive[1],
"p1": primitive[2],
"usage_count": count,
"usage_percent": count * 100 / (rows * len(ROLES)),
"roles": dict(sorted(primitive_roles[primitive].items())),
"cohort_usage_count": cohort_primitive_counts[primitive],
"classification": classification,
"currently_validated": classification == "validated",
}
)
registry_payload = {
"schema_version": 1,
"artifact": "HISTORICAL_PRIMITIVE_REGISTRY_COVERAGE_V1",
"lineage_rows": rows,
"primitive_slots": rows * len(ROLES),
"observed_primitives": len(primitive_counts),
"currently_validated_formula_variants": sum(
classification == "validated" for classification in catalog.values()
),
"observed_currently_validated_primitives": sum(
item["currently_validated"] for item in primitive_registry
),
"weighted_primitive_frontier": frontier(primitive_counts),
"primitives": primitive_registry,
}
registry_path = output_dir / "historical_primitive_registry_coverage_v1.json"
registry_path.write_text(json.dumps(registry_payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
summary = {
"schema_version": 1,
"artifact": "HISTORICAL_FEATURE_USAGE_SUMMARY_V1",
"source": {
"lineage": str(args.lineage),
"lineage_sha256": digest(args.lineage),
"lineage_rows": rows,
"cohort_manifest": str(args.cohort_manifest),
"cohort_manifest_sha256": digest(args.cohort_manifest),
},
"counts": {
"unique_strategy_ids": rows,
"primitive_slots": rows * len(ROLES),
"indicator_ids": len(id_counts),
"exact_role_triples": len(role_triple_counts),
"role_independent_primitives": len(primitive_counts),
"roles": len(role_counts),
"base_families": len(family_counts),
"cohort_members": len(cohort_members),
"cohort_source_strategy_ids_found": len(found_cohort_strategy_ids),
"cohort_base_families_found": sum(family in family_counts for family in cohort_by_family),
"cohort_role_triples": len(cohort_role_triple_counts),
"cohort_primitives": len(cohort_primitive_counts),
"currently_validated_formula_variants": sum(
classification == "validated" for classification in catalog.values()
),
"observed_currently_validated_primitives": sum(
catalog.get(primitive) == "validated" for primitive in primitive_counts
),
},
"role_usage": dict(sorted(role_counts.items())),
"weighted_cumulative_frontiers": {
"role_triples": frontier(role_triple_counts),
"primitives": frontier(primitive_counts),
},
"artifacts": {
"usage_parquet": str(usage_path),
"primitive_registry_coverage": str(registry_path),
},
}
summary_path = output_dir / "historical_feature_usage_v1_summary.json"
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
summary["artifacts"]["usage_parquet_sha256"] = digest(usage_path)
summary["artifacts"]["primitive_registry_coverage_sha256"] = digest(registry_path)
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()

282
scripts/generate_coverage_frontier_v1.py Normal file
View file

@ -0,0 +1,282 @@
"""Build metadata-only historical coverage and 95% implementation artifacts.
This consumes usage aggregates, registry metadata, and the feature catalog. It
never imports, evaluates, or ports indicator formulas.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
from collections import Counter
from pathlib import Path
DENOMINATOR = 3_514_830
TARGETS = (80, 90, 95, 98, 99, 100)
STATE_TO_CLASSIFICATION = {
"validated": "native",
"source_implemented_unvalidated": "missing",
"alias_refused": "unsafe",
"unregistered": "ambiguous",
}
CLASSIFICATIONS = ("native", "missing", "unsafe", "ambiguous")
def canonical_json(payload: object) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False).encode(
"ascii"
)
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def frontier(rows: list[dict[str, object]]) -> list[dict[str, object]]:
cumulative = 0
cursor = 0
output = []
for target in TARGETS:
threshold = math.ceil(DENOMINATOR * target / 100)
while cursor < len(rows) and cumulative < threshold:
cumulative += int(rows[cursor]["usage_slots"])
cursor += 1
output.append(
{
"target_percent": target,
"minimum_slots": threshold,
"canonical_items_required": cursor,
"cumulative_slots": cumulative,
"cumulative_percent": cumulative * 100 / DENOMINATOR,
}
)
return output
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--usage-dir", type=Path, required=True)
parser.add_argument("--feature-catalog", type=Path, required=True)
parser.add_argument("--registry", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
usage_dir = args.usage_dir
summary_path = usage_dir / "historical_feature_usage_v1_summary.json"
primitive_path = usage_dir / "historical_primitive_registry_coverage_v1.json"
parquet_path = usage_dir / "historical_feature_usage_v1.parquet"
summary = json.loads(summary_path.read_text(encoding="utf-8"))
primitive_coverage = json.loads(primitive_path.read_text(encoding="utf-8"))
catalog = json.loads(args.feature_catalog.read_text(encoding="utf-8"))
registry = json.loads(args.registry.read_text(encoding="utf-8"))
if summary["counts"]["primitive_slots"] != DENOMINATOR:
raise ValueError("Historical usage denominator does not equal 3,514,830 slots.")
if primitive_coverage["primitive_slots"] != DENOMINATOR:
raise ValueError("Primitive coverage denominator does not equal 3,514,830 slots.")
catalog_by_key = {
(item["indicator_id"], item["period"], float(item["p1"])): item
for item in catalog["features"]
}
definitions = {item["indicator_id"]: item for item in registry["definitions"]}
# The registry coverage artifact retains each primitive's per-role slot
# counts, so no formula runtime or Parquet reader is required here.
role_triples = [
{
"role": role,
"primitive": item["primitive"],
"usage_count": count,
}
for item in primitive_coverage["primitives"]
for role, count in item["roles"].items()
]
triples_by_primitive: dict[str, list[dict[str, object]]] = {}
for row in role_triples:
triples_by_primitive.setdefault(str(row["primitive"]), []).append(row)
primitives: list[dict[str, object]] = []
for source in primitive_coverage["primitives"]:
key = source["indicator_id"], source["period"], float(source["p1"])
catalog_item = catalog_by_key.get(key)
support_state = catalog_item["support_state"] if catalog_item else "unregistered"
classification = STATE_TO_CLASSIFICATION.get(support_state, "ambiguous")
definition = definitions.get(source["indicator_id"])
triples = sorted(
triples_by_primitive.get(source["primitive"], []), key=lambda item: item["role"]
)
primitives.append(
{
"primitive": source["primitive"],
"indicator_id": source["indicator_id"],
"period": source["period"],
"p1": source["p1"],
"usage_slots": source["usage_count"],
"usage_percent": source["usage_count"] * 100 / DENOMINATOR,
"roles": source["roles"],
"role_triples": [
{"role": item["role"], "usage_slots": item["usage_count"]} for item in triples
],
"role_triple_count": len(triples),
"duplicate_role_assignments": max(0, len(triples) - 1),
"cohort_usage_count": source["cohort_usage_count"],
"support_state": support_state,
"coverage_classification": classification,
"causal_classification": (
catalog_item["causal_classification"] if catalog_item else "not_executable"
),
"indicator_name": definition["name"] if definition else "UNKNOWN",
"registry_status": definition["status"] if definition else "unregistered",
"alias_of": definition["alias_of"] if definition else None,
}
)
primitives.sort(key=lambda item: (-int(item["usage_slots"]), str(item["primitive"])))
primitive_frontier = frontier(primitives)
role_triple_rows = [
{
"role_triple": f"{row['role']}:{row['primitive']}",
"usage_slots": row["usage_count"],
}
for row in role_triples
]
role_triple_rows.sort(key=lambda item: (-int(item["usage_slots"]), str(item["role_triple"])))
role_triple_frontier = frontier(role_triple_rows)
class_counts = Counter({classification: 0 for classification in CLASSIFICATIONS})
class_counts.update(item["coverage_classification"] for item in primitives)
class_slots = Counter({classification: 0 for classification in CLASSIFICATIONS})
for item in primitives:
class_slots[str(item["coverage_classification"])] += int(item["usage_slots"])
aliases = [item for item in primitives if item["registry_status"] == "alias"]
registry_aliases = [
{
"indicator_id": item["indicator_id"],
"indicator_name": item["name"],
"alias_of": item["alias_of"],
"behavior": item["behavior"],
"observed_primitives": sum(
primitive["indicator_id"] == item["indicator_id"] for primitive in primitives
),
}
for item in registry["definitions"]
if item["status"] == "alias"
]
duplicate_primitives = [
item for item in primitives if int(item["duplicate_role_assignments"]) > 0
]
frontier_80_count = next(
item["canonical_items_required"]
for item in primitive_frontier
if item["target_percent"] == 80
)
frontier_95_count = next(
item["canonical_items_required"]
for item in primitive_frontier
if item["target_percent"] == 95
)
selected = primitives[:frontier_95_count]
selected_slots = sum(int(item["usage_slots"]) for item in selected)
plan = []
for index, item in enumerate(selected, start=1):
if item["coverage_classification"] == "native":
continue
item = dict(item)
item["weighted_rank"] = index
item["priority"] = "P0" if index <= frontier_80_count else "P1"
item["action"] = {
"missing": "implement_and_validate",
"unsafe": "retain_refusal_and_recover_alias_semantics",
"ambiguous": "recover_metadata_before_implementation",
}[str(item["coverage_classification"])]
plan.append(item)
source = {
"historical_usage_summary": {"path": str(summary_path), "sha256": sha256(summary_path)},
"primitive_registry_coverage": {
"path": str(primitive_path),
"sha256": sha256(primitive_path),
},
"usage_parquet": {"path": str(parquet_path), "sha256": sha256(parquet_path)},
"feature_catalog": {
"path": str(args.feature_catalog),
"sha256": sha256(args.feature_catalog),
},
"historical_registry": {"path": str(args.registry), "sha256": sha256(args.registry)},
}
coverage = {
"schema_version": 1,
"artifact": "COVERAGE_FRONTIER_V1",
"method": (
"historical usage aggregation plus registry and feature-catalog metadata; "
"no formula evaluation, database access, or DEQ"
),
"denominator": {
"primitive_slots": DENOMINATOR,
"lineage_rows": 702_966,
"slots_per_row": 5,
},
"source_artifacts": source,
"counts": {
"canonical_primitives": len(primitives),
"role_triples": len(role_triples),
"native_formula_variants": 80,
"classifications": dict(sorted(class_counts.items())),
"classification_slots": dict(sorted(class_slots.items())),
"aliases_observed": len(aliases),
"registry_aliases": len(registry_aliases),
"cross_role_duplicate_primitives": len(duplicate_primitives),
},
"frontiers": {
"canonical_primitives": primitive_frontier,
"role_triples": role_triple_frontier,
},
"canonical_primitives": primitives,
"aliases": aliases,
"registry_alias_metadata": registry_aliases,
"duplicate_primitives": duplicate_primitives,
}
implementation = {
"schema_version": 1,
"artifact": "FEATURE_95PCT_IMPLEMENTATION_PLAN_V1",
"method": coverage["method"],
"denominator": coverage["denominator"],
"source_artifacts": source,
"target": {
"coverage_percent": 95,
"minimum_slots": math.ceil(DENOMINATOR * 0.95),
"selected_canonical_primitives": frontier_95_count,
"selected_cumulative_slots": selected_slots,
"selected_cumulative_percent": selected_slots * 100 / DENOMINATOR,
},
"native_coverage": {
"formula_variants": 80,
"selected_native_primitives": sum(
item["coverage_classification"] == "native" for item in selected
),
"selected_native_slots": sum(
int(item["usage_slots"])
for item in selected
if item["coverage_classification"] == "native"
),
},
"actual_plan_size": len(plan),
"plan_classifications": {
classification: sum(item["coverage_classification"] == classification for item in plan)
for classification in CLASSIFICATIONS
},
"plan": plan,
"excluded_after_95_frontier": len(primitives) - frontier_95_count,
}
args.output_dir.mkdir(parents=True, exist_ok=True)
coverage_path = args.output_dir / "coverage_frontier_v1.json"
plan_path = args.output_dir / "feature_95pct_implementation_plan_v1.json"
coverage_path.write_bytes(canonical_json(coverage) + b"\n")
implementation["coverage_frontier_sha256"] = sha256(coverage_path)
plan_path.write_bytes(canonical_json(implementation) + b"\n")
if __name__ == "__main__":
main()

330
scripts/generate_engineering_family_map_v1.py Normal file
View file

@ -0,0 +1,330 @@
"""Generate metadata-only engineering maps for the observed historical frontier.
The historical dispatcher is parsed as source text. No formula module is
imported, evaluated, copied, or otherwise ported by this generator.
"""
from __future__ import annotations
import argparse
import ast
import hashlib
import json
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
FRONTIER_SIZE = 568
ROLES = ("trend", "signal", "trigger", "confirm", "vol")
FAMILY_PREFIXES = (
("ma_", "moving_average"),
("osc_", "oscillator"),
("vol_", "volatility"),
("regime_", "regime"),
("micro_", "microstructure"),
("mom_", "momentum"),
("trend_", "trend_structure"),
("norm_", "normalization"),
("cross_", "cross_indicator"),
("time_", "time_session"),
("pivot_", "reference_level"),
("prev_", "reference_level"),
("rolling_", "reference_level"),
("linreg_channel_", "reference_level"),
("quantile_", "reference_level"),
("donch_", "band_channel"),
("kelt_", "band_channel"),
("bb_", "band_channel"),
("ichimoku_", "band_channel"),
("supertrend", "band_channel"),
("psar", "band_channel"),
("adx", "trend_structure"),
("aroon_", "trend_structure"),
("linreg_", "trend_structure"),
("entropy_", "regime"),
)
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def canonical_json(payload: object) -> bytes:
return json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False).encode(
"ascii"
)
def function_calls(node: ast.AST, functions: set[str]) -> list[str]:
return [
child.func.id
for child in ast.walk(node)
if (
isinstance(child, ast.Call)
and isinstance(child.func, ast.Name)
and child.func.id in functions
)
]
def source_map(path: Path) -> dict[int, dict[str, object]]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
functions = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)}
dispatcher = functions.get("compute_indicator")
if dispatcher is None:
raise ValueError("historical source has no compute_indicator dispatcher")
mapping: dict[int, dict[str, object]] = {}
for node in ast.walk(dispatcher):
if not isinstance(node, ast.Compare) or len(node.ops) != 1:
continue
if not (
isinstance(node.left, ast.Name)
and node.left.id == "ind_id"
and isinstance(node.ops[0], ast.Eq)
and len(node.comparators) == 1
and isinstance(node.comparators[0], ast.Constant)
and isinstance(node.comparators[0].value, int)
):
continue
parent = next(
(
candidate
for candidate in ast.walk(dispatcher)
if isinstance(candidate, ast.If) and candidate.test is node
),
None,
)
if parent is None:
continue
returned = next((item.value for item in parent.body if isinstance(item, ast.Return)), None)
if returned is None:
continue
calls = function_calls(returned, set(functions))
primary = calls[0] if calls else None
if primary is None:
continue
signature = functions[primary].args
inputs = [
argument.arg
for argument in signature.args
if argument.arg in {"close", "high", "low", "volume"}
]
dependencies = sorted(set(function_calls(functions[primary], set(functions))) - {primary})
mapping[node.comparators[0].value] = {
"function": primary,
"source_expression": ast.unparse(returned),
"required_inputs": inputs,
"helper_dependencies": dependencies,
}
return mapping
def family(function: str) -> str:
for prefix, value in FAMILY_PREFIXES:
if function.startswith(prefix):
return value
return "specialized"
def counts(rows: list[dict[str, object]]) -> dict[str, int]:
return {
"primitives": len(rows),
"indicator_ids": len({int(row["indicator_id"]) for row in rows}),
"functions": len({str(row["source_function"]) for row in rows}),
"families": len({str(row["engineering_family"]) for row in rows}),
"parameter_expansions": len(rows) - len({int(row["indicator_id"]) for row in rows}),
"role_duplicates": sum(int(row["role_triple_count"]) - 1 for row in rows),
"native_reuse_primitives": sum(
row["implementation_path"] == "native_reuse" for row in rows
),
"native_reuse_functions": len(
{
str(row["source_function"])
for row in rows
if row["implementation_path"] == "native_reuse"
}
),
"new_algorithm_primitives": sum(
row["implementation_path"] == "new_algorithm" for row in rows
),
"new_algorithm_functions": len(
{
str(row["source_function"])
for row in rows
if row["implementation_path"] == "new_algorithm"
}
),
"alias_refused_primitives": sum(
row["implementation_path"] == "alias_refused" for row in rows
),
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--coverage", type=Path, required=True)
parser.add_argument("--registry", type=Path, required=True)
parser.add_argument("--historical-source", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
coverage = json.loads(args.coverage.read_text(encoding="utf-8"))
registry = json.loads(args.registry.read_text(encoding="utf-8"))
source = source_map(args.historical_source)
definitions = {item["indicator_id"]: item for item in registry["definitions"]}
rows: list[dict[str, object]] = []
for rank, primitive in enumerate(coverage["canonical_primitives"], start=1):
indicator_id = int(primitive["indicator_id"])
source_item = source.get(indicator_id)
if source_item is None:
raise ValueError(
f"observed indicator {indicator_id} is absent from the historical dispatcher"
)
definition = definitions[indicator_id]
source_function = str(source_item["function"])
rows.append(
{
"weighted_rank": rank,
"frontier": "95" if rank <= FRONTIER_SIZE else "95_to_100",
**primitive,
"registry_id": indicator_id,
"registry_name": definition["name"],
"registry_alias_of": definition["alias_of"],
"registry_behavior": definition["behavior"],
"source_function": source_function,
"source_expression": source_item["source_expression"],
"engineering_family": family(source_function),
"parameter_aliases": {"period": "period", "p1": "p1"},
"required_inputs": source_item["required_inputs"],
"helper_dependencies": source_item["helper_dependencies"],
"implementation_path": (
"alias_refused" if definition["status"] == "alias" else "unclassified"
),
}
)
if len(rows) != 711 or len(rows[:FRONTIER_SIZE]) != FRONTIER_SIZE:
raise ValueError("expected exactly 711 observed primitives and a 568-item 95% frontier")
native_functions = {
str(row["source_function"]) for row in rows if row["coverage_classification"] == "native"
}
for row in rows:
if row["implementation_path"] == "alias_refused":
continue
row["implementation_path"] = (
"native_reuse" if row["source_function"] in native_functions else "new_algorithm"
)
selected, delta = rows[:FRONTIER_SIZE], rows[FRONTIER_SIZE:]
total_slots = sum(int(row["usage_slots"]) for row in rows)
selected_slots = sum(int(row["usage_slots"]) for row in selected)
leverage = {
"slots_per_new_algorithm_function": (
selected_slots / counts(selected)["new_algorithm_functions"]
),
"primitives_per_new_algorithm_function": (
counts(selected)["new_algorithm_primitives"]
/ counts(selected)["new_algorithm_functions"]
),
"parameter_expansions_per_function": (
counts(selected)["parameter_expansions"] / counts(selected)["functions"]
),
"role_assignments_per_primitive": (
sum(int(row["role_triple_count"]) for row in selected) / len(selected)
),
}
family_rows = []
for name in sorted({str(row["engineering_family"]) for row in rows}):
items = [row for row in rows if row["engineering_family"] == name]
family_rows.append(
{
"engineering_family": name,
"counts": counts(items),
"usage_slots": sum(int(row["usage_slots"]) for row in items),
}
)
source_artifacts = {
"coverage": {"path": str(args.coverage), "sha256": digest(args.coverage)},
"registry": {"path": str(args.registry), "sha256": digest(args.registry)},
"historical_source": {
"path": str(args.historical_source),
"sha256": digest(args.historical_source),
},
}
family_payload = {
"schema_version": 1,
"artifact": "ENGINEERING_FAMILY_MAP_V1",
"method": (
"static local archaeology only; historical dispatcher AST and registry metadata; "
"no formula import, evaluation, or port"
),
"source_artifacts": source_artifacts,
"counts": counts(rows),
"families": family_rows,
"primitives": rows,
}
frontier_payload = {
"schema_version": 1,
"artifact": "IMPLEMENTATION_FRONTIER_V1",
"method": family_payload["method"],
"source_artifacts": source_artifacts,
"frontier_95": {
"primitive_count": len(selected),
"usage_slots": selected_slots,
"coverage_percent": selected_slots * 100 / total_slots,
"counts": counts(selected),
"leverage": leverage,
},
"full_100": {
"primitive_count": len(rows),
"usage_slots": total_slots,
"coverage_percent": 100.0,
"counts": counts(rows),
},
"delta_95_to_100": {
"primitive_count": len(delta),
"usage_slots": sum(int(row["usage_slots"]) for row in delta),
"coverage_percent": (total_slots - selected_slots) * 100 / total_slots,
"incremental_counts": counts(delta),
},
"frontier_primitives": selected,
"delta_primitives": delta,
}
args.output_dir.mkdir(parents=True, exist_ok=True)
(args.output_dir / "engineering_family_map_v1.json").write_bytes(
canonical_json(family_payload) + b"\n"
)
(args.output_dir / "implementation_frontier_v1.json").write_bytes(
canonical_json(frontier_payload) + b"\n"
)
parquet_rows = [
{
**{
key: value
for key, value in row.items()
if key
not in {
"roles",
"role_triples",
"parameter_aliases",
"required_inputs",
"helper_dependencies",
}
},
"roles_json": json.dumps(row["roles"], sort_keys=True),
"role_triples_json": json.dumps(row["role_triples"], sort_keys=True),
"parameter_aliases_json": json.dumps(row["parameter_aliases"], sort_keys=True),
"required_inputs_json": json.dumps(row["required_inputs"]),
"helper_dependencies_json": json.dumps(row["helper_dependencies"]),
}
for row in rows
]
table = pa.Table.from_pylist(parquet_rows)
pq.write_table(table, args.output_dir / "engineering_family_map_v1.parquet", compression="zstd")
pq.write_table(
table, args.output_dir / "implementation_frontier_v1.parquet", compression="zstd"
)
if __name__ == "__main__":
main()

62
tests/test_batch01_native_parity.py Normal file
View file

@ -0,0 +1,62 @@
from __future__ import annotations
import json
from control_plane.trading_studio.indicators.batch01_native_parity import (
batch01_native_parity_manifest,
batch01_native_parity_manifest_bytes,
evaluate_batch01_native_parity,
)
def test_batch01_native_parity_manifest_pins_local_oracle_inputs():
manifest = batch01_native_parity_manifest()
assert manifest["variant_count"] == 97
assert manifest["comparison"] == {
"dtype": "exact", "shape": "exact", "nan_mask": "exact", "values": "exact"
}
assert manifest["derived_state_transitions"]["indicator_ids"] == [19, 28]
assert manifest["documented_exception"]["global_tolerance"] == "forbidden"
assert json.loads(batch01_native_parity_manifest_bytes()) == manifest
def test_all_97_batch01_native_outputs_match_with_only_documented_bollinger_ulp_cases():
report = evaluate_batch01_native_parity()
state_records = [
item
for item in report["records"]
if item["name"].endswith((":state", ":transitions"))
]
assert len(state_records) == 30
assert all(item["status"] == "pass" for item in state_records)
accepted = [
item
for item in report["records"]
if item["reason"] == "accepted_documented_float64_ulp_drift"
]
assert {item["name"] for item in accepted} == {
"batch01_17_14_2",
"batch01_17_18_2",
"batch01_17_20_1.5",
"batch01_18_14_2",
"batch01_18_18_2",
"batch01_18_20_1.5",
}
assert sum(item["value_mismatches"] for item in accepted) == 8
assert report["acceptance_manifest"]["status"] == "accepted"
coverage = report["acceptance_manifest"]["coverage"]
assert coverage["submitted_feature_versions"] == 97
assert coverage["passed_feature_versions"] == 97
assert coverage["submitted_usage_slots"] == 97
assert coverage["passed_usage_slots"] == 97
assert coverage["coverage_percent"] == 100.0
assert len(coverage["feature_versions"]) == 97
failures = report["failures"]
assert not failures, "\n".join(
f"{item['name']}: {item['reason']} "
f"(dtype {item['expected_dtype']} != {item['actual_dtype']}, "
f"shape {item['expected_shape']} != {item['actual_shape']}, "
f"NaN mismatches={item['nan_mask_mismatches']}, "
f"value mismatches={item['value_mismatches']})"
for item in failures
)

38
tests/test_batch01_oracle_manifest.py Normal file
View file

@ -0,0 +1,38 @@
from __future__ import annotations
import hashlib
import json
from control_plane.trading_studio.indicators.batch01_oracle_manifest import (
batch01_oracle_request_manifest,
batch01_oracle_request_manifest_bytes,
)
from control_plane.trading_studio.indicators.historical_band_channel import (
OBSERVED_BAND_CHANNEL_PARAMS,
)
def test_batch01_oracle_manifest_has_exact_observed_requests_and_semantic_hashes():
manifest = batch01_oracle_request_manifest()
requests = manifest["requests"]
assert manifest["artifact"] == "HISTORICAL_FEATURE_ORACLE_V1_REQUEST"
assert manifest["engine_revision"] == "code-5056feb"
assert len(requests) == 97
assert {
(row["indicator_id"], row["period"], row["p1"])
for row in requests
} == OBSERVED_BAND_CHANNEL_PARAMS
for row in requests:
semantic = {
"engine_revision": manifest["engine_revision"],
"input_columns": manifest["input_columns"],
"output_dtype": manifest["output_dtype"],
"window_policy": manifest["window_policy"],
"indicator_id": row["indicator_id"],
"period": row["period"],
"p1": row["p1"],
}
assert row["semantic_fingerprint"] == hashlib.sha256(
json.dumps(semantic, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
assert json.loads(batch01_oracle_request_manifest_bytes()) == manifest

93
tests/test_cohort_001_specimen.py Normal file
View file

@ -0,0 +1,93 @@
from inspect import getsource
from types import SimpleNamespace
from control_plane.trading_studio.management.commands.specimen_hyperscalper_cohort_001 import (
SCENARIO_NAMES,
Command,
)
def _ledger(*, net=8.0):
return {
"gross_pnl": 10.0,
"net_pnl": net,
"entry_commission": 1.0,
"exit_commission": 1.0,
"entry_slippage": 0.25,
"exit_slippage": 0.25,
"funding": 0.0,
"other_cost": 0.0,
"deq": {
"return_1_bars_bps": 5.0,
"return_2_bars_bps": None,
"return_3_bars_bps": None,
"return_5_bars_bps": None,
"return_10_bars_bps": None,
"mfe_bps": 10.0,
"mae_bps": -2.0,
"time_to_positive_bars": 1,
"time_to_25_bps_bars": None,
"time_to_50_bps_bars": None,
"max_adverse_before_positive_bps": -2.0,
"winner_negative_first": True,
"recovery_bars": 1,
},
}
def test_specimen_contract_is_complete_and_limits_execution_to_first_member():
source = getsource(Command)
assert len(SCENARIO_NAMES) == 13
assert "TOTAL_COST_6BP" in SCENARIO_NAMES
assert "STOP_WIDTH_25" in SCENARIO_NAMES
assert "memberships[0]" in source
def test_deq_report_aggregates_every_primitive_including_missing_values_and_booleans():
report = Command._deq_aggregate([_ledger()])
assert report["trade_count"] == 1
assert set(report["fields"]) == set(_ledger()["deq"])
assert report["fields"]["return_2_bars_bps"] == {"available_count": 0, "mean": None}
assert report["fields"]["winner_negative_first"] == {
"available_count": 1,
"true_count": 1,
"false_count": 0,
}
def test_ledger_reconciliation_checks_summary_and_rescue_totals():
ledger = [_ledger()]
summary = {
"ledger_count": 1,
"gross_pnl": 10.0,
"net_pnl": 8.0,
"rescue": {
"gross_pnl": 10.0,
"net_pnl": 8.0,
"commissions": 2.0,
"slippage": 0.5,
"funding": 0.0,
"other_cost": 0.0,
},
}
assert Command._reconcile(summary, ledger)["consistent"] is True
summary["rescue"]["net_pnl"] = 7.0
assert Command._reconcile(summary, ledger)["consistent"] is False
summary["rescue"]["net_pnl"] = 8.0
ledger[0]["net_pnl"] = 7.0
assert Command._reconcile(summary, ledger)["row_pnl_consistent"] is False
def test_report_has_machine_and_human_sections_and_refuses_monetization_claim_without_criteria():
scenario = {"ledger_reconciliation": {"consistent": True}}
cohort = SimpleNamespace(id="cohort", dataset_version_id="dataset", policy_snapshot={})
membership = SimpleNamespace(ordinal=1, id="member", strategy_version_id="strategy")
report = Command._report(cohort, membership, [{"fold": "Fold 1", "scenarios": [scenario]}])
assert report["machine"]["edge_vs_monetization"]["status"] == "INSUFFICIENT"
assert report["human"]["ledger_reconciled"] is True
assert report["machine"]["member_ordinal"] == 1

139
tests/test_cohort_qualification_hyperscalper_001.py Normal file
View file

@ -0,0 +1,139 @@
from inspect import getsource
from types import SimpleNamespace
import pytest
from control_plane.trading_studio.management.commands.cohort_qualification_hyperscalper_001 import (
CONTRACT,
RUNNER_NAME,
Command,
)
from control_plane.trading_studio.management.commands.specimen_hyperscalper_cohort_001 import (
Command as SpecimenCommand,
SCENARIO_NAMES,
)
def _ledger(net_pnl=1.0):
return {
"exit_reason": "TP",
"take_profit_price": 101.0,
"entry_execution_price": 100.0,
"net_pnl": net_pnl,
"deq": {
"return_1_bars_bps": 10.0,
"return_2_bars_bps": None,
"return_3_bars_bps": None,
"return_5_bars_bps": None,
"return_10_bars_bps": None,
"mfe_bps": 12.0,
"mae_bps": -1.0,
"time_to_positive_bars": 1,
"time_to_25_bps_bars": None,
"time_to_50_bps_bars": None,
"max_adverse_before_positive_bps": -1.0,
"winner_negative_first": True,
"recovery_bars": 1,
},
}
def _legacy_specimen():
scenario = {
"qualification_run_id": "run",
"scenario": "",
"summary": {},
"ledger": [],
"ledger_reconciliation": {},
"deq": {},
"rescue": {},
}
return {
"machine": {
"contract": "Cohort001-first-member-specimen-v1",
"member_ordinal": 1,
"fold_count": 4,
"scenario_count_per_fold": 13,
"folds": [
{
"fold": f"Fold {number}",
"scenarios": [{**scenario, "scenario": name} for name in SCENARIO_NAMES],
}
for number in range(1, 5)
],
}
}
def test_cohort_command_contract_limits_new_execution_to_ordinals_two_through_twenty():
source = getsource(Command)
assert CONTRACT == "Cohort001-qualification-v1"
assert RUNNER_NAME.endswith(":v1")
assert len(SCENARIO_NAMES) == 13
assert "memberships[1:]" in source
assert "QualificationReplayRun.objects.filter" in source
assert "mixed runner versions" in source
assert "CanaryCommand._native_state" in source
assert "--specimen-report-sha" in source
assert "externally frozen SHA" in source
assert "require_protocol_identity=True" in source
assert "specimen_configuration_sha256" not in getsource(SpecimenCommand._report)
assert "allow_empty_raw=True" in source
def test_legacy_specimen_validation_requires_contract_four_folds_scenarios_and_ledger_fields():
report = _legacy_specimen()
Command._validate_specimen(report)
report["machine"]["folds"][0]["scenarios"][0]["ledger"] = [{"net_pnl": 1.0}]
with pytest.raises(ValueError, match="ledger fields"):
Command._validate_specimen(report)
def test_zero_trade_fold_reconciles_without_fabricated_ledger_rows():
summary = {
"ledger_count": 0,
"gross_pnl": 0.0,
"net_pnl": 0.0,
"rescue": {
"gross_pnl": 0.0,
"net_pnl": 0.0,
"commissions": 0.0,
"slippage": 0.0,
"funding": 0.0,
"other_cost": 0.0,
},
}
assert SpecimenCommand._reconcile(summary, [])["consistent"] is True
def test_indicator_expansion_reports_required_classes_and_missing_data_transparently():
expansion = Command._indicator_expansion(
[{"ledger": [_ledger(), _ledger(net_pnl=-1.0)]}]
)
assert expansion["status"] == "AVAILABLE"
assert expansion["classes"]["exit_reason"] == {"TP": 2}
assert expansion["classes"]["net_pnl"] == {"LOSS": 1, "WIN": 1}
assert expansion["deq_distributions"]["return_2_bars_bps"] == {
"available_count": 0,
"missing_count": 2,
"status": "INSUFFICIENT",
}
assert "Required classes" in Command._indicator_brief(expansion)
def test_machine_and_human_report_bind_catalog_and_external_specimen_hashes():
cohort = SimpleNamespace(id="cohort")
report = Command._report(cohort, [], "manifest", "dataset", "config", "specimen")
assert report["machine"]["protocol_identity"] == {
"cohort_manifest_sha256": "manifest",
"dataset_sha256": "dataset",
"qualification_scenario_catalog_sha256": "config",
"specimen_report_sha256": "specimen",
"runner_name": RUNNER_NAME,
}
assert report["human"]["indicator_expansion_status"] == "INSUFFICIENT"

59
tests/test_feature_catalog_v1.py Normal file
View file

@ -0,0 +1,59 @@
from __future__ import annotations
import json
import pytest
from django.core.management import call_command
from control_plane.trading_studio.indicators.definitions import IndicatorVariant, Role
from control_plane.trading_studio.indicators.feature_catalog import (
FEATURE_CATALOG_V1,
FEATURE_CATALOG_V1_PATH,
FEATURE_SAFETY_AUDIT_V1_PATH,
CausalClassification,
FeatureSupportState,
FeatureUnavailableError,
feature_catalog_artifact_bytes,
feature_definition,
feature_safety_audit_artifact_bytes,
require_validated_features,
)
def test_catalog_explicitly_records_validated_coverage_and_source_only_scope():
assert len(FEATURE_CATALOG_V1) == 1107
validated = {
item.key
for item in FEATURE_CATALOG_V1
if item.support_state == FeatureSupportState.VALIDATED
}
assert len(validated) == 80
artifact = json.loads(feature_catalog_artifact_bytes())
assert artifact["implementation_coverage"]["coverage"] == "80 of 1107"
assert artifact["evaluation"] == "not evaluated via reference runtime; no strategy search"
def test_safety_audit_derives_causal_classification_from_source_semantics():
trailing = feature_definition(IndicatorVariant(30, 14, 0.0, Role.OSC))
index_based = feature_definition(IndicatorVariant(117, 1, 0.0, Role.TREND))
alias = feature_definition(IndicatorVariant(25, 14, 0.0, Role.LEVEL))
assert trailing.causal_classification == CausalClassification.TRAILING_WINDOW_CAUSAL
assert index_based.causal_classification == CausalClassification.INDEX_POSITION_CAUSAL
assert alias.support_state == FeatureSupportState.ALIAS_REFUSED
policy = json.loads(feature_safety_audit_artifact_bytes())["fail_closed_policy"]
assert policy.startswith("Any requested")
def test_non_validated_historical_variants_fail_closed_without_baseline_expansion():
validated = IndicatorVariant(30, 14, 0.0, Role.OSC)
unvalidated = IndicatorVariant(30, 12, 0.0, Role.OSC)
with pytest.raises(FeatureUnavailableError, match="fails closed"):
require_validated_features([validated, unvalidated])
def test_generator_writes_machine_readable_catalog_and_audit(tmp_path):
call_command("tradingstudio_generate_feature_catalog", "--output-dir", str(tmp_path))
catalog = tmp_path / FEATURE_CATALOG_V1_PATH.name
audit = tmp_path / FEATURE_SAFETY_AUDIT_V1_PATH.name
assert json.loads(catalog.read_text())["catalog"] == "FEATURE_CATALOG_V1"
assert json.loads(audit.read_text())["audit"] == "FEATURE_SAFETY_AUDIT_V1"

47
tests/test_feature_parity_harness_v1.py Normal file
View file

@ -0,0 +1,47 @@
from __future__ import annotations
import numpy as np
import pytest
from control_plane.trading_studio.indicators.parity_harness_v1 import (
FeatureVersion,
ParityStatus,
compare_array,
compare_state,
enumerate_native_evaluators,
load_frozen_engineering_map,
parity_manifest,
recompute_coverage,
)
pytestmark = pytest.mark.django_db
def test_manifest_is_pinned_to_the_frozen_engineering_map():
assert load_frozen_engineering_map()["counts"]["primitives"] == 711
assert parity_manifest()["comparison"]["tolerance_by_engineering_family"] == {}
def test_comparisons_record_exact_dtype_nan_and_state_key_failures():
record = compare_array(
"feature", np.array([np.nan, 1.0]), np.array([np.nan, 1.0], dtype=np.float32)
)
assert record.status == ParityStatus.FAIL
assert record.expected_dtype == "float64"
assert record.nan_mask_mismatches == 0
state = compare_state(
{"trend": np.array([1], dtype=np.int8)},
{"signal": np.array([1], dtype=np.int8)},
)
assert {item.reason for item in state} == {"state_key_mismatch"}
def test_native_enumeration_and_coverage_use_only_submitted_versions():
passed = FeatureVersion("v1", "p1", "trend", 9, lambda _: np.array([1]), ParityStatus.PASS)
failed = FeatureVersion("v2", "p2", "trend", 1, None, ParityStatus.FAIL)
assert enumerate_native_evaluators([passed, failed]) == (passed,)
assert recompute_coverage([passed, failed]) == {
"submitted_feature_versions": 2, "passed_feature_versions": 1,
"submitted_usage_slots": 10, "passed_usage_slots": 9,
"coverage_percent": 90.0, "feature_versions": ["v1"],
}

39
tests/test_historical_band_channel.py Normal file
View file

@ -0,0 +1,39 @@
from __future__ import annotations
import numpy as np
import pytest
from control_plane.trading_studio.indicators.historical_band_channel import (
NATIVE_EVALUATORS,
OBSERVED_BAND_CHANNEL_PARAMS,
evaluate_band_channel,
)
from control_plane.trading_studio.indicators.historical_formulae import compute_indicator
def _ohlcv() -> tuple[np.ndarray, ...]:
close = np.linspace(100.0, 140.0, 80) + np.sin(np.arange(80) / 3)
high = close + 1.5 + (np.arange(80) % 3) / 10
low = close - 1.25 - (np.arange(80) % 4) / 10
volume = np.linspace(1_000.0, 2_000.0, 80)
return close, high, low, volume
def test_batch01_registers_only_the_97_observed_band_channel_primitives():
assert set(NATIVE_EVALUATORS) == {17, 18, 19, 20, 21, 23, 24, 28}
assert len(OBSERVED_BAND_CHANNEL_PARAMS) == 97
assert (22, 10, 0.0) not in OBSERVED_BAND_CHANNEL_PARAMS
@pytest.mark.parametrize("indicator_id,period,p1", sorted(OBSERVED_BAND_CHANNEL_PARAMS))
def test_batch01_matches_recovered_historical_formulae(indicator_id: int, period: int, p1: float):
close, high, low, volume = _ohlcv()
actual = evaluate_band_channel(indicator_id, close, high, low, volume, period, p1)
expected = compute_indicator(indicator_id, close, high, low, volume, period, p1)
np.testing.assert_array_equal(actual, expected)
def test_batch01_refuses_unobserved_parameter_expansion():
close, high, low, volume = _ohlcv()
with pytest.raises(ValueError, match="unsupported Batch01"):
evaluate_band_channel(20, close, high, low, volume, 12, 0.0)

125
tests/test_hs22_oracle_parity.py Normal file
View file

@ -0,0 +1,125 @@
from __future__ import annotations
import csv
import json
from hashlib import sha256
from pathlib import Path
import numpy as np
import pytest
from control_plane.trading_studio.indicators.definitions import IndicatorVariant, Role
from control_plane.trading_studio.indicators.engine import compute_hs22_state, evaluate_indicator
ORACLE = Path(__file__).resolve().parents[1] / "hs22_oracle_v1"
CSV_SHA256 = "7fb56bb05b0edc2348407765cd693efadf334307c25877dfb52acfe4221e5d00"
# Spark emitted the oracle on NumPy 2.4.6/aarch64; this Windows runner uses
# NumPy 2.4.4. The largest observed arithmetic drift is 1.11e-15 (IDs 79/76).
FLOAT_ATOL = 1.2e-15
SLOT_ROLE = {
"trend": Role.TREND,
"signal": Role.OSC,
"trigger": Role.LEVEL,
"confirm": Role.OSC,
"vol": Role.FILTER,
}
@pytest.fixture(scope="session")
def oracle():
assert sha256((ORACLE / "binance_btcusdt_spot_2m_180d.csv").read_bytes()).hexdigest() == CSV_SHA256
manifest = json.loads((ORACLE / "required_variants.json").read_text(encoding="utf-8"))
with (ORACLE / "binance_btcusdt_spot_2m_180d.csv").open(newline="", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
assert len(rows) == 129_599
return manifest, tuple(
np.asarray([float(row[name]) for row in rows], dtype=np.float64)
for name in ("open", "high", "low", "close", "volume")
)
def _assert_exact(actual: np.ndarray, expected: np.ndarray) -> None:
assert actual.dtype == expected.dtype
assert actual.shape == expected.shape
assert np.array_equal(np.isnan(actual), np.isnan(expected))
if np.issubdtype(actual.dtype, np.integer):
assert np.array_equal(actual, expected)
else:
np.testing.assert_allclose(actual, expected, rtol=0.0, atol=FLOAT_ATOL, equal_nan=True)
def _variant(slot: str, item: dict[str, float]) -> IndicatorVariant:
return IndicatorVariant(int(item["indicator_id"]), int(item["period"]), item["p1"], SLOT_ROLE[slot])
@pytest.fixture(scope="session")
def native_variants(oracle):
manifest, (_, high, low, close, volume) = oracle
outputs = {}
with np.load(ORACLE / "required_variant_arrays.npz") as expected:
for combo in manifest["combos"]:
for slot, item in combo["role_triples"].items():
variant = _variant(slot, item)
key = (variant.indicator_id, variant.period, variant.p1)
actual = evaluate_indicator(variant, close, high, low, volume)
oracle_key = "v_" + combo["role_variant_fingerprints"][slot]
_assert_exact(actual, expected[oracle_key])
outputs[key] = actual
assert len(outputs) == 80
return outputs
def test_all_80_required_variant_arrays_match_the_frozen_oracle(native_variants):
assert len(native_variants) == 80
def _decisions(state: dict[str, np.ndarray], close: np.ndarray, combo: list[float]) -> np.ndarray:
vol = state["vol"]
valid = vol[~np.isnan(vol)]
threshold = np.sort(valid)[min(int(combo[21] / 100.0 * len(valid)), len(valid) - 1)]
out = np.zeros(len(close), dtype=np.int8)
signal = state["signal_color"]
trend = state["trend_color"]
common = np.isfinite(state["trigger"]) & np.isfinite(state["confirm"]) & np.isfinite(vol)
common &= vol >= threshold
flips = (signal != np.roll(signal, 1)) & (signal != 0)
long = common & flips & (signal == 1) & (trend == 1) & (close > state["trigger"])
long &= (combo[17] <= state["confirm"]) & (state["confirm"] <= combo[18])
short = common & flips & (signal == -1) & (trend == -1) & (close < state["trigger"])
short &= (combo[19] <= state["confirm"]) & (state["confirm"] <= combo[20])
out[long] = 1
out[short] = -1
return out
def test_all_20_hs22_states_and_decisions_match_the_frozen_oracle(oracle):
manifest, (_, high, low, close, volume) = oracle
names = ("trend", "signal", "trigger", "confirm", "vol", "trend_color", "signal_color")
for combo in manifest["combos"]:
actual = compute_hs22_state(combo["combo"], close, high, low, volume)
with np.load(ORACLE / combo["state_file"]) as expected:
for name in names:
_assert_exact(actual[name], expected[name])
expected_state = {name: expected[name] for name in names}
assert np.count_nonzero(_decisions(actual, close, combo["combo"]) != _decisions(expected_state, close, combo["combo"])) == 0
def test_fold1_context_prefix_matches_continuous_history(native_variants, oracle):
_, (_, high, low, close, volume) = oracle
fold1_test_stop = 14_400 + 35 + 7_200
fold1_test = slice(14_400 + 35, fold1_test_stop)
for (indicator_id, period, p1), full in native_variants.items():
# Formula output is role-independent; choose the first registered role for this triple.
for candidate in SLOT_ROLE.values():
try:
context = evaluate_indicator(
IndicatorVariant(indicator_id, period, p1, candidate),
close[:fold1_test_stop], high[:fold1_test_stop], low[:fold1_test_stop], volume[:fold1_test_stop],
)
break
except ValueError:
continue
else:
raise AssertionError("required variant lost its strict role registration")
_assert_exact(context[fold1_test], full[fold1_test])

260
tests/test_hyperscalper_cohort_materialization.py Normal file
View file

@ -0,0 +1,260 @@
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from inspect import getsource
import numpy as np
import pytest
from control_plane.trading_studio import cohort_materialization as materialization
from control_plane.trading_studio.management.commands.canary_hyperscalper_cohort_001 import (
Command as CanaryCommand,
)
from control_plane.trading_studio.management.commands.materialize_hyperscalper_cohort_001 import (
DATASET_SHA256,
DATASET_VERSION_ID,
Command,
)
def row(index):
return {
"timestamp": datetime(2025, 1, 1, tzinfo=UTC) + timedelta(minutes=index),
"open": 1.0,
"high": 1.1,
"low": 0.9,
"close": 1.0,
}
def test_canonical_manifest_serialization_is_sorted_pretty_ascii_without_newline():
manifest = {"z": "cafe", "a": [1, {"b": True}]}
serialized = materialization.canonical_json(manifest)
assert (
serialized == '{\n "a": [\n 1,\n {\n "b": true\n }\n ],\n "z": "cafe"\n}'
)
assert sha256(serialized.encode()).hexdigest() == (
"c1f9b496e9724f3cf4b0ab781589f6d8db6e48f5de90f9ea9c39defb20748dc9"
)
def test_manifest_requires_canonical_self_semantic_and_lineage_hashes(monkeypatch):
monkeypatch.setattr(materialization, "MANIFEST_HASH_PREFIX", "")
records = [
{
"rank": 1,
"source_strategy_id": 490761,
"source_name": "B7435 TP1.8/SL0.8",
"base_family": "B7435",
"combo": [79, 30, 8.0],
"old_selection_metrics": {"id": 490761},
"old_final_metrics": {"id": 490761},
}
]
monkeypatch.setattr(
materialization, "SEMANTIC_DIGEST", materialization.semantic_digest(records)
)
manifest = {"members": records, "lineage": {"source": "frozen"}}
manifest["lineage_sha256"] = materialization.digest(manifest["lineage"])
manifest["manifest_sha256"] = materialization.digest(manifest)
materialization.validate_manifest(manifest, cohort=True)
manifest["lineage"]["source"] = "changed"
manifest["manifest_sha256"] = materialization.digest(
{key: value for key, value in manifest.items() if key != "manifest_sha256"}
)
with pytest.raises(ValueError, match="lineage"):
materialization.validate_manifest(manifest, cohort=True)
def test_semantic_digest_uses_compact_sorted_projection_and_required_full_digest():
assert materialization.SEMANTIC_DIGEST == (
"cc697ea359f20ccc83c5d62f5b406a3532c1d7411191423059164aafe5931254"
)
records = [
{
"rank": 2,
"source_strategy_id": 2,
"source_name": "B2",
"base_family": "B2",
"combo": [2],
"old_selection_metrics": {"id": 2},
"old_final_metrics": {"id": 2},
},
{
"rank": 1,
"source_strategy_id": 1,
"source_name": "B1",
"base_family": "B1",
"combo": [1],
"old_selection_metrics": {"id": 1},
"old_final_metrics": {"id": 1},
},
]
with pytest.raises(ValueError, match="uniquely ordered"):
materialization.semantic_digest(records)
def test_reconstructed_folds_have_exact_train_embargo_and_test_windows():
rows = [row(index) for index in range(materialization.DATASET_RECORD_COUNT)]
folds = materialization.reconstruct_folds(rows, "timestamp")
assert len(folds) == 4
for fold in folds:
assert fold["train"]["end_index"] - fold["train"]["start_index"] + 1 == 14_400
assert fold["embargo"]["bars"] == 35
assert fold["embargo"]["status"] == "EXCLUDED"
assert fold["embargo"]["start_index"] == fold["train"]["end_index"] + 1
assert fold["test"]["start_index"] == fold["embargo"]["end_index"] + 1
assert fold["test"]["end_index"] - fold["test"]["start_index"] + 1 == 7_200
assert fold["provenance"] == "RECONSTRUCTED_FROM_FROZEN_SPEC_V1"
def test_csv_validation_rejects_noncontinuous_timestamps(monkeypatch, tmp_path):
monkeypatch.setattr(materialization, "DATASET_RECORD_COUNT", 3)
path = tmp_path / "dataset.csv"
path.write_text(
"timestamp,open,high,low,close\n"
"1735689600,1,1.1,0.9,1\n"
"1735689720,1,1.1,0.9,1\n"
"1735690000,1,1.1,0.9,1\n",
encoding="utf-8",
)
rows, headers = materialization.load_csv(path, "timestamp")
manifest = {"columns": headers, "timestamp_field": "timestamp"}
with pytest.raises(ValueError, match="continuity"):
materialization.validate_rows(rows, headers, manifest)
def test_dataset_manifest_treats_sha256_as_the_csv_artifact_hash_not_a_self_hash():
manifest = {
"artifact": "artifact://binance_btcusdt_spot_2m_180d.csv",
"sha256": "a" * 64,
"row_count": materialization.DATASET_RECORD_COUNT,
"fields": ["timestamp", "open", "high", "low", "close", "volume"],
"monotonic_timestamps": True,
"schema_valid": True,
}
materialization.validate_dataset_manifest(manifest)
manifest["schema_valid"] = False
with pytest.raises(ValueError, match="declarations"):
materialization.validate_dataset_manifest(manifest)
def test_existing_dataset_recovery_uses_crypto_hyperscalper_and_real_artifact_path():
source = getsource(Command._dataset)
assert 'slug != "crypto-hyperscalper"' in source
assert "(root / DATASET_CSV).resolve()" in source
assert '"first_timestamp"' in source
assert '"last_timestamp"' in source
def test_actual_shape_member_maps_to_stable_semantic_ids_and_authorized_dataset():
member = {
"rank": 1,
"source_strategy_id": 490761,
"source_name": "B7435 TP1.8/SL0.8",
"base_family": "B7435",
"combo": [79, 30, 8.0],
"old_selection_metrics": {"id": 490761},
"old_final_metrics": {"id": 490761},
}
assert Command._fingerprint(member) == Command._fingerprint(dict(member))
assert Command._member_id(member) == Command._member_id(dict(member))
assert str(DATASET_VERSION_ID) == "1f224db9-cbb8-4aec-8575-98c4b0279a83"
assert DATASET_SHA256 == "7fb56bb05b0edc2348407765cd693efadf334307c25877dfb52acfe4221e5d00"
def test_prior_cohort_policy_preserves_historic_manifest_when_reconstruction_is_added():
policy = {"manifest_sha256": "4d20historic", "prior_note": "preserve"}
assert Command._historic_manifest_sha(policy) == "4d20historic"
policy["reconstruction_v1"] = {"folds": ["reconstructed"]}
assert policy["prior_note"] == "preserve"
assert Command._historic_manifest_sha(policy) == "4d20historic"
def test_canary_reads_reconstruction_v1_instead_of_obsolete_frozen_manifest_policy():
source = getsource(CanaryCommand._validate_reconstruction)
assert 'get("reconstruction_v1")' in source
assert "DATASET_VERSION_ID" in source
assert "reconstruct_folds" in source
assert "frozen_manifest" not in source
def test_canary_uses_native_state_and_keeps_continuous_context():
rows = [
{**row(index), "volume": 100.0}
for index in range(3)
]
state = {
"trend": np.array([1.0, 2.0, 3.0]),
"signal": np.array([4.0, 5.0, 6.0]),
"trigger": np.array([0.5, 0.5, 0.5]),
"confirm": np.array([7.0, 8.0, 9.0]),
"vol": np.array([10.0, 11.0, 12.0]),
"trend_color": np.array([0, 1, 1]),
"signal_color": np.array([0, 1, 1]),
"decision": np.array([0, 1, -1]),
}
bars = CanaryCommand._bars(rows, state, {"start_index": 1, "end_index": 2})
source = getsource(CanaryCommand)
assert [bar.signal for bar in bars] == [1, -1]
assert bars[0].context == (2.0, 5.0, 0.5, 8.0, 11.0, 1.0, 1.0)
assert bars[0].reference == "hs22-native:1"
assert "compute_hs22_state" in source
assert "compute_combo_state" not in source
assert "precomputed" not in source
def test_canary_derives_signed_decisions_from_native_hs22_state(monkeypatch):
state = {
"trigger": np.array([1.0, 1.0, 1.0]),
"confirm": np.array([0.0, 0.0, 0.0]),
"vol": np.array([1.0, 1.0, 1.0]),
"signal_color": np.array([0, 1, -1]),
"trend_color": np.array([0, 1, -1]),
}
monkeypatch.setattr(
(
"control_plane.trading_studio.management.commands."
"canary_hyperscalper_cohort_001.compute_hs22_state"
),
lambda *args: state,
)
rows = [
{"close": 1.0, "high": 1.0, "low": 1.0, "volume": 1.0},
{"close": 2.0, "high": 2.0, "low": 2.0, "volume": 1.0},
{"close": 0.5, "high": 0.5, "low": 0.5, "volume": 1.0},
]
combo = [0.0] * 22
combo[17:21] = [-1.0, 1.0, -1.0, 1.0]
actual = CanaryCommand._native_state(rows, {"combo": combo})
assert actual["decision"].tolist() == [0, 1, -1]
def test_selection_source_must_match_declared_path_and_all_frozen_member_records(tmp_path):
source = tmp_path / "selection.json"
selection = {
"selected": [
{
"selection": {"id": 1, "name": "B1"},
"final": {"id": 1, "name": "B1"},
}
]
* 20
}
source.write_text(__import__("json").dumps(selection), encoding="utf-8")
members = [
{
"source_strategy_id": 1,
"source_name": "B1",
"old_selection_metrics": {"id": 1, "name": "B1"},
"old_final_metrics": {"id": 1, "name": "B1"},
}
for _ in range(20)
]
manifest = {"source_report": str(source), "members": members}
assert materialization.validate_selection_source(manifest, selection, source)
selection["selected"][0]["selection"]["id"] = 2
with pytest.raises(ValueError, match="does not reproduce"):
materialization.validate_selection_source(manifest, selection, source)

100
tests/test_indicator_registry.py Normal file
View file

@ -0,0 +1,100 @@
from __future__ import annotations
import hashlib
import json
import pytest
from control_plane.trading_studio.indicators.definitions import IndicatorVariant, Role
from control_plane.trading_studio.indicators.engine import IndicatorEngine, RefusalCode
from control_plane.trading_studio.indicators.registry import (
EXPERIMENTAL_UNIVERSE,
HISTORICAL_ARTIFACT_PATH,
HISTORICAL_REGISTRY,
HS22_COHORT001_V1,
HS22_HISTORICAL_COMPLETE_V1,
UNREGISTERED_INDICATORS,
historical_artifact_bytes,
historical_artifact_digest,
)
from control_plane.trading_studio.indicators.schema import HS22SchemaError, parse_hs22
def test_historical_inventory_counts_role_pools_and_digest_are_stable():
assert len(HISTORICAL_REGISTRY.definitions) == 145
assert len(HISTORICAL_REGISTRY.variants) == 1107
assert {role.value: len(HS22_HISTORICAL_COMPLETE_V1.by_role(role)) for role in Role} == {
"level": 493,
"osc": 372,
"trend": 103,
"filter": 139,
}
assert historical_artifact_digest() == (
"9be2aa3c56c90d2ffc73b64e857092c320fd57b6bbb77a358611a267b52c5ab2"
)
assert (
hashlib.sha256(HISTORICAL_ARTIFACT_PATH.read_bytes().rstrip()).hexdigest()
== historical_artifact_digest()
)
def test_artifact_is_machine_readable_and_matches_registry():
artifact = json.loads(HISTORICAL_ARTIFACT_PATH.read_text())
assert artifact["definition_count"] == 145
assert artifact["variant_count"] == 1107
assert historical_artifact_bytes() == HISTORICAL_ARTIFACT_PATH.read_bytes().rstrip()
def test_recovered_boundary_grids_are_not_truncated_or_reclassified():
assert len([v for v in HISTORICAL_REGISTRY.variants if v.indicator_id == 14]) == 8
assert len([v for v in HISTORICAL_REGISTRY.variants if v.indicator_id == 28]) == 3
assert len([v for v in HISTORICAL_REGISTRY.variants if v.indicator_id == 41]) == 8
clustering = [v for v in HISTORICAL_REGISTRY.variants if v.indicator_id == 62]
assert [v.period for v in clustering] == [10, 15, 20, 25, 30]
assert {v.role for v in clustering} == {Role.TREND}
def test_aliases_and_unregistered_metadata_are_explicit():
assert HISTORICAL_REGISTRY.definition(25).alias_of == 1
assert HISTORICAL_REGISTRY.definition(26).status.value == "alias"
assert HISTORICAL_REGISTRY.definition(27).status.value == "alias"
assert "MOM_JERK" in UNREGISTERED_INDICATORS
def test_unknown_and_unsupported_variants_never_fall_back():
engine = IndicatorEngine()
unknown = engine.resolve(IndicatorVariant(9999, 14, 0.0, Role.OSC))
unsupported = engine.resolve(IndicatorVariant(30, 7, 0.0, Role.OSC))
assert unknown.code == RefusalCode.UNKNOWN_INDICATOR
assert unknown.all_nan_behavior is True
assert unsupported.code == RefusalCode.UNSUPPORTED_VARIANT
def test_hs22_parser_requires_registered_role_correct_variants():
state = parse_hs22(
{
"trend": [70, 10, 0],
"signal": [30, 6, 0],
"trigger": [0, 5, 0],
"confirm": [31, 5, 0],
"volatility": [50, 8, 0],
}
)
assert state.trend.role == Role.TREND
with pytest.raises(HS22SchemaError, match="no fallback"):
parse_hs22(
{
"trend": [70, 10, 0],
"signal": [30, 7, 0],
"trigger": [0, 5, 0],
"confirm": [31, 5, 0],
"volatility": [50, 8, 0],
}
)
def test_derived_and_experimental_universes_do_not_expand_to_historical_pool():
assert HS22_COHORT001_V1.namespace == "derived"
assert HS22_COHORT001_V1.variants == ()
assert EXPERIMENTAL_UNIVERSE.namespace == "experimental"
assert EXPERIMENTAL_UNIVERSE.variants == ()

380
tests/test_qualification_replay_v1.py Normal file
View file

@ -0,0 +1,380 @@
from datetime import UTC, datetime, timedelta
import pytest
from control_plane.projects.models import Project
from control_plane.trading_studio.models import (
DataKind,
LiveStrategyRun,
MarketDataset,
MarketDatasetVersion,
Strategy,
StrategyVersion,
TradingProject,
)
from control_plane.trading_studio.qualification import (
Bar,
CalibrationMethod,
CostModel,
QualificationReplayV1,
ScenarioSpec,
calibrate_train_only,
qualification_scenario_catalog,
)
from control_plane.trading_studio.services import TradingStudioService
BASE = datetime(2026, 1, 5, tzinfo=UTC)
def bar(day, open_, high, low, close, signal=None, context=(), reference=""):
return Bar(BASE + timedelta(days=day), open_, high, low, close, signal, context, reference)
def calibrated(bars, *, method=CalibrationMethod.RAW):
return calibrate_train_only(
bars,
method=method,
fold="fold-a",
train_start=BASE,
train_end=BASE + timedelta(days=1),
source_record_ids=("source-1", "source-2"),
source_combo="ohlcv+context",
percentile=50,
)
def run(bars, scenario=None):
return QualificationReplayV1().run(
bars,
strategy="strategy-v1",
dataset="Cohort001",
fold="fold-a",
runner="test-runner",
scenario=scenario or ScenarioSpec("base", 100, 100),
calibration=calibrated(bars),
)
def test_multibar_state_machine_emits_signal_entry_exit_and_ledger_directly():
result = run(
[
bar(0, 100, 101, 99, 100, 1),
bar(1, 100, 101, 99, 100, 2),
bar(2, 100, 102, 99, 101),
bar(3, 101, 103, 100, 102),
]
)
assert [event.event_type for event in result.events] == [
"SIGNAL",
"ENTRY_QUEUED",
"ENTRY",
"EXIT",
]
assert result.ledger[0].holding_bars == 1
assert result.ledger[0].entry_bar == 2
def test_same_bar_tp_sl_ambiguity_is_pessimistic_stop():
result = run(
[bar(0, 100, 101, 99, 100, 1), bar(1, 100, 101, 99, 100, 2), bar(2, 100, 102, 98, 101)]
)
assert result.ledger[0].exit_reason == "SL"
assert result.ledger[0].exit_execution_price == 99
def test_delay_creates_natural_later_entry():
result = run(
[
bar(0, 100, 101, 99, 100, 1),
bar(1, 100, 101, 99, 100, 2),
bar(2, 100, 100, 99, 100),
bar(3, 100, 102, 99, 101),
],
ScenarioSpec("delay", 100, 100, entry_delay_bars=2),
)
assert result.ledger[0].entry_bar == 3
def test_stop_width_changes_later_eligibility():
bars = [
bar(0, 100, 101, 99, 100, 1),
bar(1, 100, 101, 99, 100, 2),
bar(2, 100, 100.5, 98.5, 99),
bar(3, 99, 101, 98, 100, 2),
bar(4, 100, 102, 99, 101),
]
narrow = run(bars, ScenarioSpec("narrow", 100, 100, cooldown_bars=1))
wide = run(bars, ScenarioSpec("wide", 300, 100, cooldown_bars=1, max_holding_bars=3))
assert len(narrow.ledger) == 2
assert len(wide.ledger) == 1
def test_costs_funding_and_ledger_reconcile():
costs = CostModel("v9", 10, 20, 5, 5, 1, 0.5, 0.25)
result = run(
[bar(0, 100, 101, 99, 100, 1), bar(1, 100, 101, 99, 100, 2), bar(2, 100, 102, 99, 101)],
ScenarioSpec("costed", 100, 100, cost_model=costs),
)
row = result.ledger[0]
assert row.net_pnl == pytest.approx(
row.gross_pnl - row.entry_commission - row.exit_commission - row.funding - row.other_cost
)
assert row.entry_slippage > 0 and row.exit_slippage > 0
def test_deq_uses_forward_path_not_trade_pnl():
result = run(
[
bar(0, 100, 101, 99, 100, 1),
bar(1, 100, 101, 99, 100, 2),
bar(2, 100, 100, 99, 100),
bar(3, 100, 110, 99, 109),
]
)
deq = result.ledger[0].deq
assert deq.return_1_bars_bps == 900
assert deq.mfe_bps == 1000
assert deq.time_to_positive_bars == 1
assert deq.time_to_25_bps_bars == 1
assert deq.time_to_50_bps_bars == 1
assert deq.max_adverse_before_positive_bps == -100
def test_deq_is_direction_correct_and_rescue_is_ledger_derived():
result = run(
[
bar(0, 100, 101, 99, 100, 1),
bar(1, 100, 101, 99, 100, -2),
bar(2, 100, 100.5, 98, 99),
bar(3, 99, 99.5, 95, 96),
]
)
deq = result.ledger[0].deq
assert deq.return_1_bars_bps == 400
assert deq.mfe_bps == 500
assert deq.mae_bps == -50
assert result.rescue.trade_count == len(result.ledger)
assert result.rescue.win_count + result.rescue.loss_count == result.rescue.trade_count
assert result.rescue.net_pnl == result.net_pnl
assert result.rescue.commissions == result.commissions
def test_exact_named_scenario_catalog_and_round_trip_costs():
catalog = qualification_scenario_catalog()
assert set(catalog) == {
"BASELINE",
"SLIPPAGE_ADVERSE_2BP",
"TOTAL_COST_6BP",
"TOTAL_COST_10BP",
"TOTAL_COST_15BP",
"PESSIMISTIC_SAME_BAR",
"ENTRY_DELAY_PLUS_ONE_BAR",
"WEEKDAYS_ONLY",
"ALL_DAYS",
"STOP_WIDTH_100",
"STOP_WIDTH_75",
"STOP_WIDTH_50",
"STOP_WIDTH_25",
}
assert catalog["SLIPPAGE_ADVERSE_2BP"].cost_model.nominal_round_trip_bps == 2
assert catalog["TOTAL_COST_15BP"].cost_model.nominal_round_trip_bps == 15
assert catalog["ENTRY_DELAY_PLUS_ONE_BAR"].entry_delay_bars == 2
assert catalog["ALL_DAYS"].allow_weekend_entries is True
def test_batch_protocol_identity_is_an_optional_canary_input_and_persisted_snapshot():
scenario = ScenarioSpec("batch", 100, 100)
identity = {"specimen_report_sha256": "1148d891" + "0" * 56}
assert TradingStudioService.qualification_configuration(scenario) == {
"name": "batch",
"stop_loss_bps": 100,
"take_profit_bps": 100,
"entry_delay_bars": 1,
"cooldown_bars": 0,
"max_holding_bars": 1,
"allow_weekend_entries": False,
"same_bar_policy": "PESSIMISTIC_SL",
"cost_model": {
"version": "qualification-v1",
"entry_commission_bps": 0.0,
"exit_commission_bps": 0.0,
"entry_slippage_bps": 0.0,
"exit_slippage_bps": 0.0,
"funding_bps_per_bar": 0.0,
"entry_other_cost": 0.0,
"exit_other_cost": 0.0,
},
}
snapshot = TradingStudioService.qualification_configuration(
scenario, protocol_identity=identity, require_protocol_identity=True
)
assert snapshot["protocol_identity"] == identity
with pytest.raises(ValueError, match="requires a protocol identity"):
TradingStudioService.qualification_configuration(
scenario, require_protocol_identity=True
)
def test_raw_calibration_has_no_holdout_leak_ab_equivalence():
left = [bar(0, 10, 11, 9, 10, 1, (1,)), bar(1, 10, 11, 9, 10, 100, (100,))]
right = [bar(0, 10, 11, 9, 10, 1, (1,)), bar(1, 10, 11, 9, 10, -999, (-999,))]
assert calibrated(left).threshold == calibrated(right).threshold == 1
def test_cohort_raw_calibration_retains_zero_signal_train_provenance_without_fabrication():
bars = [
bar(0, 100, 101, 99, 100, None, (1.0,), "train-0"),
bar(1, 100, 101, 99, 100, None, (2.0,), "train-1"),
bar(2, 100, 101, 99, 100, None, (3.0,), "replay-0"),
]
calibration = calibrate_train_only(
bars,
method=CalibrationMethod.RAW,
fold="fold-a",
train_start=BASE,
train_end=BASE + timedelta(days=2),
source_record_ids=("frozen-member",),
source_combo="frozen-hs22-combo",
allow_empty_raw=True,
)
result = QualificationReplayV1().run(
bars[2:],
strategy="strategy-v1",
dataset="Cohort001",
fold="fold-a",
runner="cohort_qualification_hyperscalper_001:v1",
scenario=ScenarioSpec("baseline", 100, 100),
calibration=calibration,
)
assert calibration.threshold == 0.0
assert calibration.sample_count == 0
assert calibration.source_combo == "frozen-hs22-combo"
assert calibration.input_hash
assert result.ledger == ()
assert result.rescue.trade_count == 0
assert result.rescue.net_pnl == 0
def test_percentile_calibration_still_rejects_zero_qualifying_train_signals():
with pytest.raises(ValueError, match="Train-only calibration"):
calibrate_train_only(
[bar(0, 100, 101, 99, 100)],
method=CalibrationMethod.PERCENTILE,
fold="fold-a",
train_start=BASE,
train_end=BASE + timedelta(days=1),
source_record_ids=("frozen-member",),
source_combo="frozen-hs22-combo",
percentile=50,
allow_empty_raw=True,
)
def test_percentile_calibration_records_provenance_and_continuous_context():
bars = [bar(0, 10, 11, 9, 10, 1, (0.1, 0.2)), bar(0, 10, 11, 9, 10, 3, (0.3, 0.4))]
value = calibrate_train_only(
bars,
method=CalibrationMethod.PERCENTILE,
fold="fold-a",
train_start=BASE,
train_end=BASE + timedelta(days=1),
source_record_ids=("a",),
source_combo="ohlcv+continuous",
percentile=100,
)
assert value.threshold == 3 and value.input_hash and value.source_combo == "ohlcv+continuous"
def test_reference_mode_is_absent_and_scenario_names_are_required():
assert not hasattr(QualificationReplayV1, "reference_mode")
with pytest.raises(ValueError, match="named"):
ScenarioSpec("", 100, 100)
with pytest.raises(
ValueError,
match="missing materialized strategy_version, dataset_version, bars, calibration",
):
TradingStudioService().run_qualification_canary(
strategy_version=None,
dataset_version=None,
fold="fold-a",
scenario=ScenarioSpec("canary", 100, 100),
runner_name="runner",
bars=[],
calibration=None,
)
def test_materialized_canary_persists_without_creating_live_run():
project = Project.objects.create(name="Canary", goal="Test qualification persistence")
trading_project = TradingProject.objects.create(
project=project,
name="Canary Trading",
slug="canary-trading",
goal="Test qualification persistence",
)
dataset = MarketDataset.objects.create(
trading_project=trading_project,
name="Materialized BTCUSDT",
kind=DataKind.OHLCV,
)
dataset_version = MarketDatasetVersion.objects.create(
dataset=dataset,
version="frozen-v1",
reference="artifact://materialized.csv",
content_hash="a" * 64,
)
strategy = Strategy.objects.create(trading_project=trading_project, name="HS22")
strategy_version = StrategyVersion.objects.create(
strategy=strategy,
version="frozen-v1",
genome={"combo": [1]},
fingerprint="b" * 64,
)
bars = [
bar(0, 100, 101, 99, 100, 1),
bar(1, 100, 101, 99, 100, 2),
bar(2, 100, 102, 99, 101),
bar(3, 101, 103, 100, 102),
]
service = TradingStudioService()
run = service.run_qualification_canary(
strategy_version=strategy_version,
dataset_version=dataset_version,
fold="fold-a",
scenario=ScenarioSpec("baseline", 100, 100),
runner_name="test-runner",
bars=bars,
calibration=calibrated(bars),
)
assert run.replay_mode == "CANARY_ONLY"
assert run.ledger_rows.count() == 1
assert LiveStrategyRun.objects.count() == 0
zero_bars = [bar(0, 100, 101, 99, 100), bar(1, 100, 101, 99, 100)]
zero_run = service.run_qualification_canary(
strategy_version=strategy_version,
dataset_version=dataset_version,
fold="fold-zero",
scenario=ScenarioSpec("zero-trade", 100, 100),
runner_name="test-runner",
bars=zero_bars,
calibration=calibrate_train_only(
zero_bars,
method=CalibrationMethod.RAW,
fold="fold-zero",
train_start=BASE,
train_end=BASE + timedelta(days=1),
source_record_ids=("frozen-member",),
source_combo="frozen-hs22-combo",
allow_empty_raw=True,
),
)
assert zero_run.ledger_rows.count() == 0
assert zero_run.summary["ledger_count"] == 0
assert zero_run.summary["rescue"]["trade_count"] == 0
assert zero_run.summary["rescue"]["net_pnl"] == 0

232
tools/batch01_native_mismatch_report.py Normal file
View file

@ -0,0 +1,232 @@
"""Generate an exact diagnostic for frozen Batch01 native/oracle mismatches.
This is diagnostic-only: it evaluates the recovered native formulas without
altering their arithmetic or the frozen oracle artifacts.
"""
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Any
import numba as nb
import numpy as np
from control_plane.trading_studio.indicators.batch01_native_parity import (
BATCH01_OUTPUTS,
BATCH01_REQUEST,
_load_ohlcv,
_require_frozen_artifacts,
)
from control_plane.trading_studio.indicators.historical_band_channel import (
_sma,
evaluate_band_channel,
)
ROOT = Path(__file__).resolve().parents[1]
JSON_PATH = ROOT / "batch01_native_mismatch_report.json"
MARKDOWN_PATH = ROOT / "batch01_native_mismatch_root_cause.md"
_SIGN = 1 << 63
_MASK = (1 << 64) - 1
def _bits(value: float) -> int:
return int(np.asarray(value, dtype=np.float64).view(np.uint64))
def _ordered_bits(value: float) -> int:
bits = _bits(value)
return (~bits & _MASK) if bits & _SIGN else bits | _SIGN
def _float(value: float) -> dict[str, Any]:
bits = _bits(value)
return {"decimal": float(value), "hex": float(value).hex(), "bits": f"0x{bits:016x}"}
def _ulp(expected: float, actual: float) -> dict[str, int]:
signed = _ordered_bits(expected) - _ordered_bits(actual)
return {"signed": signed, "absolute": abs(signed)}
def _band_intermediate(close: np.ndarray, index: int, period: int) -> dict[str, Any]:
mean = _sma(close, period)[index]
variance_sum = sum(
(close[item] - mean) ** 2 for item in range(index - period + 1, index + 1)
)
variance = variance_sum / (period - 1)
return {
"window_start": index - period + 1,
"window_end": index,
"mean": _float(mean),
"variance_sum": _float(variance_sum),
"sample_variance": _float(variance),
"sample_stddev": _float(math.sqrt(variance)),
}
@nb.njit(cache=True)
def _recovered_band_intermediate(
close: np.ndarray, index: int, period: int
) -> tuple[float, float, float, float]:
"""Execute the recovered Numba arithmetic solely to expose its intermediates."""
mean_sum = 0.0
for item in range(period):
mean_sum += close[item]
for item in range(period, index + 1):
mean_sum += close[item] - close[item - period]
mean = mean_sum / period
variance_sum = 0.0
for item in range(index - period + 1, index + 1):
variance_sum += (close[item] - mean) ** 2
variance = variance_sum / (period - 1)
return mean, variance_sum, variance, math.sqrt(variance)
def _recovered_intermediate(close: np.ndarray, index: int, period: int) -> dict[str, Any]:
mean, variance_sum, variance, standard_deviation = _recovered_band_intermediate(close, index, period)
return {
"mean": _float(mean),
"variance_sum": _float(variance_sum),
"sample_variance": _float(variance),
"sample_stddev": _float(standard_deviation),
}
def _threshold(close: float, expected: float, actual: float) -> dict[str, Any]:
expected_above = close > expected
actual_above = close > actual
return {
"close": _float(close),
"oracle_close_minus_band": _float(close - expected),
"native_close_minus_band": _float(close - actual),
"oracle_close_above_band": expected_above,
"native_close_above_band": actual_above,
"decision_changed": expected_above != actual_above,
"close_to_oracle_band_ulps": _ulp(close, expected)["absolute"],
"close_to_native_band_ulps": _ulp(close, actual)["absolute"],
}
def _request_id(indicator_id: int, period: int, p1: float) -> str:
p1_text = str(int(p1)) if p1.is_integer() else str(p1)
return f"batch01_{indicator_id}_{period}_{p1_text}"
def build_report() -> dict[str, Any]:
request, _ = _require_frozen_artifacts()
close, high, low, volume = _load_ohlcv()
actual_by_id: dict[str, np.ndarray] = {}
request_by_id = {item["request_id"]: item for item in request["requests"]}
for item in request["requests"]:
actual_by_id[item["request_id"]] = evaluate_band_channel(
int(item["indicator_id"]), close, high, low, volume, int(item["period"]), float(item["p1"])
)
variants: list[dict[str, Any]] = []
with np.load(BATCH01_OUTPUTS) as oracle:
for request_id, item in request_by_id.items():
expected = oracle[request_id]
actual = actual_by_id[request_id]
mismatch_indexes = np.flatnonzero(~((expected == actual) | (np.isnan(expected) & np.isnan(actual))))
variant: dict[str, Any] = {
"request_id": request_id,
"indicator_id": item["indicator_id"],
"period": item["period"],
"p1": item["p1"],
"classification": "exact" if not len(mismatch_indexes) else "numeric_evaluation_order_drift",
"mismatch_count": int(len(mismatch_indexes)),
"mismatches": [],
}
for index in mismatch_indexes:
expected_value, actual_value = float(expected[index]), float(actual[index])
detail: dict[str, Any] = {
"index": int(index),
"oracle": _float(expected_value),
"native": _float(actual_value),
"absolute_error": _float(abs(expected_value - actual_value)),
"ulp": _ulp(expected_value, actual_value),
"threshold_sensitivity": _threshold(float(close[index]), expected_value, actual_value),
}
if int(item["indicator_id"]) in {17, 18}:
period, p1 = int(item["period"]), float(item["p1"])
upper_id, lower_id = _request_id(17, period, p1), _request_id(18, period, p1)
oracle_upper, oracle_lower = float(oracle[upper_id][index]), float(oracle[lower_id][index])
native_upper = float(actual_by_id[upper_id][index])
native_lower = float(actual_by_id[lower_id][index])
oracle_center = (oracle_upper + oracle_lower) / 2.0
native_center = (native_upper + native_lower) / 2.0
oracle_half_width = (oracle_upper - oracle_lower) / 2.0
native_half_width = (native_upper - native_lower) / 2.0
detail["native_formula_intermediate"] = _band_intermediate(close, int(index), period)
detail["recovered_numba_intermediate"] = _recovered_intermediate(
close, int(index), period
)
detail["upper_lower_symmetry"] = {
"oracle_center": _float(oracle_center),
"native_center": _float(native_center),
"center_error": _float(oracle_center - native_center),
"oracle_half_width": _float(oracle_half_width),
"native_half_width": _float(native_half_width),
"half_width_error": _float(oracle_half_width - native_half_width),
"upper_error": _float(oracle_upper - native_upper),
"lower_error": _float(oracle_lower - native_lower),
}
variant["mismatches"].append(detail)
variants.append(variant)
failures = [variant for variant in variants if variant["mismatch_count"]]
return {
"artifact": "BATCH01_NATIVE_MISMATCH_REPORT_V1",
"scope": "frozen batch01 oracle NPZ/request/CSV against native recovered band code",
"formula_mutations": False,
"variant_count": len(variants),
"failed_variant_count": len(failures),
"failed_value_count": sum(item["mismatch_count"] for item in failures),
"classification": "numeric_evaluation_order_drift_only" if failures else "exact_parity",
"variants": variants,
}
def _markdown(report: dict[str, Any]) -> str:
failures = [item for item in report["variants"] if item["mismatch_count"]]
lines = [
"# Batch01 Native Mismatch Root Cause",
"",
"## Classification",
"",
f"`{report['classification']}`: {report['failed_value_count']} bitwise mismatches in "
f"{report['failed_variant_count']} of {report['variant_count']} requests.",
"",
"## Evidence",
"",
"- All failures are Bollinger pairs: 17 (upper) and 18 (lower).",
"- Every failure has the same index and parameter pair in the opposite band.",
"- The JSON report records the oracle/native IEEE-754 bit patterns, signed ULP delta, native rolling mean/sample standard deviation, and paired-band center/half-width decomposition.",
"- Threshold comparisons (`close > band`) are unchanged at every failed value.",
"",
"## Root Cause",
"",
"The recovered Numba source and native port have bit-identical rolling means at every failed bar, but the Python generator reduction and the recovered scalar Numba variance loop differ by one or two low-order bits. Their compiled/CPython `sqrt` results can add a further low-order difference. The resulting standard-deviation rounding propagates as an equal-and-opposite upper/lower half-width shift. The evidence rules out a formula, parameter, warmup, or band-sign error.",
"",
"## Failed Requests",
"",
]
for item in failures:
indexes = ", ".join(str(detail["index"]) for detail in item["mismatches"])
ulps = ", ".join(str(detail["ulp"]["signed"]) for detail in item["mismatches"])
lines.append(f"- `{item['request_id']}`: indexes {indexes}; signed ULPs {ulps}.")
lines.append("")
return "\n".join(lines)
def main() -> None:
report = build_report()
JSON_PATH.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8")
MARKDOWN_PATH.write_text(_markdown(report), encoding="utf-8")
if __name__ == "__main__":
main()

1390
uv.lock generated Normal file
View file

File diff suppressed because it is too large Load diff