Artifex/tests/test_hyperscalper_cohort_materialization.py

from datetime import UTC, datetime, timedelta
from hashlib import sha256
from inspect import getsource

import numpy as np
import pytest

from control_plane.trading_studio import cohort_materialization as materialization
from control_plane.trading_studio.management.commands.canary_hyperscalper_cohort_001 import (
    Command as CanaryCommand,
)
from control_plane.trading_studio.management.commands.materialize_hyperscalper_cohort_001 import (
    DATASET_SHA256,
    DATASET_VERSION_ID,
    Command,
)


def row(index):
    return {
        "timestamp": datetime(2025, 1, 1, tzinfo=UTC) + timedelta(minutes=index),
        "open": 1.0,
        "high": 1.1,
        "low": 0.9,
        "close": 1.0,
    }


def test_canonical_manifest_serialization_is_sorted_pretty_ascii_without_newline():
    manifest = {"z": "cafe", "a": [1, {"b": True}]}
    serialized = materialization.canonical_json(manifest)
    assert (
        serialized == '{\n  "a": [\n    1,\n    {\n      "b": true\n    }\n  ],\n  "z": "cafe"\n}'
    )
    assert sha256(serialized.encode()).hexdigest() == (
        "c1f9b496e9724f3cf4b0ab781589f6d8db6e48f5de90f9ea9c39defb20748dc9"
    )


def test_manifest_requires_canonical_self_semantic_and_lineage_hashes(monkeypatch):
    monkeypatch.setattr(materialization, "MANIFEST_HASH_PREFIX", "")
    records = [
        {
            "rank": 1,
            "source_strategy_id": 490761,
            "source_name": "B7435 TP1.8/SL0.8",
            "base_family": "B7435",
            "combo": [79, 30, 8.0],
            "old_selection_metrics": {"id": 490761},
            "old_final_metrics": {"id": 490761},
        }
    ]
    monkeypatch.setattr(
        materialization, "SEMANTIC_DIGEST", materialization.semantic_digest(records)
    )
    manifest = {"members": records, "lineage": {"source": "frozen"}}
    manifest["lineage_sha256"] = materialization.digest(manifest["lineage"])
    manifest["manifest_sha256"] = materialization.digest(manifest)
    materialization.validate_manifest(manifest, cohort=True)
    manifest["lineage"]["source"] = "changed"
    manifest["manifest_sha256"] = materialization.digest(
        {key: value for key, value in manifest.items() if key != "manifest_sha256"}
    )
    with pytest.raises(ValueError, match="lineage"):
        materialization.validate_manifest(manifest, cohort=True)


def test_semantic_digest_uses_compact_sorted_projection_and_required_full_digest():
    assert materialization.SEMANTIC_DIGEST == (
        "cc697ea359f20ccc83c5d62f5b406a3532c1d7411191423059164aafe5931254"
    )
    records = [
        {
            "rank": 2,
            "source_strategy_id": 2,
            "source_name": "B2",
            "base_family": "B2",
            "combo": [2],
            "old_selection_metrics": {"id": 2},
            "old_final_metrics": {"id": 2},
        },
        {
            "rank": 1,
            "source_strategy_id": 1,
            "source_name": "B1",
            "base_family": "B1",
            "combo": [1],
            "old_selection_metrics": {"id": 1},
            "old_final_metrics": {"id": 1},
        },
    ]
    with pytest.raises(ValueError, match="uniquely ordered"):
        materialization.semantic_digest(records)


def test_reconstructed_folds_have_exact_train_embargo_and_test_windows():
    rows = [row(index) for index in range(materialization.DATASET_RECORD_COUNT)]
    folds = materialization.reconstruct_folds(rows, "timestamp")
    assert len(folds) == 4
    for fold in folds:
        assert fold["train"]["end_index"] - fold["train"]["start_index"] + 1 == 14_400
        assert fold["embargo"]["bars"] == 35
        assert fold["embargo"]["status"] == "EXCLUDED"
        assert fold["embargo"]["start_index"] == fold["train"]["end_index"] + 1
        assert fold["test"]["start_index"] == fold["embargo"]["end_index"] + 1
        assert fold["test"]["end_index"] - fold["test"]["start_index"] + 1 == 7_200
        assert fold["provenance"] == "RECONSTRUCTED_FROM_FROZEN_SPEC_V1"


def test_csv_validation_rejects_noncontinuous_timestamps(monkeypatch, tmp_path):
    monkeypatch.setattr(materialization, "DATASET_RECORD_COUNT", 3)
    path = tmp_path / "dataset.csv"
    path.write_text(
        "timestamp,open,high,low,close\n"
        "1735689600,1,1.1,0.9,1\n"
        "1735689720,1,1.1,0.9,1\n"
        "1735690000,1,1.1,0.9,1\n",
        encoding="utf-8",
    )
    rows, headers = materialization.load_csv(path, "timestamp")
    manifest = {"columns": headers, "timestamp_field": "timestamp"}
    with pytest.raises(ValueError, match="continuity"):
        materialization.validate_rows(rows, headers, manifest)


def test_dataset_manifest_treats_sha256_as_the_csv_artifact_hash_not_a_self_hash():
    manifest = {
        "artifact": "artifact://binance_btcusdt_spot_2m_180d.csv",
        "sha256": "a" * 64,
        "row_count": materialization.DATASET_RECORD_COUNT,
        "fields": ["timestamp", "open", "high", "low", "close", "volume"],
        "monotonic_timestamps": True,
        "schema_valid": True,
    }
    materialization.validate_dataset_manifest(manifest)
    manifest["schema_valid"] = False
    with pytest.raises(ValueError, match="declarations"):
        materialization.validate_dataset_manifest(manifest)


def test_existing_dataset_recovery_uses_crypto_hyperscalper_and_real_artifact_path():
    source = getsource(Command._dataset)
    assert 'slug != "crypto-hyperscalper"' in source
    assert "(root / DATASET_CSV).resolve()" in source
    assert '"first_timestamp"' in source
    assert '"last_timestamp"' in source


def test_actual_shape_member_maps_to_stable_semantic_ids_and_authorized_dataset():
    member = {
        "rank": 1,
        "source_strategy_id": 490761,
        "source_name": "B7435 TP1.8/SL0.8",
        "base_family": "B7435",
        "combo": [79, 30, 8.0],
        "old_selection_metrics": {"id": 490761},
        "old_final_metrics": {"id": 490761},
    }
    assert Command._fingerprint(member) == Command._fingerprint(dict(member))
    assert Command._member_id(member) == Command._member_id(dict(member))
    assert str(DATASET_VERSION_ID) == "1f224db9-cbb8-4aec-8575-98c4b0279a83"
    assert DATASET_SHA256 == "7fb56bb05b0edc2348407765cd693efadf334307c25877dfb52acfe4221e5d00"


def test_prior_cohort_policy_preserves_historic_manifest_when_reconstruction_is_added():
    policy = {"manifest_sha256": "4d20historic", "prior_note": "preserve"}
    assert Command._historic_manifest_sha(policy) == "4d20historic"
    policy["reconstruction_v1"] = {"folds": ["reconstructed"]}
    assert policy["prior_note"] == "preserve"
    assert Command._historic_manifest_sha(policy) == "4d20historic"


def test_canary_reads_reconstruction_v1_instead_of_obsolete_frozen_manifest_policy():
    source = getsource(CanaryCommand._validate_reconstruction)
    assert 'get("reconstruction_v1")' in source
    assert "DATASET_VERSION_ID" in source
    assert "reconstruct_folds" in source
    assert "frozen_manifest" not in source


def test_canary_uses_native_state_and_keeps_continuous_context():
    rows = [
        {**row(index), "volume": 100.0}
        for index in range(3)
    ]
    state = {
        "trend": np.array([1.0, 2.0, 3.0]),
        "signal": np.array([4.0, 5.0, 6.0]),
        "trigger": np.array([0.5, 0.5, 0.5]),
        "confirm": np.array([7.0, 8.0, 9.0]),
        "vol": np.array([10.0, 11.0, 12.0]),
        "trend_color": np.array([0, 1, 1]),
        "signal_color": np.array([0, 1, 1]),
        "decision": np.array([0, 1, -1]),
    }
    bars = CanaryCommand._bars(rows, state, {"start_index": 1, "end_index": 2})
    source = getsource(CanaryCommand)

    assert [bar.signal for bar in bars] == [1, -1]
    assert bars[0].context == (2.0, 5.0, 0.5, 8.0, 11.0, 1.0, 1.0)
    assert bars[0].reference == "hs22-native:1"
    assert "compute_hs22_state" in source
    assert "compute_combo_state" not in source
    assert "precomputed" not in source


def test_canary_derives_signed_decisions_from_native_hs22_state(monkeypatch):
    state = {
        "trigger": np.array([1.0, 1.0, 1.0]),
        "confirm": np.array([0.0, 0.0, 0.0]),
        "vol": np.array([1.0, 1.0, 1.0]),
        "signal_color": np.array([0, 1, -1]),
        "trend_color": np.array([0, 1, -1]),
    }
    monkeypatch.setattr(
        (
            "control_plane.trading_studio.management.commands."
            "canary_hyperscalper_cohort_001.compute_hs22_state"
        ),
        lambda *args: state,
    )
    rows = [
        {"close": 1.0, "high": 1.0, "low": 1.0, "volume": 1.0},
        {"close": 2.0, "high": 2.0, "low": 2.0, "volume": 1.0},
        {"close": 0.5, "high": 0.5, "low": 0.5, "volume": 1.0},
    ]
    combo = [0.0] * 22
    combo[17:21] = [-1.0, 1.0, -1.0, 1.0]

    actual = CanaryCommand._native_state(rows, {"combo": combo})

    assert actual["decision"].tolist() == [0, 1, -1]


def test_selection_source_must_match_declared_path_and_all_frozen_member_records(tmp_path):
    source = tmp_path / "selection.json"
    selection = {
        "selected": [
            {
                "selection": {"id": 1, "name": "B1"},
                "final": {"id": 1, "name": "B1"},
            }
        ]
        * 20
    }
    source.write_text(__import__("json").dumps(selection), encoding="utf-8")
    members = [
        {
            "source_strategy_id": 1,
            "source_name": "B1",
            "old_selection_metrics": {"id": 1, "name": "B1"},
            "old_final_metrics": {"id": 1, "name": "B1"},
        }
        for _ in range(20)
    ]
    manifest = {"source_report": str(source), "members": members}
    assert materialization.validate_selection_source(manifest, selection, source)
    selection["selected"][0]["selection"]["id"] = 2
    with pytest.raises(ValueError, match="does not reproduce"):
        materialization.validate_selection_source(manifest, selection, source)