from __future__ import annotations

import json
import sys
import time

import numpy as np
import pytest

import hyperscalper_feature_gap_analysis_v1 as runner

pytestmark = pytest.mark.django_db


def test_read_only_runner_writes_711_map_and_honest_deq_blocker(tmp_path, monkeypatch):
    rows = [
        {"request_id": f"r{index}", "indicator_id": index, "period": 10, "p1": 0.0}
        for index in range(711)
    ]
    request = tmp_path / "request.json"
    request.write_text(json.dumps({"requests": rows}), encoding="utf-8")
    checkpoint = tmp_path / "checkpoint.npz"
    values = {f"r{index}": np.arange(8, dtype=float) + index for index in range(711)}
    values["r710"] = np.full(8, np.nan)
    np.savez(checkpoint, **values)
    manifest = tmp_path / "manifest.json"
    manifest.write_text(
        json.dumps({"coverage": {"feature_versions": [f"r{index}" for index in range(711)]}}),
        encoding="utf-8",
    )
    for name in ("usage.parquet", "lineage.parquet"):
        (tmp_path / name).write_bytes(b"not-read-without-pyarrow")
    output = tmp_path / "out"
    monkeypatch.setattr(
        sys,
        "argv",
        [
            "runner",
            "--oracle-request",
            str(request),
            "--oracle-checkpoint",
            str(checkpoint),
            "--acceptance-manifest",
            str(manifest),
            "--historical-usage",
            str(tmp_path / "usage.parquet"),
            "--lineage",
            str(tmp_path / "lineage.parquet"),
            "--output-dir",
            str(output),
            "--sample-rows",
            "8",
        ],
    )
    runner.main()
    result = json.loads((output / "hyperscalper_feature_gap_analysis_v1.json").read_text())
    assert result["read_only"] is True
    assert len(result["information_map"]["features"]) == 711
    assert result["information_map"]["features"][-1]["redundancy_status"] == (
        "unusable_no_finite_observations"
    )
    assert result["redundancy"]["map"]["format"] == "parquet"
    assert result["decision_equivalence"]["status"] == "BLOCKED"


def test_checkpoint_directory_maps_hs22_filenames_to_engineering_primitives(tmp_path):
    rows = [
        {"indicator_id": index, "period": index + 10, "p1": float(index % 3)}
        for index in range(711)
    ]
    checkpoint_dir = tmp_path / "checkpoints"
    checkpoint_dir.mkdir()
    for row in reversed(rows):
        np.save(
            checkpoint_dir
            / f"hs22_{row['indicator_id']}_{row['period']}_{row['p1']:g}.npy",
            np.array([row["indicator_id"], row["period"]], dtype=float),
            allow_pickle=False,
        )

    matrix, names = runner.checkpoint_directory_columns(
        checkpoint_dir,
        [
            {
                **row,
                "feature_key": f"{row['indicator_id']}:{row['period']}:{row['p1']:g}",
                "request_id": None,
            }
            for row in rows
        ],
    )

    assert matrix.shape == (2, 711)
    assert np.array_equal(matrix[:, 0], [0.0, 10.0])
    assert np.array_equal(matrix[:, -1], [710.0, 720.0])
    assert names[0] == "hs22_0_10_0.npy"


def test_pairwise_redundancy_uses_staggered_finite_observations_for_clustering():
    sample = np.array(
        [
            [0.0, 0.0, np.nan, np.nan],
            [1.0, 2.0, np.nan, np.nan],
            [2.0, 4.0, np.nan, np.nan],
            [np.nan, np.nan, 5.0, np.nan],
            [np.nan, np.nan, 6.0, np.nan],
        ]
    )

    pearson, spearman, counts = runner.pairwise_redundancy(sample, min_pair_samples=3)

    assert counts[0, 1] == 3
    assert pearson[0, 1] == pytest.approx(1.0)
    assert spearman[0, 1] == pytest.approx(1.0)
    assert np.isnan(pearson[0, 2])
    assert runner.clusters(pearson) == [[0, 1]]


def test_pairwise_redundancy_precomputes_ranks_once_per_column_with_bounded_runtime(monkeypatch):
    sample = np.arange(512 * 64, dtype=float).reshape(512, 64)
    sample[::17, ::7] = np.nan
    calls = 0
    original = runner.rank_finite_columns

    def counted_ranks(values):
        nonlocal calls
        calls += 1
        return original(values)

    monkeypatch.setattr(runner, "rank_finite_columns", counted_ranks)
    started = time.perf_counter()
    pearson, spearman, counts = runner.pairwise_redundancy(sample, min_pair_samples=2)

    assert time.perf_counter() - started < 3.0
    assert calls == 1
    assert counts[0, 1] > 2
    assert pearson[0, 1] == pytest.approx(1.0)
    assert spearman[0, 1] == pytest.approx(1.0, abs=1e-5)


def test_type_aware_redundancy_uses_agreement_and_jaccard_for_detection_outputs():
    sample = np.array(
        [
            [0.0, 1.0, 1.0, 0.0],
            [1.0, 1.0, 1.0, 1.0],
            [1.0, 0.0, 0.0, 1.0],
            [0.0, 0.0, 0.0, 0.0],
        ]
    )

    pearson, spearman, agreement, jaccard, counts = runner.type_aware_redundancy(
        sample, ["state", "state", "event", "event"], min_pair_samples=2
    )
    primary = runner.primary_redundancy_matrix(
        pearson, agreement, jaccard, ["state", "state", "event", "event"]
    )

    assert counts[0, 1] == 4
    assert agreement[0, 1] == pytest.approx(0.5)
    assert jaccard[2, 3] == pytest.approx(1 / 3)
    assert np.isnan(pearson[0, 1])
    assert primary[0, 1] == pytest.approx(0.5)
    assert primary[2, 3] == pytest.approx(1 / 3)


def test_semantic_types_derives_type_and_domain_from_primitive_map(tmp_path):
    semantic_map = tmp_path / "semantic-map.json"
    semantic_map.write_text(
        json.dumps(
            {
                "primitives": [
                    {"indicator_id": 1, "period": 10, "p1": 0, "output_type": "boolean", "engineering_family": "regime"},
                    {"indicator_id": 2, "period": 20, "p1": 1, "output_type": "event", "domain": "trigger"},
                ]
            }
        ),
        encoding="utf-8",
    )
    rows = [
        {"feature_key": "1:10:0", "indicator_id": 1, "period": 10, "p1": 0.0},
        {"feature_key": "2:20:1", "indicator_id": 2, "period": 20, "p1": 1.0},
    ]

    types, blocker = runner.semantic_types(semantic_map, rows)

    assert blocker is None
    assert types == [
        {"output_type": "state", "domain": "regime"},
        {"output_type": "event", "domain": "trigger"},
    ]


def test_lineage_combo_rejects_incomplete_data():
    with pytest.raises(ValueError, match="at least five triples"):
        runner.lineage_combo([1, 10, 0.0])
    with pytest.raises(ValueError, match="at least five triples"):
        runner.lineage_combo([[1, 10, 0.0]] * 5)


def test_lineage_combo_reads_only_the_five_primitives_from_real_flat_encoding():
    # Flat lineage rows append tp, sl, confirmation bounds, and volume threshold.
    combo = [
        17, 20, 2.0,
        31, 14, 0.0,
        47, 9, 1.5,
        62, 30, 0.0,
        70, 20, 0.0,
        0.015, 0.01, 5, 20, 5, 20, 75,
    ]

    assert runner.lineage_combo(json.dumps(combo)) == (
        (17, 20, 2.0),
        (31, 14, 0.0),
        (47, 9, 1.5),
        (62, 30, 0.0),
        (70, 20, 0.0),
    )


def test_lineage_usage_parses_combo_json_role_triples(tmp_path):
    pa = pytest.importorskip("pyarrow")
    pq = pytest.importorskip("pyarrow.parquet")
    lineage = tmp_path / "lineage.parquet"
    combo = [1, 10, 0.0, 2, 11, 1.5, 1, 10, 0.0, 4, 13, 2.0, 5, 14, 3.0]
    pq.write_table(
        pa.Table.from_pylist(
            [
                {"strategy_id": 100, "combo_json": json.dumps(combo)},
                {"strategy_id": 101, "combo_json": json.dumps(combo + [0.01, 0.02, 3, 6, 3, 6, 75])},
            ]
        ),
        lineage,
    )

    counts, evidence, blocker = runner.lineage_usage(lineage)

    assert blocker is None
    assert counts == {"1:10:0": 4, "2:11:1.5": 2, "4:13:2": 2, "5:14:3": 2}
    assert evidence["role_counts"] == {
        "confirm": 2,
        "signal": 2,
        "trend": 2,
        "trigger": 2,
        "vol": 2,
    }
    assert evidence["strategy_ids_observed"] == 2
    assert evidence["combo_counts"][0]["usage_count"] == 2


def test_cohort_deq_evidence_aggregates_exported_ledger_associatively(tmp_path):
    deq_path = tmp_path / "cohort-deq.json"
    sample = {
        "deq": {
            "return_1_bars_bps": 10.0,
            "return_2_bars_bps": -5.0,
            "return_3_bars_bps": None,
            "return_5_bars_bps": 0.0,
            "return_10_bars_bps": 20.0,
            "mfe_bps": 25.0,
            "mae_bps": -4.0,
            "time_to_positive_bars": 2,
        }
    }
    deq_path.write_text(
        json.dumps(
            {
                "contract": "Cohort001-DEQ-ledger-export-v1",
                "strategies": [
                    {
                        "strategy_version_id": "strategy-a",
                        "feature_triples": [
                            {"indicator_id": 7, "period": 10, "p1": 1.0},
                            {"indicator_id": 7, "period": 20, "p1": 2.0},
                        ],
                        "folds": [
                            {
                                "fold": "Fold 1",
                                "scenarios": [
                                    {"scenario": "BASE", "deq_samples": [sample]},
                                    {"scenario": "STRESS", "deq_samples": [sample]},
                                ],
                            }
                        ],
                    }
                ],
            }
        ),
        encoding="utf-8",
    )

    evidence, parquet_rows, blocker = runner.cohort_deq_evidence(deq_path)

    assert blocker is None
    assert evidence["status"] == "AVAILABLE"
    assert evidence["attribution"] == "ASSOCIATIVE_NOT_CAUSAL"
    assert evidence["strategy_summaries"][0]["trade_count"] == 2
    assert evidence["strategy_summaries"][0]["scenario_coverage"] == ["BASE", "STRESS"]
    returns = evidence["feature_summaries"][0]["returns"]
    assert returns["return_1_bars_bps"]["mean"] == 10.0
    assert returns["return_2_bars_bps"]["directional_incorrect_count"] == 2
    assert evidence["family_summaries"][0]["trade_count"] == 4
    assert {row["entity_type"] for row in parquet_rows} == {"strategy", "feature", "family"}


def test_cohort_deq_evidence_uses_genome_combo_when_exported_triples_are_null(tmp_path):
    deq_path = tmp_path / "cohort-deq.json"
    deq_path.write_text(
        json.dumps(
            {
                "contract": "Cohort001-DEQ-ledger-export-v1",
                "strategies": [
                    {
                        "strategy_version_id": "strategy-a",
                        "feature_triples": None,
                        "genome": {
                            "combo": [
                                7, 10, 1.0,
                                8, 11, 2.0,
                                9, 12, 3.0,
                                10, 13, 4.0,
                                11, 14, 5.0,
                                0.01, 0.02, 3, 6, 3, 6, 75,
                            ]
                        },
                        "folds": [
                            {
                                "fold": "Fold 1",
                                "scenarios": [
                                    {
                                        "scenario": "BASE",
                                        "deq_samples": [{"deq": {"return_1_bars_bps": 10.0}}],
                                    }
                                ],
                            }
                        ],
                    }
                ],
            }
        ),
        encoding="utf-8",
    )

    evidence, _, blocker = runner.cohort_deq_evidence(deq_path)

    assert blocker is None
    assert [item["feature_key"] for item in evidence["feature_summaries"]] == [
        "10:13:4",
        "11:14:5",
        "7:10:1",
        "8:11:2",
        "9:12:3",
    ]


def test_deq_strategy_feature_keys_falls_back_when_exported_triples_are_invalid():
    strategy = {
        "feature_triples": [{"indicator_id": "not-an-id"}],
        "genome": {"combo": [7, 10, 1.0, 8, 11, 2.0, 9, 12, 3.0, 10, 13, 4.0, 11, 14, 5.0]},
    }

    assert runner.deq_strategy_feature_keys(strategy) == [
        ("7:10:1", "7"),
        ("8:11:2", "8"),
        ("9:12:3", "9"),
        ("10:13:4", "10"),
        ("11:14:5", "11"),
    ]