381 lines
12 KiB
Python
381 lines
12 KiB
Python
|
from datetime import UTC, datetime, timedelta
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from control_plane.projects.models import Project
|
||
|
|
from control_plane.trading_studio.models import (
|
||
|
|
DataKind,
|
||
|
|
LiveStrategyRun,
|
||
|
|
MarketDataset,
|
||
|
|
MarketDatasetVersion,
|
||
|
|
Strategy,
|
||
|
|
StrategyVersion,
|
||
|
|
TradingProject,
|
||
|
|
)
|
||
|
|
from control_plane.trading_studio.qualification import (
|
||
|
|
Bar,
|
||
|
|
CalibrationMethod,
|
||
|
|
CostModel,
|
||
|
|
QualificationReplayV1,
|
||
|
|
ScenarioSpec,
|
||
|
|
calibrate_train_only,
|
||
|
|
qualification_scenario_catalog,
|
||
|
|
)
|
||
|
|
from control_plane.trading_studio.services import TradingStudioService
|
||
|
|
|
||
|
|
BASE = datetime(2026, 1, 5, tzinfo=UTC)
|
||
|
|
|
||
|
|
|
||
|
|
def bar(day, open_, high, low, close, signal=None, context=(), reference=""):
|
||
|
|
return Bar(BASE + timedelta(days=day), open_, high, low, close, signal, context, reference)
|
||
|
|
|
||
|
|
|
||
|
|
def calibrated(bars, *, method=CalibrationMethod.RAW):
|
||
|
|
return calibrate_train_only(
|
||
|
|
bars,
|
||
|
|
method=method,
|
||
|
|
fold="fold-a",
|
||
|
|
train_start=BASE,
|
||
|
|
train_end=BASE + timedelta(days=1),
|
||
|
|
source_record_ids=("source-1", "source-2"),
|
||
|
|
source_combo="ohlcv+context",
|
||
|
|
percentile=50,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def run(bars, scenario=None):
|
||
|
|
return QualificationReplayV1().run(
|
||
|
|
bars,
|
||
|
|
strategy="strategy-v1",
|
||
|
|
dataset="Cohort001",
|
||
|
|
fold="fold-a",
|
||
|
|
runner="test-runner",
|
||
|
|
scenario=scenario or ScenarioSpec("base", 100, 100),
|
||
|
|
calibration=calibrated(bars),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_multibar_state_machine_emits_signal_entry_exit_and_ledger_directly():
|
||
|
|
result = run(
|
||
|
|
[
|
||
|
|
bar(0, 100, 101, 99, 100, 1),
|
||
|
|
bar(1, 100, 101, 99, 100, 2),
|
||
|
|
bar(2, 100, 102, 99, 101),
|
||
|
|
bar(3, 101, 103, 100, 102),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
assert [event.event_type for event in result.events] == [
|
||
|
|
"SIGNAL",
|
||
|
|
"ENTRY_QUEUED",
|
||
|
|
"ENTRY",
|
||
|
|
"EXIT",
|
||
|
|
]
|
||
|
|
assert result.ledger[0].holding_bars == 1
|
||
|
|
assert result.ledger[0].entry_bar == 2
|
||
|
|
|
||
|
|
|
||
|
|
def test_same_bar_tp_sl_ambiguity_is_pessimistic_stop():
|
||
|
|
result = run(
|
||
|
|
[bar(0, 100, 101, 99, 100, 1), bar(1, 100, 101, 99, 100, 2), bar(2, 100, 102, 98, 101)]
|
||
|
|
)
|
||
|
|
assert result.ledger[0].exit_reason == "SL"
|
||
|
|
assert result.ledger[0].exit_execution_price == 99
|
||
|
|
|
||
|
|
|
||
|
|
def test_delay_creates_natural_later_entry():
|
||
|
|
result = run(
|
||
|
|
[
|
||
|
|
bar(0, 100, 101, 99, 100, 1),
|
||
|
|
bar(1, 100, 101, 99, 100, 2),
|
||
|
|
bar(2, 100, 100, 99, 100),
|
||
|
|
bar(3, 100, 102, 99, 101),
|
||
|
|
],
|
||
|
|
ScenarioSpec("delay", 100, 100, entry_delay_bars=2),
|
||
|
|
)
|
||
|
|
assert result.ledger[0].entry_bar == 3
|
||
|
|
|
||
|
|
|
||
|
|
def test_stop_width_changes_later_eligibility():
|
||
|
|
bars = [
|
||
|
|
bar(0, 100, 101, 99, 100, 1),
|
||
|
|
bar(1, 100, 101, 99, 100, 2),
|
||
|
|
bar(2, 100, 100.5, 98.5, 99),
|
||
|
|
bar(3, 99, 101, 98, 100, 2),
|
||
|
|
bar(4, 100, 102, 99, 101),
|
||
|
|
]
|
||
|
|
narrow = run(bars, ScenarioSpec("narrow", 100, 100, cooldown_bars=1))
|
||
|
|
wide = run(bars, ScenarioSpec("wide", 300, 100, cooldown_bars=1, max_holding_bars=3))
|
||
|
|
assert len(narrow.ledger) == 2
|
||
|
|
assert len(wide.ledger) == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_costs_funding_and_ledger_reconcile():
|
||
|
|
costs = CostModel("v9", 10, 20, 5, 5, 1, 0.5, 0.25)
|
||
|
|
result = run(
|
||
|
|
[bar(0, 100, 101, 99, 100, 1), bar(1, 100, 101, 99, 100, 2), bar(2, 100, 102, 99, 101)],
|
||
|
|
ScenarioSpec("costed", 100, 100, cost_model=costs),
|
||
|
|
)
|
||
|
|
row = result.ledger[0]
|
||
|
|
assert row.net_pnl == pytest.approx(
|
||
|
|
row.gross_pnl - row.entry_commission - row.exit_commission - row.funding - row.other_cost
|
||
|
|
)
|
||
|
|
assert row.entry_slippage > 0 and row.exit_slippage > 0
|
||
|
|
|
||
|
|
|
||
|
|
def test_deq_uses_forward_path_not_trade_pnl():
|
||
|
|
result = run(
|
||
|
|
[
|
||
|
|
bar(0, 100, 101, 99, 100, 1),
|
||
|
|
bar(1, 100, 101, 99, 100, 2),
|
||
|
|
bar(2, 100, 100, 99, 100),
|
||
|
|
bar(3, 100, 110, 99, 109),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
deq = result.ledger[0].deq
|
||
|
|
assert deq.return_1_bars_bps == 900
|
||
|
|
assert deq.mfe_bps == 1000
|
||
|
|
assert deq.time_to_positive_bars == 1
|
||
|
|
assert deq.time_to_25_bps_bars == 1
|
||
|
|
assert deq.time_to_50_bps_bars == 1
|
||
|
|
assert deq.max_adverse_before_positive_bps == -100
|
||
|
|
|
||
|
|
|
||
|
|
def test_deq_is_direction_correct_and_rescue_is_ledger_derived():
|
||
|
|
result = run(
|
||
|
|
[
|
||
|
|
bar(0, 100, 101, 99, 100, 1),
|
||
|
|
bar(1, 100, 101, 99, 100, -2),
|
||
|
|
bar(2, 100, 100.5, 98, 99),
|
||
|
|
bar(3, 99, 99.5, 95, 96),
|
||
|
|
]
|
||
|
|
)
|
||
|
|
deq = result.ledger[0].deq
|
||
|
|
assert deq.return_1_bars_bps == 400
|
||
|
|
assert deq.mfe_bps == 500
|
||
|
|
assert deq.mae_bps == -50
|
||
|
|
assert result.rescue.trade_count == len(result.ledger)
|
||
|
|
assert result.rescue.win_count + result.rescue.loss_count == result.rescue.trade_count
|
||
|
|
assert result.rescue.net_pnl == result.net_pnl
|
||
|
|
assert result.rescue.commissions == result.commissions
|
||
|
|
|
||
|
|
|
||
|
|
def test_exact_named_scenario_catalog_and_round_trip_costs():
|
||
|
|
catalog = qualification_scenario_catalog()
|
||
|
|
assert set(catalog) == {
|
||
|
|
"BASELINE",
|
||
|
|
"SLIPPAGE_ADVERSE_2BP",
|
||
|
|
"TOTAL_COST_6BP",
|
||
|
|
"TOTAL_COST_10BP",
|
||
|
|
"TOTAL_COST_15BP",
|
||
|
|
"PESSIMISTIC_SAME_BAR",
|
||
|
|
"ENTRY_DELAY_PLUS_ONE_BAR",
|
||
|
|
"WEEKDAYS_ONLY",
|
||
|
|
"ALL_DAYS",
|
||
|
|
"STOP_WIDTH_100",
|
||
|
|
"STOP_WIDTH_75",
|
||
|
|
"STOP_WIDTH_50",
|
||
|
|
"STOP_WIDTH_25",
|
||
|
|
}
|
||
|
|
assert catalog["SLIPPAGE_ADVERSE_2BP"].cost_model.nominal_round_trip_bps == 2
|
||
|
|
assert catalog["TOTAL_COST_15BP"].cost_model.nominal_round_trip_bps == 15
|
||
|
|
assert catalog["ENTRY_DELAY_PLUS_ONE_BAR"].entry_delay_bars == 2
|
||
|
|
assert catalog["ALL_DAYS"].allow_weekend_entries is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_batch_protocol_identity_is_an_optional_canary_input_and_persisted_snapshot():
|
||
|
|
scenario = ScenarioSpec("batch", 100, 100)
|
||
|
|
identity = {"specimen_report_sha256": "1148d891" + "0" * 56}
|
||
|
|
|
||
|
|
assert TradingStudioService.qualification_configuration(scenario) == {
|
||
|
|
"name": "batch",
|
||
|
|
"stop_loss_bps": 100,
|
||
|
|
"take_profit_bps": 100,
|
||
|
|
"entry_delay_bars": 1,
|
||
|
|
"cooldown_bars": 0,
|
||
|
|
"max_holding_bars": 1,
|
||
|
|
"allow_weekend_entries": False,
|
||
|
|
"same_bar_policy": "PESSIMISTIC_SL",
|
||
|
|
"cost_model": {
|
||
|
|
"version": "qualification-v1",
|
||
|
|
"entry_commission_bps": 0.0,
|
||
|
|
"exit_commission_bps": 0.0,
|
||
|
|
"entry_slippage_bps": 0.0,
|
||
|
|
"exit_slippage_bps": 0.0,
|
||
|
|
"funding_bps_per_bar": 0.0,
|
||
|
|
"entry_other_cost": 0.0,
|
||
|
|
"exit_other_cost": 0.0,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
snapshot = TradingStudioService.qualification_configuration(
|
||
|
|
scenario, protocol_identity=identity, require_protocol_identity=True
|
||
|
|
)
|
||
|
|
assert snapshot["protocol_identity"] == identity
|
||
|
|
with pytest.raises(ValueError, match="requires a protocol identity"):
|
||
|
|
TradingStudioService.qualification_configuration(
|
||
|
|
scenario, require_protocol_identity=True
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_raw_calibration_has_no_holdout_leak_ab_equivalence():
|
||
|
|
left = [bar(0, 10, 11, 9, 10, 1, (1,)), bar(1, 10, 11, 9, 10, 100, (100,))]
|
||
|
|
right = [bar(0, 10, 11, 9, 10, 1, (1,)), bar(1, 10, 11, 9, 10, -999, (-999,))]
|
||
|
|
assert calibrated(left).threshold == calibrated(right).threshold == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_cohort_raw_calibration_retains_zero_signal_train_provenance_without_fabrication():
|
||
|
|
bars = [
|
||
|
|
bar(0, 100, 101, 99, 100, None, (1.0,), "train-0"),
|
||
|
|
bar(1, 100, 101, 99, 100, None, (2.0,), "train-1"),
|
||
|
|
bar(2, 100, 101, 99, 100, None, (3.0,), "replay-0"),
|
||
|
|
]
|
||
|
|
calibration = calibrate_train_only(
|
||
|
|
bars,
|
||
|
|
method=CalibrationMethod.RAW,
|
||
|
|
fold="fold-a",
|
||
|
|
train_start=BASE,
|
||
|
|
train_end=BASE + timedelta(days=2),
|
||
|
|
source_record_ids=("frozen-member",),
|
||
|
|
source_combo="frozen-hs22-combo",
|
||
|
|
allow_empty_raw=True,
|
||
|
|
)
|
||
|
|
result = QualificationReplayV1().run(
|
||
|
|
bars[2:],
|
||
|
|
strategy="strategy-v1",
|
||
|
|
dataset="Cohort001",
|
||
|
|
fold="fold-a",
|
||
|
|
runner="cohort_qualification_hyperscalper_001:v1",
|
||
|
|
scenario=ScenarioSpec("baseline", 100, 100),
|
||
|
|
calibration=calibration,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert calibration.threshold == 0.0
|
||
|
|
assert calibration.sample_count == 0
|
||
|
|
assert calibration.source_combo == "frozen-hs22-combo"
|
||
|
|
assert calibration.input_hash
|
||
|
|
assert result.ledger == ()
|
||
|
|
assert result.rescue.trade_count == 0
|
||
|
|
assert result.rescue.net_pnl == 0
|
||
|
|
|
||
|
|
|
||
|
|
def test_percentile_calibration_still_rejects_zero_qualifying_train_signals():
|
||
|
|
with pytest.raises(ValueError, match="Train-only calibration"):
|
||
|
|
calibrate_train_only(
|
||
|
|
[bar(0, 100, 101, 99, 100)],
|
||
|
|
method=CalibrationMethod.PERCENTILE,
|
||
|
|
fold="fold-a",
|
||
|
|
train_start=BASE,
|
||
|
|
train_end=BASE + timedelta(days=1),
|
||
|
|
source_record_ids=("frozen-member",),
|
||
|
|
source_combo="frozen-hs22-combo",
|
||
|
|
percentile=50,
|
||
|
|
allow_empty_raw=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_percentile_calibration_records_provenance_and_continuous_context():
|
||
|
|
bars = [bar(0, 10, 11, 9, 10, 1, (0.1, 0.2)), bar(0, 10, 11, 9, 10, 3, (0.3, 0.4))]
|
||
|
|
value = calibrate_train_only(
|
||
|
|
bars,
|
||
|
|
method=CalibrationMethod.PERCENTILE,
|
||
|
|
fold="fold-a",
|
||
|
|
train_start=BASE,
|
||
|
|
train_end=BASE + timedelta(days=1),
|
||
|
|
source_record_ids=("a",),
|
||
|
|
source_combo="ohlcv+continuous",
|
||
|
|
percentile=100,
|
||
|
|
)
|
||
|
|
assert value.threshold == 3 and value.input_hash and value.source_combo == "ohlcv+continuous"
|
||
|
|
|
||
|
|
|
||
|
|
def test_reference_mode_is_absent_and_scenario_names_are_required():
|
||
|
|
assert not hasattr(QualificationReplayV1, "reference_mode")
|
||
|
|
with pytest.raises(ValueError, match="named"):
|
||
|
|
ScenarioSpec("", 100, 100)
|
||
|
|
with pytest.raises(
|
||
|
|
ValueError,
|
||
|
|
match="missing materialized strategy_version, dataset_version, bars, calibration",
|
||
|
|
):
|
||
|
|
TradingStudioService().run_qualification_canary(
|
||
|
|
strategy_version=None,
|
||
|
|
dataset_version=None,
|
||
|
|
fold="fold-a",
|
||
|
|
scenario=ScenarioSpec("canary", 100, 100),
|
||
|
|
runner_name="runner",
|
||
|
|
bars=[],
|
||
|
|
calibration=None,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_materialized_canary_persists_without_creating_live_run():
|
||
|
|
project = Project.objects.create(name="Canary", goal="Test qualification persistence")
|
||
|
|
trading_project = TradingProject.objects.create(
|
||
|
|
project=project,
|
||
|
|
name="Canary Trading",
|
||
|
|
slug="canary-trading",
|
||
|
|
goal="Test qualification persistence",
|
||
|
|
)
|
||
|
|
dataset = MarketDataset.objects.create(
|
||
|
|
trading_project=trading_project,
|
||
|
|
name="Materialized BTCUSDT",
|
||
|
|
kind=DataKind.OHLCV,
|
||
|
|
)
|
||
|
|
dataset_version = MarketDatasetVersion.objects.create(
|
||
|
|
dataset=dataset,
|
||
|
|
version="frozen-v1",
|
||
|
|
reference="artifact://materialized.csv",
|
||
|
|
content_hash="a" * 64,
|
||
|
|
)
|
||
|
|
strategy = Strategy.objects.create(trading_project=trading_project, name="HS22")
|
||
|
|
strategy_version = StrategyVersion.objects.create(
|
||
|
|
strategy=strategy,
|
||
|
|
version="frozen-v1",
|
||
|
|
genome={"combo": [1]},
|
||
|
|
fingerprint="b" * 64,
|
||
|
|
)
|
||
|
|
bars = [
|
||
|
|
bar(0, 100, 101, 99, 100, 1),
|
||
|
|
bar(1, 100, 101, 99, 100, 2),
|
||
|
|
bar(2, 100, 102, 99, 101),
|
||
|
|
bar(3, 101, 103, 100, 102),
|
||
|
|
]
|
||
|
|
|
||
|
|
service = TradingStudioService()
|
||
|
|
run = service.run_qualification_canary(
|
||
|
|
strategy_version=strategy_version,
|
||
|
|
dataset_version=dataset_version,
|
||
|
|
fold="fold-a",
|
||
|
|
scenario=ScenarioSpec("baseline", 100, 100),
|
||
|
|
runner_name="test-runner",
|
||
|
|
bars=bars,
|
||
|
|
calibration=calibrated(bars),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert run.replay_mode == "CANARY_ONLY"
|
||
|
|
assert run.ledger_rows.count() == 1
|
||
|
|
assert LiveStrategyRun.objects.count() == 0
|
||
|
|
|
||
|
|
zero_bars = [bar(0, 100, 101, 99, 100), bar(1, 100, 101, 99, 100)]
|
||
|
|
zero_run = service.run_qualification_canary(
|
||
|
|
strategy_version=strategy_version,
|
||
|
|
dataset_version=dataset_version,
|
||
|
|
fold="fold-zero",
|
||
|
|
scenario=ScenarioSpec("zero-trade", 100, 100),
|
||
|
|
runner_name="test-runner",
|
||
|
|
bars=zero_bars,
|
||
|
|
calibration=calibrate_train_only(
|
||
|
|
zero_bars,
|
||
|
|
method=CalibrationMethod.RAW,
|
||
|
|
fold="fold-zero",
|
||
|
|
train_start=BASE,
|
||
|
|
train_end=BASE + timedelta(days=1),
|
||
|
|
source_record_ids=("frozen-member",),
|
||
|
|
source_combo="frozen-hs22-combo",
|
||
|
|
allow_empty_raw=True,
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
assert zero_run.ledger_rows.count() == 0
|
||
|
|
assert zero_run.summary["ledger_count"] == 0
|
||
|
|
assert zero_run.summary["rescue"]["trade_count"] == 0
|
||
|
|
assert zero_run.summary["rescue"]["net_pnl"] == 0
|