1096 lines
49 KiB
Python
1096 lines
49 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only HYPERSCALPER_FEATURE_GAP_ANALYSIS_V1 evidence runner.
|
|
|
|
This program never imports a strategy runtime, invokes an optimizer, or writes to
|
|
an input artifact. It only summarizes frozen oracle and historical evidence.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
ARTIFACT = "HYPERSCALPER_FEATURE_GAP_ANALYSIS_V1"
|
|
EXPECTED_FEATURES = 711
|
|
MAX_SAMPLE_ROWS = 16_384
|
|
LINEAGE_ROLES = ("trend", "signal", "trigger", "confirm", "vol")
|
|
DEQ_RETURN_FIELDS = tuple(f"return_{bars}_bars_bps" for bars in (1, 2, 3, 5, 10))
|
|
DEQ_MEAN_FIELDS = ("mfe_bps", "mae_bps", "time_to_positive_bars")
|
|
DEQ_PRIMITIVE_FIELDS = DEQ_RETURN_FIELDS + (
|
|
"mfe_bps",
|
|
"mae_bps",
|
|
"time_to_positive_bars",
|
|
"time_to_25_bps_bars",
|
|
"time_to_50_bps_bars",
|
|
"max_adverse_before_positive_bps",
|
|
"winner_negative_first",
|
|
"recovery_bars",
|
|
)
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def source_hash(path: Path) -> str:
|
|
if path.is_file():
|
|
return sha256(path)
|
|
digest = hashlib.sha256()
|
|
for item in sorted(path.glob("*.npy")):
|
|
digest.update(item.name.encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(bytes.fromhex(sha256(item)))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"{path} must contain a JSON object")
|
|
return value
|
|
|
|
|
|
def feature_rows(path: Path) -> list[dict[str, Any]]:
|
|
payload = load_json(path)
|
|
rows = payload.get("primitives") or payload.get("requests")
|
|
if not isinstance(rows, list):
|
|
raise ValueError("engineering map/request must contain primitives or requests")
|
|
result = []
|
|
seen = set()
|
|
for row in rows:
|
|
key = (int(row["indicator_id"]), int(row["period"]), float(row["p1"]))
|
|
if key in seen:
|
|
raise ValueError(f"duplicate feature primitive: {key}")
|
|
seen.add(key)
|
|
result.append(
|
|
{
|
|
"feature_key": f"{key[0]}:{key[1]}:{key[2]:g}",
|
|
"indicator_id": key[0],
|
|
"period": key[1],
|
|
"p1": key[2],
|
|
"request_id": row.get("request_id"),
|
|
}
|
|
)
|
|
if len(result) != EXPECTED_FEATURES:
|
|
raise ValueError(
|
|
f"source map must contain exactly {EXPECTED_FEATURES} unique primitives; "
|
|
f"found {len(result)}"
|
|
)
|
|
return result
|
|
|
|
|
|
def accepted_ids(manifest: dict[str, Any]) -> set[str]:
|
|
coverage = manifest.get("coverage", {})
|
|
values = coverage.get("feature_versions", manifest.get("accepted_request_ids", []))
|
|
return {str(value) for value in values} if isinstance(values, list) else set()
|
|
|
|
|
|
def checkpoint_columns(
|
|
checkpoint: Path, rows: list[dict[str, Any]]
|
|
) -> tuple[np.ndarray, list[str]]:
|
|
with np.load(checkpoint, allow_pickle=False) as archive:
|
|
names = list(archive.files)
|
|
by_request = {str(row["request_id"]): row for row in rows if row["request_id"]}
|
|
if set(names) == set(by_request):
|
|
ordered_names = [str(row["request_id"]) for row in rows]
|
|
elif len(names) == EXPECTED_FEATURES:
|
|
# A frozen raw NPZ may lack request IDs; source-map order is the only asserted mapping.
|
|
ordered_names = sorted(names)
|
|
else:
|
|
raise ValueError(
|
|
"checkpoint keys do not match request IDs and do not contain 711 arrays"
|
|
)
|
|
arrays = [np.asarray(archive[name], dtype=np.float64).reshape(-1) for name in ordered_names]
|
|
lengths = {array.size for array in arrays}
|
|
if len(lengths) != 1 or not next(iter(lengths)):
|
|
raise ValueError("checkpoint arrays must be non-empty and have equal length")
|
|
return np.column_stack(arrays), ordered_names
|
|
|
|
|
|
def checkpoint_directory_columns(
|
|
checkpoint_dir: Path, rows: list[dict[str, Any]]
|
|
) -> tuple[np.ndarray, list[str]]:
|
|
files = sorted(checkpoint_dir.glob("*.npy"))
|
|
if len(files) != EXPECTED_FEATURES:
|
|
raise ValueError(
|
|
f"checkpoint directory must contain exactly {EXPECTED_FEATURES} .npy files; "
|
|
f"found {len(files)}"
|
|
)
|
|
by_primitive = {
|
|
(row["indicator_id"], row["period"], row["p1"]): row for row in rows
|
|
}
|
|
mapped: dict[tuple[int, int, float], Path] = {}
|
|
for path in files:
|
|
parts = path.stem.split("_")
|
|
if len(parts) != 4 or parts[0] != "hs22":
|
|
raise ValueError(f"invalid checkpoint filename: {path.name}")
|
|
try:
|
|
primitive = (int(parts[1]), int(parts[2]), float(parts[3]))
|
|
except ValueError as error:
|
|
raise ValueError(f"invalid checkpoint filename: {path.name}") from error
|
|
if primitive not in by_primitive:
|
|
raise ValueError(f"checkpoint filename is not an engineering primitive: {path.name}")
|
|
if primitive in mapped:
|
|
raise ValueError(f"duplicate checkpoint primitive: {primitive}")
|
|
mapped[primitive] = path
|
|
if set(mapped) != set(by_primitive):
|
|
raise ValueError("checkpoint filenames do not cover every engineering primitive")
|
|
ordered_paths = [
|
|
mapped[(row["indicator_id"], row["period"], row["p1"])] for row in rows
|
|
]
|
|
arrays = [
|
|
np.asarray(np.load(path, allow_pickle=False), dtype=np.float64).reshape(-1)
|
|
for path in ordered_paths
|
|
]
|
|
lengths = {array.size for array in arrays}
|
|
if len(lengths) != 1 or not next(iter(lengths)):
|
|
raise ValueError("checkpoint arrays must be non-empty and have equal length")
|
|
return np.column_stack(arrays), [path.name for path in ordered_paths]
|
|
|
|
|
|
def sampled_checkpoint_columns(
|
|
checkpoint: Path | None,
|
|
checkpoint_dir: Path | None,
|
|
rows: list[dict[str, Any]],
|
|
sample_rows: int,
|
|
) -> tuple[np.ndarray, list[str], int]:
|
|
"""Read only selected observations from each checkpoint column.
|
|
|
|
This intentionally avoids materializing the 129k by 711 source matrix. NPZ
|
|
members are decompressed one at a time; directory checkpoints are mmap'd.
|
|
"""
|
|
if checkpoint_dir:
|
|
files = sorted(checkpoint_dir.glob("*.npy"))
|
|
if len(files) != EXPECTED_FEATURES:
|
|
raise ValueError("checkpoint directory must contain exactly 711 .npy files")
|
|
by_primitive = {
|
|
(row["indicator_id"], row["period"], row["p1"]): row for row in rows
|
|
}
|
|
mapped: dict[tuple[int, int, float], Path] = {}
|
|
for path in files:
|
|
parts = path.stem.split("_")
|
|
if len(parts) != 4 or parts[0] != "hs22":
|
|
raise ValueError(f"invalid checkpoint filename: {path.name}")
|
|
primitive = (int(parts[1]), int(parts[2]), float(parts[3]))
|
|
if primitive not in by_primitive or primitive in mapped:
|
|
raise ValueError(f"invalid or duplicate checkpoint primitive: {path.name}")
|
|
mapped[primitive] = path
|
|
if set(mapped) != set(by_primitive):
|
|
raise ValueError("checkpoint filenames do not cover every engineering primitive")
|
|
ordered = [mapped[(row["indicator_id"], row["period"], row["p1"])] for row in rows]
|
|
first = np.load(ordered[0], allow_pickle=False, mmap_mode="r").reshape(-1)
|
|
size = first.size
|
|
indices = np.linspace(0, size - 1, min(sample_rows, size), dtype=int)
|
|
arrays = []
|
|
for path in ordered:
|
|
values = np.load(path, allow_pickle=False, mmap_mode="r").reshape(-1)
|
|
if values.size != size:
|
|
raise ValueError("checkpoint arrays must be non-empty and have equal length")
|
|
arrays.append(np.asarray(values[indices], dtype=np.float64))
|
|
return np.column_stack(arrays), [path.name for path in ordered], size
|
|
|
|
assert checkpoint is not None
|
|
with np.load(checkpoint, allow_pickle=False) as archive:
|
|
names = list(archive.files)
|
|
by_request = {str(row["request_id"]): row for row in rows if row["request_id"]}
|
|
if set(names) == set(by_request):
|
|
ordered_names = [str(row["request_id"]) for row in rows]
|
|
elif len(names) == EXPECTED_FEATURES:
|
|
ordered_names = sorted(names)
|
|
else:
|
|
raise ValueError("checkpoint keys do not match request IDs and do not contain 711 arrays")
|
|
first = np.asarray(archive[ordered_names[0]]).reshape(-1)
|
|
size = first.size
|
|
if not size:
|
|
raise ValueError("checkpoint arrays must be non-empty and have equal length")
|
|
indices = np.linspace(0, size - 1, min(sample_rows, size), dtype=int)
|
|
arrays = []
|
|
for name in ordered_names:
|
|
values = np.asarray(archive[name], dtype=np.float64).reshape(-1)
|
|
if values.size != size:
|
|
raise ValueError("checkpoint arrays must be non-empty and have equal length")
|
|
arrays.append(values[indices])
|
|
return np.column_stack(arrays), ordered_names, size
|
|
|
|
|
|
def semantic_types(path: Path | None, rows: list[dict[str, Any]]) -> tuple[list[dict[str, str]], str | None]:
|
|
"""Resolve output type and domain from a metadata-only semantic map."""
|
|
defaults = [{"output_type": "continuous", "domain": "unclassified"} for _ in rows]
|
|
if path is None:
|
|
return defaults, "no semantic map supplied; output types default to continuous"
|
|
try:
|
|
if path.suffix.lower() == ".parquet":
|
|
import pyarrow.parquet as pq
|
|
|
|
items = pq.read_table(path).to_pylist()
|
|
else:
|
|
payload = load_json(path)
|
|
items = payload.get("primitives", payload.get("features", payload.get("rows", payload.get("semantic_map"))))
|
|
if isinstance(items, dict):
|
|
items = items.get("primitives", items.get("features", items.get("rows")))
|
|
except (ImportError, OSError, ValueError) as error:
|
|
return defaults, f"cannot read semantic map: {error}"
|
|
if not isinstance(items, list):
|
|
return defaults, "semantic map has no primitives/features/rows array"
|
|
mapped: dict[str, dict[str, str]] = {}
|
|
for item in items:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
key = item.get("feature_key")
|
|
if key is None and all(name in item for name in ("indicator_id", "period", "p1")):
|
|
key = f"{int(item['indicator_id'])}:{int(item['period'])}:{float(item['p1']):g}"
|
|
if key is None:
|
|
continue
|
|
raw_type = str(item.get("output_type", item.get("semantic_type", item.get("value_type", "continuous")))).lower()
|
|
output_type = "event" if raw_type in {"event", "detection", "binary_event"} else "state" if raw_type in {"state", "categorical", "boolean"} else "continuous"
|
|
domain = str(item.get("domain", item.get("semantic_domain", item.get("engineering_family", item.get("family", "unclassified")))))
|
|
mapped[str(key)] = {"output_type": output_type, "domain": domain}
|
|
result = [mapped.get(row["feature_key"], defaults[index]) for index, row in enumerate(rows)]
|
|
missing = sum(row["feature_key"] not in mapped for row in rows)
|
|
return result, f"semantic map does not classify {missing} features" if missing else None
|
|
|
|
|
|
def rank_finite_columns(matrix: np.ndarray) -> np.ndarray:
|
|
"""Return deterministic ordinal ranks, leaving non-finite observations masked."""
|
|
ranks = np.full(matrix.shape, np.nan, dtype=np.float64)
|
|
for column in range(matrix.shape[1]):
|
|
finite = np.isfinite(matrix[:, column])
|
|
count = int(finite.sum())
|
|
if count:
|
|
order = np.argsort(matrix[finite, column], kind="mergesort")
|
|
column_ranks = np.empty(count, dtype=np.float64)
|
|
column_ranks[order] = np.arange(count)
|
|
ranks[finite, column] = column_ranks
|
|
return ranks
|
|
|
|
|
|
def pairwise_correlations(
|
|
values: np.ndarray, finite: np.ndarray, pair_samples: np.ndarray, min_pair_samples: int
|
|
) -> np.ndarray:
|
|
"""Correlate columns using only rows finite for each pair, without pair materialization."""
|
|
masked_values = np.where(finite, values, 0.0)
|
|
finite_float = finite.astype(np.float64)
|
|
counts = pair_samples.astype(np.float64)
|
|
sums = masked_values.T @ finite_float
|
|
squared_sums = (masked_values * masked_values).T @ finite_float
|
|
covariance = masked_values.T @ masked_values
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
covariance -= sums * sums.T / counts
|
|
variance = squared_sums - sums * sums / counts
|
|
correlation = covariance / np.sqrt(variance * variance.T)
|
|
invalid = (pair_samples < min_pair_samples) | (variance <= 0.0) | (variance.T <= 0.0)
|
|
correlation[invalid] = np.nan
|
|
np.fill_diagonal(correlation, np.nan)
|
|
return correlation
|
|
|
|
|
|
def pairwise_redundancy(
|
|
sample: np.ndarray, min_pair_samples: int
|
|
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
"""Calculate pairwise-finite Pearson and pre-ranked ordinal correlations."""
|
|
finite = np.isfinite(sample)
|
|
pair_samples = finite.astype(np.int32).T @ finite.astype(np.int32)
|
|
pearson = pairwise_correlations(sample, finite, pair_samples, min_pair_samples)
|
|
# Ranks are computed once per feature, rather than once per feature pair.
|
|
ranks = rank_finite_columns(sample)
|
|
spearman = pairwise_correlations(ranks, finite, pair_samples, min_pair_samples)
|
|
return pearson, spearman, pair_samples
|
|
|
|
|
|
def type_aware_redundancy(
|
|
sample: np.ndarray, output_types: list[str], min_pair_samples: int
|
|
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
|
"""Use correlations for continuous values and overlap metrics for detections."""
|
|
feature_count = sample.shape[1]
|
|
finite = np.isfinite(sample)
|
|
pair_samples = finite.astype(np.int32).T @ finite.astype(np.int32)
|
|
pearson = np.full((feature_count, feature_count), np.nan)
|
|
spearman = np.full((feature_count, feature_count), np.nan)
|
|
agreement = np.full((feature_count, feature_count), np.nan)
|
|
jaccard = np.full((feature_count, feature_count), np.nan)
|
|
continuous = np.array([kind == "continuous" for kind in output_types])
|
|
if continuous.any():
|
|
indices = np.flatnonzero(continuous)
|
|
continuous_values = sample[:, indices]
|
|
continuous_finite = finite[:, indices]
|
|
continuous_counts = pair_samples[np.ix_(indices, indices)]
|
|
pearson[np.ix_(indices, indices)] = pairwise_correlations(
|
|
continuous_values, continuous_finite, continuous_counts, min_pair_samples
|
|
)
|
|
ranks = rank_finite_columns(continuous_values)
|
|
spearman[np.ix_(indices, indices)] = pairwise_correlations(
|
|
ranks, continuous_finite, continuous_counts, min_pair_samples
|
|
)
|
|
|
|
discrete = ~continuous
|
|
if discrete.any():
|
|
indices = np.flatnonzero(discrete)
|
|
values = sample[:, indices]
|
|
valid = finite[:, indices]
|
|
detected = valid & (values != 0)
|
|
intersection = detected.astype(np.int32).T @ detected.astype(np.int32)
|
|
detected_counts = detected.sum(axis=0)
|
|
union = detected_counts[:, None] + detected_counts[None, :] - intersection
|
|
discrete_counts = pair_samples[np.ix_(indices, indices)]
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
discrete_jaccard = intersection / union
|
|
discrete_jaccard[(discrete_counts < min_pair_samples) | (union == 0)] = np.nan
|
|
np.fill_diagonal(discrete_jaccard, np.nan)
|
|
jaccard[np.ix_(indices, indices)] = discrete_jaccard
|
|
|
|
binary = np.all(~valid | ((values == 0) | (values == 1)), axis=0)
|
|
zero = valid & (values == 0)
|
|
equal_binary = (
|
|
zero.astype(np.int32).T @ zero.astype(np.int32) + intersection
|
|
)
|
|
with np.errstate(invalid="ignore", divide="ignore"):
|
|
binary_agreement = equal_binary / discrete_counts
|
|
binary_pairs = binary[:, None] & binary[None, :]
|
|
binary_agreement[(discrete_counts < min_pair_samples) | ~binary_pairs] = np.nan
|
|
np.fill_diagonal(binary_agreement, np.nan)
|
|
agreement[np.ix_(indices, indices)] = binary_agreement
|
|
|
|
# Categorical state outputs are uncommon; retain exact equality semantics for them.
|
|
for left_offset, left in enumerate(indices):
|
|
if binary[left_offset]:
|
|
continue
|
|
for right_offset, right in enumerate(indices):
|
|
if (
|
|
right == left
|
|
or (not binary[right_offset] and right < left)
|
|
or pair_samples[left, right] < min_pair_samples
|
|
):
|
|
continue
|
|
mask = finite[:, left] & finite[:, right]
|
|
value = float(np.mean(sample[mask, left] == sample[mask, right]))
|
|
agreement[left, right] = agreement[right, left] = value
|
|
return pearson, spearman, agreement, jaccard, pair_samples
|
|
|
|
|
|
def primary_redundancy_matrix(
|
|
pearson: np.ndarray, agreement: np.ndarray, jaccard: np.ndarray, output_types: list[str]
|
|
) -> np.ndarray:
|
|
result = np.full_like(pearson, np.nan)
|
|
for left in range(len(output_types)):
|
|
for right in range(len(output_types)):
|
|
if output_types[left] == output_types[right] == "continuous":
|
|
result[left, right] = abs(pearson[left, right])
|
|
elif output_types[left] == output_types[right] == "event":
|
|
result[left, right] = jaccard[left, right]
|
|
elif output_types[left] == output_types[right] == "state":
|
|
result[left, right] = agreement[left, right]
|
|
return result
|
|
|
|
|
|
def clusters(correlation: np.ndarray, threshold: float = 0.95) -> list[list[int]]:
|
|
parent = list(range(correlation.shape[0]))
|
|
|
|
def find(item: int) -> int:
|
|
while parent[item] != item:
|
|
parent[item] = parent[parent[item]]
|
|
item = parent[item]
|
|
return item
|
|
|
|
def join(left: int, right: int) -> None:
|
|
left, right = find(left), find(right)
|
|
if left != right:
|
|
parent[right] = left
|
|
|
|
for left in range(correlation.shape[0]):
|
|
for right in range(left):
|
|
if abs(correlation[left, right]) >= threshold:
|
|
join(left, right)
|
|
grouped: dict[int, list[int]] = {}
|
|
for index in range(len(parent)):
|
|
grouped.setdefault(find(index), []).append(index)
|
|
return [members for members in grouped.values() if len(members) > 1]
|
|
|
|
|
|
def parquet_counts(path: Path, label: str) -> tuple[Counter[str], str | None]:
|
|
try:
|
|
import pyarrow.parquet as pq
|
|
except ImportError:
|
|
return Counter(), "pyarrow unavailable"
|
|
try:
|
|
table = pq.read_table(path)
|
|
except Exception as error:
|
|
return Counter(), f"cannot read {label} parquet: {error}"
|
|
names = set(table.column_names)
|
|
key_column = next(
|
|
(name for name in ("feature_key", "primitive", "indicator_id") if name in names), None
|
|
)
|
|
count_column = next(
|
|
(name for name in ("usage_count", "count", "strategy_count") if name in names), None
|
|
)
|
|
if key_column is None:
|
|
return Counter(), f"{label} parquet has no recognized feature key column"
|
|
keys = table[key_column].to_pylist()
|
|
counts = table[count_column].to_pylist() if count_column else [1] * len(keys)
|
|
return Counter(
|
|
{str(key): float(count or 0) for key, count in zip(keys, counts, strict=True)}
|
|
), None
|
|
|
|
|
|
def lineage_combo(value: Any) -> tuple[tuple[int, int, float], ...]:
|
|
"""Parse the five ordered primitive triples stored at the start of lineage combos."""
|
|
if isinstance(value, bytes):
|
|
value = value.decode("utf-8")
|
|
if isinstance(value, str):
|
|
value = json.loads(value)
|
|
if isinstance(value, dict):
|
|
if "combo" in value:
|
|
return lineage_combo(value["combo"])
|
|
if all(role in value for role in LINEAGE_ROLES):
|
|
value = [value[role] for role in LINEAGE_ROLES]
|
|
elif "triples" in value:
|
|
value = value["triples"]
|
|
if not isinstance(value, (list, tuple)):
|
|
raise ValueError("combo is not an array")
|
|
if len(value) < len(LINEAGE_ROLES) * 3:
|
|
raise ValueError("combo does not contain at least five triples")
|
|
# Historical rows append strategy parameters after the five primitives.
|
|
triples = [value[index : index + 3] for index in range(0, len(LINEAGE_ROLES) * 3, 3)]
|
|
result = []
|
|
for triple in triples:
|
|
if not isinstance(triple, (list, tuple)) or len(triple) != 3:
|
|
raise ValueError("combo contains an invalid primitive triple")
|
|
indicator_id, period, p1 = triple
|
|
if isinstance(indicator_id, bool) or isinstance(period, bool):
|
|
raise ValueError("combo primitive IDs must be integers")
|
|
indicator_id, period, p1 = int(indicator_id), int(period), float(p1)
|
|
if not np.isfinite(p1):
|
|
raise ValueError("combo primitive p1 must be finite")
|
|
result.append((indicator_id, period, p1))
|
|
return tuple(result)
|
|
|
|
|
|
def lineage_usage(path: Path) -> tuple[Counter[str], dict[str, Any], str | None]:
|
|
try:
|
|
import pyarrow.parquet as pq
|
|
except ImportError:
|
|
return Counter(), {}, "pyarrow unavailable"
|
|
try:
|
|
table = pq.read_table(path)
|
|
except Exception as error:
|
|
return Counter(), {}, f"cannot read lineage parquet: {error}"
|
|
if "combo_json" not in table.column_names:
|
|
return Counter(), {}, "lineage parquet has no combo_json column"
|
|
|
|
primitive_counts: Counter[str] = Counter()
|
|
role_counts: Counter[str] = Counter()
|
|
combo_counts: Counter[str] = Counter()
|
|
strategy_ids: set[str] = set()
|
|
invalid_rows = 0
|
|
records = table.to_pylist()
|
|
for record in records:
|
|
try:
|
|
triples = lineage_combo(record["combo_json"])
|
|
except (TypeError, ValueError, json.JSONDecodeError, UnicodeDecodeError, OverflowError):
|
|
invalid_rows += 1
|
|
continue
|
|
signature = json.dumps(triples, separators=(",", ":"))
|
|
combo_counts[signature] += 1
|
|
strategy_id = record.get("strategy_id")
|
|
if strategy_id is not None:
|
|
strategy_ids.add(str(strategy_id))
|
|
for role, (indicator_id, period, p1) in zip(LINEAGE_ROLES, triples, strict=True):
|
|
primitive_counts[f"{indicator_id}:{period}:{p1:g}"] += 1
|
|
role_counts[role] += 1
|
|
blocker = (
|
|
f"lineage parquet has {invalid_rows} rows with an unsupported combo_json format"
|
|
if invalid_rows
|
|
else None
|
|
)
|
|
evidence = {
|
|
"lineage_rows": len(records),
|
|
"parsed_combo_rows": sum(combo_counts.values()),
|
|
"invalid_combo_rows": invalid_rows,
|
|
"strategy_ids_observed": len(strategy_ids),
|
|
"role_counts": dict(sorted(role_counts.items())),
|
|
"combo_counts": [
|
|
{"combo_signature": signature, "usage_count": count}
|
|
for signature, count in sorted(combo_counts.items(), key=lambda item: (-item[1], item[0]))
|
|
],
|
|
"feature_family_counts": dict(sorted(combo_counts.items())),
|
|
"primitive_counts": dict(sorted(primitive_counts.items())),
|
|
}
|
|
return primitive_counts, evidence, blocker
|
|
|
|
|
|
def deq_feature_key(value: Any) -> tuple[str, str]:
|
|
"""Return the primitive and indicator-family keys exported with a strategy genome."""
|
|
if not isinstance(value, dict):
|
|
raise ValueError("feature triple is not an object")
|
|
indicator_id, period, p1 = value.get("indicator_id"), value.get("period"), value.get("p1")
|
|
if isinstance(indicator_id, bool) or isinstance(period, bool):
|
|
raise ValueError("feature triple IDs must be integers")
|
|
indicator_id, period, p1 = int(indicator_id), int(period), float(p1)
|
|
if not np.isfinite(p1):
|
|
raise ValueError("feature triple p1 must be finite")
|
|
return f"{indicator_id}:{period}:{p1:g}", str(indicator_id)
|
|
|
|
|
|
def deq_strategy_feature_keys(strategy: dict[str, Any]) -> list[tuple[str, str]]:
|
|
"""Read exported feature triples, falling back to the verified genome combo schema."""
|
|
try:
|
|
triples = strategy.get("feature_triples")
|
|
if not isinstance(triples, list) or not triples:
|
|
raise ValueError("strategy has no feature_triples")
|
|
return [deq_feature_key(triple) for triple in triples]
|
|
except (TypeError, ValueError, OverflowError) as exported_error:
|
|
genome = strategy.get("genome")
|
|
try:
|
|
if not isinstance(genome, dict):
|
|
raise ValueError("strategy has no genome object")
|
|
return [
|
|
(f"{indicator_id}:{period}:{p1:g}", str(indicator_id))
|
|
for indicator_id, period, p1 in lineage_combo(genome.get("combo"))
|
|
]
|
|
except (TypeError, ValueError, json.JSONDecodeError, UnicodeDecodeError, OverflowError) as genome_error:
|
|
raise ValueError(
|
|
f"strategy has no usable feature triples or genome combo ({exported_error}; {genome_error})"
|
|
) from genome_error
|
|
|
|
|
|
def deq_summary(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
|
"""Summarize signed, already-recorded DEQ primitives without rerunning strategies."""
|
|
result: dict[str, Any] = {
|
|
"status": "AVAILABLE" if samples else "INSUFFICIENT",
|
|
"trade_count": len(samples),
|
|
"fold_coverage": sorted({sample["fold"] for sample in samples}),
|
|
"scenario_coverage": sorted({sample["scenario"] for sample in samples}),
|
|
"returns": {},
|
|
"means": {},
|
|
"primitives": {},
|
|
}
|
|
result["fold_count"] = len(result["fold_coverage"])
|
|
result["scenario_count"] = len(result["scenario_coverage"])
|
|
for field in DEQ_RETURN_FIELDS:
|
|
values = [sample["deq"].get(field) for sample in samples]
|
|
values = [float(value) for value in values if value is not None]
|
|
result["returns"][field] = {
|
|
"available_count": len(values),
|
|
"mean": float(np.mean(values)) if values else None,
|
|
# DEQ returns are signed in the strategy direction. Positive is therefore
|
|
# directionally correct, but says nothing causal about an associated feature.
|
|
"directional_correct_count": sum(value > 0 for value in values),
|
|
"directional_incorrect_count": sum(value < 0 for value in values),
|
|
"directional_neutral_count": sum(value == 0 for value in values),
|
|
"directional_correctness_inferable": bool(values),
|
|
}
|
|
for field in DEQ_MEAN_FIELDS:
|
|
values = [sample["deq"].get(field) for sample in samples]
|
|
values = [float(value) for value in values if value is not None]
|
|
result["means"][field] = {
|
|
"available_count": len(values),
|
|
"mean": float(np.mean(values)) if values else None,
|
|
}
|
|
for field in DEQ_PRIMITIVE_FIELDS:
|
|
values = [sample["deq"].get(field) for sample in samples]
|
|
available = [value for value in values if value is not None]
|
|
if available and all(isinstance(value, bool) for value in available):
|
|
result["primitives"][field] = {
|
|
"available_count": len(available),
|
|
"true_count": sum(available),
|
|
"false_count": len(available) - sum(available),
|
|
}
|
|
else:
|
|
numeric = [float(value) for value in available]
|
|
result["primitives"][field] = {
|
|
"available_count": len(numeric),
|
|
"mean": float(np.mean(numeric)) if numeric else None,
|
|
}
|
|
return result
|
|
|
|
|
|
def cohort_deq_evidence(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]], str | None]:
|
|
"""Aggregate the read-only Cohort001 DEQ export by strategy and associated genome inputs."""
|
|
try:
|
|
payload = load_json(path)
|
|
if payload.get("contract") != "Cohort001-DEQ-ledger-export-v1":
|
|
raise ValueError("unexpected contract")
|
|
strategies = payload.get("strategies")
|
|
if not isinstance(strategies, list) or not strategies:
|
|
raise ValueError("strategies must be a non-empty array")
|
|
strategy_summaries = []
|
|
feature_samples: dict[str, list[dict[str, Any]]] = {}
|
|
family_samples: dict[str, list[dict[str, Any]]] = {}
|
|
for strategy in strategies:
|
|
if not isinstance(strategy, dict) or strategy.get("strategy_version_id") is None:
|
|
raise ValueError("strategy has no strategy_version_id")
|
|
feature_keys = deq_strategy_feature_keys(strategy)
|
|
samples = []
|
|
for fold in strategy.get("folds", []):
|
|
if not isinstance(fold, dict) or not isinstance(fold.get("fold"), str):
|
|
raise ValueError("strategy fold is invalid")
|
|
for scenario in fold.get("scenarios", []):
|
|
if not isinstance(scenario, dict) or not isinstance(scenario.get("scenario"), str):
|
|
raise ValueError("strategy scenario is invalid")
|
|
ledger = scenario.get("deq_samples")
|
|
if not isinstance(ledger, list):
|
|
raise ValueError("scenario has no deq_samples array")
|
|
for row in ledger:
|
|
if not isinstance(row, dict) or not isinstance(row.get("deq"), dict):
|
|
raise ValueError("DEQ sample has no deq object")
|
|
samples.append({"fold": fold["fold"], "scenario": scenario["scenario"], "deq": row["deq"]})
|
|
strategy_id = str(strategy["strategy_version_id"])
|
|
summary = deq_summary(samples)
|
|
strategy_summaries.append({"strategy_version_id": strategy_id, **summary})
|
|
for feature_key, family_key in feature_keys:
|
|
feature_samples.setdefault(feature_key, []).extend(samples)
|
|
family_samples.setdefault(family_key, []).extend(samples)
|
|
feature_summaries = [
|
|
{"feature_key": key, "attribution": "ASSOCIATIVE_NOT_CAUSAL", **deq_summary(samples)}
|
|
for key, samples in sorted(feature_samples.items())
|
|
]
|
|
family_summaries = [
|
|
{"indicator_id": key, "attribution": "ASSOCIATIVE_NOT_CAUSAL", **deq_summary(samples)}
|
|
for key, samples in sorted(family_samples.items(), key=lambda item: int(item[0]))
|
|
]
|
|
parquet_rows = [
|
|
{"entity_type": "strategy", "entity_key": item["strategy_version_id"], **deq_parquet_row(item)}
|
|
for item in strategy_summaries
|
|
] + [
|
|
{"entity_type": "feature", "entity_key": item["feature_key"], **deq_parquet_row(item)}
|
|
for item in feature_summaries
|
|
] + [
|
|
{"entity_type": "family", "entity_key": item["indicator_id"], **deq_parquet_row(item)}
|
|
for item in family_summaries
|
|
]
|
|
return {
|
|
"available": True,
|
|
"status": "AVAILABLE",
|
|
"attribution": "ASSOCIATIVE_NOT_CAUSAL",
|
|
"attribution_note": "Multi-feature genome attribution is associative, not causal.",
|
|
"strategy_summaries": strategy_summaries,
|
|
"feature_summaries": feature_summaries,
|
|
"family_summaries": family_summaries,
|
|
}, parquet_rows, None
|
|
except (TypeError, ValueError, json.JSONDecodeError) as error:
|
|
return {}, [], f"cannot use cohort DEQ export: {error}"
|
|
|
|
|
|
def deq_parquet_row(summary: dict[str, Any]) -> dict[str, Any]:
|
|
row = {
|
|
"status": summary["status"],
|
|
"attribution": summary.get("attribution", "STRATEGY_LEVEL"),
|
|
"trade_count": summary["trade_count"],
|
|
"fold_count": len(summary["fold_coverage"]),
|
|
"scenario_count": len(summary["scenario_coverage"]),
|
|
"fold_coverage": json.dumps(summary["fold_coverage"]),
|
|
"scenario_coverage": json.dumps(summary["scenario_coverage"]),
|
|
}
|
|
for field, values in summary["returns"].items():
|
|
row[f"{field}_count"] = values["available_count"]
|
|
row[f"{field}_mean"] = values["mean"]
|
|
row[f"{field}_directional_correct_count"] = values["directional_correct_count"]
|
|
row[f"{field}_directional_incorrect_count"] = values["directional_incorrect_count"]
|
|
for field, values in summary["means"].items():
|
|
row[f"{field}_count"] = values["available_count"]
|
|
row[f"{field}_mean"] = values["mean"]
|
|
for field, values in summary["primitives"].items():
|
|
if "true_count" in values:
|
|
row[f"{field}_true_count"] = values["true_count"]
|
|
row[f"{field}_false_count"] = values["false_count"]
|
|
else:
|
|
row.setdefault(f"{field}_count", values["available_count"])
|
|
row.setdefault(f"{field}_mean", values["mean"])
|
|
return row
|
|
|
|
|
|
def failure_mining(report: Path | None, database: Path | None) -> tuple[Counter[str], list[str]]:
|
|
counts: Counter[str] = Counter()
|
|
blockers: list[str] = []
|
|
if report:
|
|
|
|
def walk(value: Any) -> None:
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
if (
|
|
key.lower() in {"status", "outcome", "failure_reason", "reason"}
|
|
and isinstance(item, str)
|
|
and any(
|
|
word in item.lower() for word in ("fail", "reject", "error", "block")
|
|
)
|
|
):
|
|
counts[item] += 1
|
|
walk(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
walk(item)
|
|
|
|
walk(load_json(report))
|
|
if database:
|
|
try:
|
|
connection = sqlite3.connect(f"file:{database.resolve().as_posix()}?mode=ro", uri=True)
|
|
with connection:
|
|
for (table,) in connection.execute(
|
|
"select name from sqlite_master where type='table'"
|
|
):
|
|
columns = [
|
|
row[1] for row in connection.execute(f'pragma table_info("{table}")')
|
|
]
|
|
status = next(
|
|
(
|
|
column
|
|
for column in columns
|
|
if column.lower() in {"status", "outcome", "failure_reason", "reason"}
|
|
),
|
|
None,
|
|
)
|
|
if status:
|
|
for (value,) in connection.execute(
|
|
f'select "{status}" from "{table}" where "{status}" is not null'
|
|
):
|
|
text = str(value)
|
|
if any(
|
|
word in text.lower()
|
|
for word in ("fail", "reject", "error", "block")
|
|
):
|
|
counts[text] += 1
|
|
except sqlite3.Error as error:
|
|
blockers.append(f"cannot read qualification DB export read-only: {error}")
|
|
return counts, blockers
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
source = parser.add_mutually_exclusive_group(required=True)
|
|
source.add_argument("--engineering-map", type=Path)
|
|
source.add_argument("--oracle-request", type=Path)
|
|
checkpoint = parser.add_mutually_exclusive_group(required=True)
|
|
checkpoint.add_argument("--oracle-checkpoint", type=Path, help="frozen .npz checkpoint")
|
|
checkpoint.add_argument(
|
|
"--oracle-checkpoint-dir",
|
|
type=Path,
|
|
help="directory of frozen hs22_{id}_{period}_{p1}.npy checkpoints",
|
|
)
|
|
parser.add_argument("--acceptance-manifest", type=Path, required=True)
|
|
parser.add_argument(
|
|
"--semantic-map", type=Path,
|
|
help="metadata map with feature_key or primitive fields and output_type/domain",
|
|
)
|
|
parser.add_argument(
|
|
"--historical-usage", type=Path, required=True, help="historical usage parquet"
|
|
)
|
|
parser.add_argument("--lineage", type=Path, required=True, help="lineage parquet")
|
|
parser.add_argument("--qualification-report", type=Path)
|
|
parser.add_argument("--qualification-db-export", type=Path)
|
|
parser.add_argument(
|
|
"--cohort-deq",
|
|
type=Path,
|
|
help="read-only Cohort001 DEQ JSON emitted by export_cohort_deq_v1.py",
|
|
)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--sample-rows", type=int, default=8192)
|
|
parser.add_argument(
|
|
"--min-pair-samples",
|
|
type=int,
|
|
default=2,
|
|
help="minimum jointly finite sampled observations required for a feature pair",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.sample_rows < 2:
|
|
parser.error("--sample-rows must be at least 2")
|
|
if args.sample_rows > MAX_SAMPLE_ROWS:
|
|
parser.error(
|
|
f"--sample-rows may not exceed {MAX_SAMPLE_ROWS}; exact pairwise Spearman is "
|
|
"intentionally not offered because it is not bounded for 711 x 129k inputs"
|
|
)
|
|
if args.min_pair_samples < 2:
|
|
parser.error("--min-pair-samples must be at least 2")
|
|
inputs = [
|
|
path
|
|
for path in (
|
|
args.engineering_map,
|
|
args.oracle_request,
|
|
args.oracle_checkpoint,
|
|
args.oracle_checkpoint_dir,
|
|
args.acceptance_manifest,
|
|
args.semantic_map,
|
|
args.historical_usage,
|
|
args.lineage,
|
|
args.qualification_report,
|
|
args.qualification_db_export,
|
|
args.cohort_deq,
|
|
)
|
|
if path
|
|
]
|
|
if any(not path.is_file() for path in inputs if path != args.oracle_checkpoint_dir):
|
|
parser.error("all supplied input paths except --oracle-checkpoint-dir must be files")
|
|
if args.oracle_checkpoint_dir and not args.oracle_checkpoint_dir.is_dir():
|
|
parser.error("--oracle-checkpoint-dir must be a directory")
|
|
rows = feature_rows(args.engineering_map or args.oracle_request)
|
|
manifest = load_json(args.acceptance_manifest)
|
|
sample, names, source_row_count = sampled_checkpoint_columns(
|
|
args.oracle_checkpoint, args.oracle_checkpoint_dir, rows, args.sample_rows
|
|
)
|
|
blockers: list[str] = []
|
|
semantics, semantic_blocker = semantic_types(args.semantic_map, rows)
|
|
if semantic_blocker:
|
|
blockers.append(semantic_blocker)
|
|
output_types = [item["output_type"] for item in semantics]
|
|
finite_samples = np.isfinite(sample).sum(axis=0)
|
|
pearson, spearman, agreement, jaccard, pair_samples = type_aware_redundancy(
|
|
sample, output_types, args.min_pair_samples
|
|
)
|
|
primary = primary_redundancy_matrix(pearson, agreement, jaccard, output_types)
|
|
usage, usage_blocker = parquet_counts(args.historical_usage, "historical usage")
|
|
lineage, lineage_evidence, lineage_blocker = lineage_usage(args.lineage)
|
|
blockers.extend(item for item in (usage_blocker, lineage_blocker) if item)
|
|
failures, failure_blockers = failure_mining(
|
|
args.qualification_report, args.qualification_db_export
|
|
)
|
|
blockers.extend(failure_blockers)
|
|
deq_rows: list[dict[str, Any]] = []
|
|
if args.cohort_deq:
|
|
deq_evidence, deq_rows, deq_blocker = cohort_deq_evidence(args.cohort_deq)
|
|
if deq_blocker:
|
|
blockers.append(deq_blocker)
|
|
deq_evidence = {
|
|
"available": False,
|
|
"status": "BLOCKED",
|
|
"blocker": deq_blocker,
|
|
}
|
|
else:
|
|
deq_evidence = {
|
|
"available": False,
|
|
"status": "BLOCKED",
|
|
"blocker": (
|
|
"No supplied DEQ artifact or executable decision-equivalence protocol; "
|
|
"this runner does not fabricate DEQ."
|
|
),
|
|
}
|
|
request_ids = {str(row["request_id"]) for row in rows if row["request_id"]}
|
|
accepted = accepted_ids(manifest)
|
|
if request_ids and not request_ids <= accepted:
|
|
blockers.append(
|
|
"acceptance manifest does not cover "
|
|
f"{len(request_ids - accepted)} source-map request IDs"
|
|
)
|
|
acceptance_status = "covered" if request_ids and request_ids <= accepted else "not_proven"
|
|
feature_output = []
|
|
for index, row in enumerate(rows):
|
|
scores = primary[index].copy()
|
|
scores[index] = np.nan
|
|
peer = int(np.nanargmax(scores)) if np.isfinite(scores).any() else None
|
|
finite_count = int(finite_samples[index])
|
|
redundancy_status = (
|
|
"unusable_no_finite_observations"
|
|
if finite_count == 0
|
|
else "usable"
|
|
if peer is not None
|
|
else "insufficient_pairwise_observations"
|
|
)
|
|
feature_output.append(
|
|
{
|
|
**row,
|
|
"checkpoint_key": names[index],
|
|
**semantics[index],
|
|
"historical_usage": usage.get(
|
|
row["feature_key"], usage.get(str(row["indicator_id"]), 0)
|
|
),
|
|
"lineage_mentions": lineage.get(row["feature_key"], 0),
|
|
"finite_sample_count": finite_count,
|
|
"redundancy_status": redundancy_status,
|
|
"redundancy_metric": (
|
|
"abs_pearson" if output_types[index] == "continuous" else
|
|
"jaccard_detection" if output_types[index] == "event" else "agreement"
|
|
),
|
|
"max_redundancy": float(scores[peer]) if peer is not None else None,
|
|
"max_abs_pearson": float(abs(pearson[index, peer])) if peer is not None and np.isfinite(pearson[index, peer]) else None,
|
|
"max_abs_spearman": float(abs(spearman[index, peer])) if peer is not None and np.isfinite(spearman[index, peer]) else None,
|
|
"max_agreement": float(agreement[index, peer]) if peer is not None and np.isfinite(agreement[index, peer]) else None,
|
|
"max_jaccard_detection": float(jaccard[index, peer]) if peer is not None and np.isfinite(jaccard[index, peer]) else None,
|
|
"most_redundant_feature_key": (
|
|
rows[peer]["feature_key"] if peer is not None else None
|
|
),
|
|
"most_redundant_pair_sample_count": (
|
|
int(pair_samples[index, peer]) if peer is not None else 0
|
|
),
|
|
"acceptance": acceptance_status,
|
|
}
|
|
)
|
|
deq_by_feature = {
|
|
item["feature_key"]: item for item in deq_evidence.get("feature_summaries", [])
|
|
}
|
|
cluster_rows = []
|
|
for index, members in enumerate(clusters(primary), start=1):
|
|
# Historical usage is the deterministic representative tie-breaker.
|
|
representative = max(
|
|
members,
|
|
key=lambda item: (feature_output[item]["historical_usage"], rows[item]["feature_key"]),
|
|
)
|
|
cluster_rows.append(
|
|
{
|
|
"cluster_id": index,
|
|
"metric": feature_output[representative]["redundancy_metric"],
|
|
"representative_feature_key": rows[representative]["feature_key"],
|
|
"members": [rows[item]["feature_key"] for item in members],
|
|
"size": len(members),
|
|
"domains": sorted({semantics[item]["domain"] for item in members}),
|
|
"usage_total": sum(feature_output[item]["historical_usage"] for item in members),
|
|
"lineage_mentions_total": sum(feature_output[item]["lineage_mentions"] for item in members),
|
|
"representative_deq": deq_by_feature.get(rows[representative]["feature_key"]),
|
|
}
|
|
)
|
|
taxonomy = load_json(Path(__file__).with_name("hyperscalper_candidate_taxonomy_v1.json"))
|
|
result = {
|
|
"schema_version": 1,
|
|
"artifact": ARTIFACT,
|
|
"read_only": True,
|
|
"strategy_execution": "not performed",
|
|
"source_hashes": {str(path): source_hash(path) for path in inputs},
|
|
"information_map": {
|
|
"expected_features": EXPECTED_FEATURES,
|
|
"mapped_features": len(rows),
|
|
"acceptance": acceptance_status,
|
|
"features": feature_output,
|
|
},
|
|
"redundancy": {
|
|
"method": (
|
|
"type-aware pairwise-finite sampled metrics: Pearson and pre-ranked ordinal "
|
|
"correlation for continuous, agreement for state, Jaccard detection for event"
|
|
),
|
|
"mode": "sampled_bounded_memory",
|
|
"exact_mode": "not offered: exact pairwise Spearman across 711 x 129k is not bounded-memory/time practical",
|
|
"source_rows": source_row_count,
|
|
"sample_rows_requested": args.sample_rows,
|
|
"sample_rows_used": int(sample.shape[0]),
|
|
"min_pair_samples": args.min_pair_samples,
|
|
"features_without_finite_observations": int((finite_samples == 0).sum()),
|
|
"cluster_threshold": 0.95,
|
|
"clusters": cluster_rows,
|
|
"map": {
|
|
"format": "parquet",
|
|
"path": "hyperscalper_feature_redundancy_map_v1.parquet",
|
|
"pair_count": len(rows) * (len(rows) - 1) // 2,
|
|
},
|
|
},
|
|
"lineage_usage_bias": {
|
|
"historical_usage_available": usage_blocker is None,
|
|
"lineage_available": lineage_blocker is None,
|
|
"historical_search_bias_evidence": lineage_evidence,
|
|
},
|
|
"cohort_failure_mining": {
|
|
"failures": dict(failures.most_common()),
|
|
"report_supplied": bool(args.qualification_report),
|
|
"db_export_supplied": bool(args.qualification_db_export),
|
|
},
|
|
"decision_equivalence": deq_evidence,
|
|
"candidate_taxonomy": taxonomy,
|
|
"blockers": blockers,
|
|
}
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
json_path = args.output_dir / "hyperscalper_feature_gap_analysis_v1.json"
|
|
markdown = (
|
|
"# HYPERSCALPER Feature Gap Analysis V1\n\n"
|
|
"Read-only: yes. Strategy execution: not performed.\n\n"
|
|
f"- Information map: {len(rows)}/711 features\n"
|
|
f"- Acceptance: {acceptance_status}\n"
|
|
f"- Redundancy clusters: {len(cluster_rows)}\n"
|
|
f"- DEQ: {deq_evidence['status']}"
|
|
) + (
|
|
" (supplied Cohort001 ledger; multi-feature attribution is associative, not causal)\n"
|
|
if deq_evidence["available"]
|
|
else " (not supplied or invalid; not fabricated)\n"
|
|
) + (
|
|
f"- Blockers: {len(blockers)}\n\n## Blockers\n"
|
|
+ "\n".join(f"- {item}" for item in blockers)
|
|
+ "\n"
|
|
)
|
|
(args.output_dir / "hyperscalper_feature_gap_analysis_v1.md").write_text(
|
|
markdown, encoding="utf-8"
|
|
)
|
|
(args.output_dir / "hyperscalper_feature_redundancy_clusters_v1.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"artifact": "HYPERSCALPER_FEATURE_REDUNDANCY_CLUSTERS_V1",
|
|
"read_only": True,
|
|
"strategy_execution": "not performed",
|
|
"mode": result["redundancy"]["mode"],
|
|
"clusters": cluster_rows,
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
try:
|
|
import pyarrow as pa
|
|
import pyarrow.parquet as pq
|
|
|
|
pq.write_table(
|
|
pa.Table.from_pylist(feature_output),
|
|
args.output_dir / "hyperscalper_feature_gap_analysis_v1.parquet",
|
|
compression="zstd",
|
|
)
|
|
pq.write_table(
|
|
pa.Table.from_pylist(
|
|
[
|
|
{
|
|
"left_feature_key": rows[left]["feature_key"],
|
|
"right_feature_key": rows[right]["feature_key"],
|
|
"left_output_type": output_types[left],
|
|
"right_output_type": output_types[right],
|
|
"redundancy_metric": (
|
|
"abs_pearson" if output_types[left] == output_types[right] == "continuous" else
|
|
"jaccard_detection" if output_types[left] == output_types[right] == "event" else
|
|
"agreement" if output_types[left] == output_types[right] == "state" else "not_comparable"
|
|
),
|
|
"pair_sample_count": int(pair_samples[left, right]),
|
|
"pearson": (
|
|
float(pearson[left, right])
|
|
if np.isfinite(pearson[left, right])
|
|
else None
|
|
),
|
|
"spearman_ordinal": (
|
|
float(spearman[left, right])
|
|
if np.isfinite(spearman[left, right])
|
|
else None
|
|
),
|
|
"agreement": (
|
|
float(agreement[left, right])
|
|
if np.isfinite(agreement[left, right])
|
|
else None
|
|
),
|
|
"jaccard_detection": (
|
|
float(jaccard[left, right])
|
|
if np.isfinite(jaccard[left, right])
|
|
else None
|
|
),
|
|
"primary_redundancy": (
|
|
float(primary[left, right])
|
|
if np.isfinite(primary[left, right])
|
|
else None
|
|
),
|
|
}
|
|
for left in range(len(rows))
|
|
for right in range(left + 1, len(rows))
|
|
]
|
|
),
|
|
args.output_dir / "hyperscalper_feature_redundancy_map_v1.parquet",
|
|
compression="zstd",
|
|
)
|
|
if args.cohort_deq and deq_evidence["available"]:
|
|
pq.write_table(
|
|
pa.Table.from_pylist(deq_rows),
|
|
args.output_dir / "historical_feature_deq_summary_v1.parquet",
|
|
compression="zstd",
|
|
)
|
|
except ImportError:
|
|
result["parquet"] = "not written: pyarrow unavailable"
|
|
result["redundancy"]["map"]["status"] = "not written: pyarrow unavailable"
|
|
json_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json_path)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|