Probe loaded Comfy Qwen projections

PARITY.md

@@ -115,6 +115,40 @@ contiguity, dequantization dtype, `pre_quant_scale`, SiLU ordering, or residual
 adds would be speculative and are prohibited until a new code-path difference
 is demonstrated.
 
+## Loaded-Comfy Projection Module Probe
+
+The source audit was followed by a direct replay against tensors consumed and
+produced by the actual loaded Comfy layer-0 modules. This proves the remaining
+difference is present **before attention** and is not inferred from source.
+
+| Item | Evidence |
+| --- | --- |
+| Capture location | `/tmp/fl2va-qwen0-loaded-modules` on Spark |
+| QKV capture SHA-256 | `8998722e73ba24c40c09a86133ccdb22b708998a3e8b8fb4c4293227345f19db` |
+| MLP capture SHA-256 | `e52ed70f4aa76f1ba26302b9955631d8f490fb6dc269d96a0ce3e1ea981e7ac2` |
+| Loaded Comfy class | `comfy.ops.mixed_precision_ops.<locals>.MixedPrecisionOps.Linear` |
+| Loaded Comfy weight | `comfy_kitchen.tensor.base.QuantizedTensor`, logical dtype BF16, `TensorCoreNVFP4Layout`, full precision enabled |
+| Q/K/V dimensions | `(8192,5120)`, `(1024,5120)`, `(1024,5120)`; no pre-scale |
+| Gate/up/down dimensions | `(25600,5120)`, `(25600,5120)`, `(5120,25600)`; only down has pre-scale |
+
+Direct replay on the **identical loaded-Comfy module inputs**:
+
+| Projection | Mean absolute error | Maximum absolute error |
+| --- | ---: | ---: |
+| Q | `6.11124e-05` | `0.0019514` |
+| K | `8.47002e-05` | `0.00167805` |
+| V | `5.4459e-05` | `0.00098893` |
+| Gate | `0.000308973` | `0.0082469` |
+| Up | `0.00028138` | `0.00792789` |
+| Down, using Comfy's captured activated input | `0.000424157` | `0.0500984` |
+
+**Conclusion:** same checkpoint tensors, same BF16 inputs, and the same named
+Kitchen layout do not currently produce identical projection outputs through
+direct `Nvfp4Linear` and Comfy's loaded `MixedPrecisionOps.Linear`. The next
+investigation must capture/compare the effective `QuantizedTensor` parameters
+and direct packed-weight wrapper at dispatch time. Do not revisit attention,
+sampler, or MLP ordering until that exact module-dispatch difference is found.
+
 ## Current Runtime Scope
 
 | Component | Implemented | Known limitation |

tools/compare_qwen0_loaded_modules.py

@@ -0,0 +1,30 @@
+"""Replay direct layer-0 projections from actual loaded-Comfy module inputs."""
+
+import argparse
+
+import torch
+
+from h3_blackwell_runtime.qwen3vl_text import Qwen3VL32BTextEncoder
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--capture-dir", required=True)
+parser.add_argument("--checkpoint", default="/text-encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors")
+args = parser.parse_args()
+
+encoder = Qwen3VL32BTextEncoder(args.checkpoint).eval()
+layer = encoder.layers[0]
+qkv = torch.load(f"{args.capture_dir}/qwen0_loaded_qkv.pt", map_location="cuda", weights_only=False)
+mlp = torch.load(f"{args.capture_dir}/qwen0_loaded_mlp.pt", map_location="cuda", weights_only=False)
+
+with torch.inference_mode():
+    qkv_actual = {"q": layer.q_proj(qkv["input"].to(encoder.dtype)), "k": layer.k_proj(qkv["input"].to(encoder.dtype)), "v": layer.v_proj(qkv["input"].to(encoder.dtype))}
+    gate = layer.gate_proj(mlp["input"].to(encoder.dtype))
+    up = layer.up_proj(mlp["input"].to(encoder.dtype))
+    mlp_actual = {"gate": gate, "up": up, "down": layer.down_proj(mlp["output"]["activated"].to(encoder.dtype))}
+
+for group, actual, captured in (("qkv", qkv_actual, qkv), ("mlp", mlp_actual, mlp)):
+    print(f"{group}.loaded_metadata={captured['metadata']}")
+    for name, value in actual.items():
+        delta = (value.float() - captured["output"][name].float()).abs()
+        print(f"{group}.{name} max_abs={delta.max().item():.6g} mean_abs={delta.mean().item():.6g}")

tools/patch_comfy_qwen_loaded_projection_probe.py

@@ -0,0 +1,36 @@
+"""Capture actual loaded Comfy Qwen layer-0 projection module behavior."""
+
+from pathlib import Path
+
+
+path = Path("/opt/ComfyUI/comfy/text_encoders/llama.py")
+source = path.read_text(encoding="utf-8")
+old = "        xq = self.q_proj(hidden_states)\n        xk = self.k_proj(hidden_states)\n        xv = self.v_proj(hidden_states)\n"
+new = old + (
+    "        capture_dir = os.getenv(\"H3_CAPTURE_DIR\") if getattr(self, \"_h3_trace_index\", -1) == 0 else None\n"
+    "        if capture_dir:\n"
+    "            modules = {\"q\": self.q_proj, \"k\": self.k_proj, \"v\": self.v_proj}\n"
+    "            metadata = {name: {\"class\": f\"{type(module).__module__}.{type(module).__qualname__}\", \"weight_type\": f\"{type(module.weight).__module__}.{type(module.weight).__qualname__}\", \"weight_dtype\": str(module.weight.dtype), \"weight_shape\": tuple(module.weight.shape), \"layout_type\": getattr(module, \"layout_type\", None), \"full_precision\": getattr(module, \"_full_precision_mm\", None), \"pre_quant_scale\": getattr(module, \"pre_quant_scale\", None) is not None} for name, module in modules.items()}\n"
+    "            torch.save({\"input\": hidden_states.detach().cpu(), \"output\": {\"q\": xq.detach().cpu(), \"k\": xk.detach().cpu(), \"v\": xv.detach().cpu()}, \"metadata\": metadata}, os.path.join(capture_dir, \"qwen0_loaded_qkv.pt\"))\n"
+)
+if source.count(old) != 1:
+    raise RuntimeError("Unable to locate Qwen QKV projections.")
+source = source.replace(old, new)
+
+old = "        return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))\n"
+new = (
+    "        gate = self.gate_proj(x)\n"
+    "        up = self.up_proj(x)\n"
+    "        activated = self.activation(gate) * up\n"
+    "        output = self.down_proj(activated)\n"
+    "        capture_dir = os.getenv(\"H3_CAPTURE_DIR\") if getattr(self, \"_h3_trace_index\", -1) == 0 else None\n"
+    "        if capture_dir:\n"
+    "            modules = {\"gate\": self.gate_proj, \"up\": self.up_proj, \"down\": self.down_proj}\n"
+    "            metadata = {name: {\"class\": f\"{type(module).__module__}.{type(module).__qualname__}\", \"weight_type\": f\"{type(module.weight).__module__}.{type(module.weight).__qualname__}\", \"weight_dtype\": str(module.weight.dtype), \"weight_shape\": tuple(module.weight.shape), \"layout_type\": getattr(module, \"layout_type\", None), \"full_precision\": getattr(module, \"_full_precision_mm\", None), \"pre_quant_scale\": getattr(module, \"pre_quant_scale\", None) is not None} for name, module in modules.items()}\n"
+    "            torch.save({\"input\": x.detach().cpu(), \"output\": {\"gate\": gate.detach().cpu(), \"up\": up.detach().cpu(), \"activated\": activated.detach().cpu(), \"down\": output.detach().cpu()}, \"metadata\": metadata}, os.path.join(capture_dir, \"qwen0_loaded_mlp.pt\"))\n"
+    "        return output\n"
+)
+if source.count(old) != 1:
+    raise RuntimeError("Unable to locate Qwen MLP projections.")
+path.write_text(source.replace(old, new), encoding="utf-8")
+print("Applied loaded Qwen layer-0 projection probe.")