39 lines
2 KiB
Python
39 lines
2 KiB
Python
"""Compare direct block-0 intermediates with ComfyUI captures."""
|
|
|
|
import argparse
|
|
|
|
import torch
|
|
|
|
from h3_blackwell_runtime.block import gate_segments, modulate_segments
|
|
from h3_blackwell_runtime.checkpoint import H3Checkpoint
|
|
from h3_blackwell_runtime.denoiser import H3PackedDenoiser
|
|
from h3_blackwell_runtime.attention import rms_norm
|
|
from h3_blackwell_runtime.rope import h3_rope_rotation
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--capture-dir", default="/artifacts/capture")
|
|
parser.add_argument("--model", default="/models/minimax_h3_ref2va_pruned_nvfp4.safetensors")
|
|
args = parser.parse_args()
|
|
|
|
capture_dir = args.capture_dir
|
|
inputs = torch.load(f"{capture_dir}/input.pt", map_location="cuda", weights_only=False)
|
|
checkpoint = H3Checkpoint(args.model)
|
|
model = H3PackedDenoiser.from_checkpoint(checkpoint).eval()
|
|
block = model.backbone.blocks[0]
|
|
adaln = model.backbone.adaln[0]
|
|
rotation = h3_rope_rotation(inputs["position_ids"], model.backbone.inv_freq, inputs["hidden"].dtype)
|
|
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = adaln(inputs["timesteps"])
|
|
|
|
with torch.inference_mode():
|
|
norm1 = modulate_segments(rms_norm(inputs["hidden"], block.norm1_weight, block.norm_eps), shift_msa, scale_msa, inputs["segments"])
|
|
attention = block.attention(norm1, rotation)
|
|
post_attention = gate_segments(inputs["hidden"], attention, gate_msa, inputs["segments"])
|
|
norm2 = modulate_segments(rms_norm(post_attention, block.norm2_weight, block.norm_eps), shift_mlp, scale_mlp, inputs["segments"])
|
|
mlp = block.mlp(norm2)
|
|
post_mlp = gate_segments(post_attention, mlp, gate_mlp, inputs["segments"])
|
|
|
|
for name, actual in (("norm1", norm1), ("attention", attention), ("post_attention", post_attention), ("norm2", norm2), ("mlp", mlp), ("post_mlp", post_mlp)):
|
|
expected = torch.load(f"{capture_dir}/block0_{name}.pt", map_location="cuda", weights_only=False)
|
|
delta = (actual.float() - expected.float()).abs()
|
|
print(f"{name} max_abs={delta.max().item():.6g} mean_abs={delta.mean().item():.6g}")
|