Record post-FC2 production profile

Dockerfile.postprofile

@@ -0,0 +1,14 @@
+FROM h3-blackwell-runtime:post-fc2-profiled-a29b896
+
+WORKDIR /opt/h3-blackwell-runtime
+
+# Preserve the profiled binary/ABI layers while making benchmark diagnostics opt-in.
+COPY src/h3_blackwell_runtime/runtime.py src/h3_blackwell_runtime/runtime.py
+COPY src/h3_blackwell_runtime/sampler.py src/h3_blackwell_runtime/sampler.py
+COPY tools/serve_hot_runtime.py tools/serve_hot_runtime.py
+COPY tools/benchmark_hot_runtime.py tools/benchmark_hot_runtime.py
+COPY tools/profile_sampling_stages.py tools/profile_sampling_stages.py
+COPY tools/summarize_nsys_profile.py tools/summarize_nsys_profile.py
+COPY tools/summarize_ncu_profile.py tools/summarize_ncu_profile.py
+COPY tools/build_post_fc2_profile_summary.py tools/build_post_fc2_profile_summary.py
+COPY tests/test_turbo.py tests/test_turbo.py

benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json

@@ -0,0 +1,31 @@
+{
+  "block_index": 24,
+  "hidden_shape": [
+    37810,
+    5376
+  ],
+  "segments": [
+    [
+      0,
+      100,
+      1
+    ],
+    [
+      100,
+      514,
+      2
+    ],
+    [
+      514,
+      37810,
+      0
+    ]
+  ],
+  "fused_elementwise": true,
+  "fused_nvfp4_modulation": true,
+  "fused_nvfp4_swiglu": true,
+  "nvfp4_scale_backend": "vortex",
+  "sage_qkv_layout": "strided_nhd",
+  "module_forward_checksum": 303055616.0,
+  "capture": "one warmed block between cudaProfilerStart/Stop"
+}

benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json

@@ -0,0 +1,219 @@
+{
+  "source_csv": "benchmarks\\gb10-post-fc2-block24-targeted-20260826.csv",
+  "source_traffic_csv": "benchmarks\\gb10-post-fc2-block24-targeted-traffic-20260826.csv",
+  "launch_order_contract": [
+    "qkv",
+    "sage2",
+    "attention_output",
+    "fc1",
+    "fc2"
+  ],
+  "cache_control": "none (warmed/uncontrolled cache, as reported by NCU)",
+  "metrics": {
+    "qkv": {
+      "launch_id": 0,
+      "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+      "grid_size": "(296, 168, 1)",
+      "block_size": "(384, 1, 1)",
+      "duration_ns": 26376032.0,
+      "registers_per_thread": 168.0,
+      "achieved_occupancy_percent": 20.83,
+      "eligible_warps_per_scheduler": 0.24,
+      "issue_active_percent": 18.29,
+      "sm_throughput_percent": 78.99,
+      "tensor_pipe_active_percent": 78.99,
+      "l2_requested_bytes": 40128950176.0,
+      "l2_hit_rate_percent": 89.11,
+      "l2_throughput_percent": 77.1,
+      "memory_throughput_percent": 76.92,
+      "local_spilling_requests": 0.0,
+      "top_scheduler_stalls": [
+        {
+          "reason": "sleeping",
+          "warps_per_issue_active": 3.93
+        },
+        {
+          "reason": "math_pipe_throttle",
+          "warps_per_issue_active": 3.82
+        },
+        {
+          "reason": "wait",
+          "warps_per_issue_active": 3.78
+        },
+        {
+          "reason": "selected",
+          "warps_per_issue_active": 1.0
+        },
+        {
+          "reason": "long_scoreboard",
+          "warps_per_issue_active": 0.61
+        }
+      ]
+    },
+    "sage2": {
+      "launch_id": 1,
+      "kernel_name": "void qk_int_sv_f8_attn_kernel<128, 64, 32, 64, 128, 1, 2, 2, float, 1, __nv_bfloat16, 1, 0, 0, 1, 0, 1>(signed char *, signed char *, signed char *, T11 *, float *, float *, float *, float *, float *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, float)",
+      "grid_size": "(296, 56, 1)",
+      "block_size": "(32, 4, 1)",
+      "duration_ns": 258996000.0,
+      "registers_per_thread": 255.0,
+      "achieved_occupancy_percent": 16.65,
+      "eligible_warps_per_scheduler": 0.46,
+      "issue_active_percent": 36.49,
+      "sm_throughput_percent": 75.51,
+      "tensor_pipe_active_percent": 75.51,
+      "l2_requested_bytes": 161497982464.0,
+      "l2_hit_rate_percent": 98.85,
+      "l2_throughput_percent": 31.59,
+      "memory_throughput_percent": 31.74,
+      "local_spilling_requests": 1458688.0,
+      "top_scheduler_stalls": [
+        {
+          "reason": "wait",
+          "warps_per_issue_active": 2.01
+        },
+        {
+          "reason": "math_pipe_throttle",
+          "warps_per_issue_active": 1.24
+        },
+        {
+          "reason": "selected",
+          "warps_per_issue_active": 1.0
+        },
+        {
+          "reason": "not_selected",
+          "warps_per_issue_active": 0.26
+        },
+        {
+          "reason": "short_scoreboard",
+          "warps_per_issue_active": 0.25
+        }
+      ]
+    },
+    "attention_output": {
+      "launch_id": 2,
+      "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+      "grid_size": "(296, 42, 1)",
+      "block_size": "(384, 1, 1)",
+      "duration_ns": 8481152.0,
+      "registers_per_thread": 168.0,
+      "achieved_occupancy_percent": 20.83,
+      "eligible_warps_per_scheduler": 0.23,
+      "issue_active_percent": 17.51,
+      "sm_throughput_percent": 81.88,
+      "tensor_pipe_active_percent": 81.88,
+      "l2_requested_bytes": 13241354176.0,
+      "l2_hit_rate_percent": 89.89,
+      "l2_throughput_percent": 79.1,
+      "memory_throughput_percent": 79.51,
+      "local_spilling_requests": 0.0,
+      "top_scheduler_stalls": [
+        {
+          "reason": "sleeping",
+          "warps_per_issue_active": 4.08
+        },
+        {
+          "reason": "math_pipe_throttle",
+          "warps_per_issue_active": 3.9
+        },
+        {
+          "reason": "wait",
+          "warps_per_issue_active": 3.85
+        },
+        {
+          "reason": "selected",
+          "warps_per_issue_active": 1.0
+        },
+        {
+          "reason": "long_scoreboard",
+          "warps_per_issue_active": 0.6
+        }
+      ]
+    },
+    "fc1": {
+      "launch_id": 3,
+      "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+      "grid_size": "(296, 224, 1)",
+      "block_size": "(384, 1, 1)",
+      "duration_ns": 36172608.0,
+      "registers_per_thread": 168.0,
+      "achieved_occupancy_percent": 20.83,
+      "eligible_warps_per_scheduler": 0.23,
+      "issue_active_percent": 17.39,
+      "sm_throughput_percent": 76.85,
+      "tensor_pipe_active_percent": 76.85,
+      "l2_requested_bytes": 53505320928.0,
+      "l2_hit_rate_percent": 89.11,
+      "l2_throughput_percent": 74.99,
+      "memory_throughput_percent": 74.85,
+      "local_spilling_requests": 0.0,
+      "top_scheduler_stalls": [
+        {
+          "reason": "sleeping",
+          "warps_per_issue_active": 4.41
+        },
+        {
+          "reason": "math_pipe_throttle",
+          "warps_per_issue_active": 3.82
+        },
+        {
+          "reason": "wait",
+          "warps_per_issue_active": 3.78
+        },
+        {
+          "reason": "selected",
+          "warps_per_issue_active": 1.0
+        },
+        {
+          "reason": "long_scoreboard",
+          "warps_per_issue_active": 0.61
+        }
+      ]
+    },
+    "fc2": {
+      "launch_id": 4,
+      "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+      "grid_size": "(296, 42, 1)",
+      "block_size": "(384, 1, 1)",
+      "duration_ns": 16198976.0,
+      "registers_per_thread": 168.0,
+      "achieved_occupancy_percent": 20.83,
+      "eligible_warps_per_scheduler": 0.23,
+      "issue_active_percent": 17.44,
+      "sm_throughput_percent": 85.88,
+      "tensor_pipe_active_percent": 85.88,
+      "l2_requested_bytes": 26072923840.0,
+      "l2_hit_rate_percent": 91.06,
+      "l2_throughput_percent": 81.58,
+      "memory_throughput_percent": 77.65,
+      "local_spilling_requests": 0.0,
+      "top_scheduler_stalls": [
+        {
+          "reason": "math_pipe_throttle",
+          "warps_per_issue_active": 4.06
+        },
+        {
+          "reason": "wait",
+          "warps_per_issue_active": 3.98
+        },
+        {
+          "reason": "sleeping",
+          "warps_per_issue_active": 3.81
+        },
+        {
+          "reason": "selected",
+          "warps_per_issue_active": 1.0
+        },
+        {
+          "reason": "long_scoreboard",
+          "warps_per_issue_active": 0.6
+        }
+      ]
+    }
+  },
+  "units": {
+    "duration_ns": "ns",
+    "l2_requested_bytes": "lts__t_bytes.sum",
+    "throughput_and_hit_rate": "%"
+  }
+}

benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json

@@ -0,0 +1,31 @@
+{
+  "block_index": 24,
+  "hidden_shape": [
+    37810,
+    5376
+  ],
+  "segments": [
+    [
+      0,
+      100,
+      1
+    ],
+    [
+      100,
+      514,
+      2
+    ],
+    [
+      514,
+      37810,
+      0
+    ]
+  ],
+  "fused_elementwise": true,
+  "fused_nvfp4_modulation": true,
+  "fused_nvfp4_swiglu": true,
+  "nvfp4_scale_backend": "vortex",
+  "sage_qkv_layout": "strided_nhd",
+  "module_forward_checksum": 303055616.0,
+  "capture": "one warmed block between cudaProfilerStart/Stop"
+}

benchmarks/gb10-post-fc2-production-profile-summary-20260826.json

@@ -0,0 +1,640 @@
+{
+  "name": "gb10-post-fc2-production-profile",
+  "measurement_date": "2026-08-26",
+  "source_commit": "a29b8960b0f887c20e74dafa16a24c37d6508b4e",
+  "measurement_overlay": "the measurement image included deferred step events and latent hashes plus an opt-in 100-token synthetic-text request; the final production overlay gates diagnostics to that benchmark request, and model math is unchanged",
+  "image": "sha256:a15d0c09dd8cc82aaf2b564d3da760ea5ab8dec974f73075f7d30ac3a504815c",
+  "device": "NVIDIA GB10",
+  "compute_capability": "SM121",
+  "tools": {
+    "torch": "2.9.1+cu130",
+    "cuda": "13.0",
+    "nsight_systems": "2025.3.2.474-253236389321v0",
+    "nsight_compute": "2025.3.1"
+  },
+  "workload": {
+    "resolution": [
+      1344,
+      768
+    ],
+    "frames": 124,
+    "packed_tokens": 37810,
+    "text_tokens": 100,
+    "steps": 12,
+    "seed": 440420,
+    "attention": "sage2"
+  },
+  "configuration": {
+    "H3_NVFP4_SCALE_BACKEND": "vortex",
+    "H3_NVFP4_SCALE_VERSION": "1",
+    "H3_FUSED_ELEMENTWISE": "1",
+    "H3_NVFP4_MODULATE_FUSION": "1",
+    "H3_NVFP4_SWIGLU_FUSION": "1",
+    "H3_NVFP4_FC2_LT_SPLITK1": "1",
+    "H3_SAGE_QKV_LAYOUT": "strided_nhd"
+  },
+  "authoritative_resident_baseline": {
+    "sampling_seconds": [
+      256.46369375299946,
+      255.44699439899978,
+      255.13481797700024
+    ],
+    "median_sampling_seconds": 255.44699439899978,
+    "per_step_seconds": [
+      [
+        {
+          "step": 1,
+          "seconds": 21.337705078125
+        },
+        {
+          "step": 2,
+          "seconds": 21.40503515625
+        },
+        {
+          "step": 3,
+          "seconds": 21.370228515625
+        },
+        {
+          "step": 4,
+          "seconds": 21.38410546875
+        },
+        {
+          "step": 5,
+          "seconds": 21.38205078125
+        },
+        {
+          "step": 6,
+          "seconds": 21.373046875
+        },
+        {
+          "step": 7,
+          "seconds": 21.376810546875
+        },
+        {
+          "step": 8,
+          "seconds": 21.367021484375
+        },
+        {
+          "step": 9,
+          "seconds": 21.386291015625
+        },
+        {
+          "step": 10,
+          "seconds": 21.36256640625
+        },
+        {
+          "step": 11,
+          "seconds": 21.404072265625
+        },
+        {
+          "step": 12,
+          "seconds": 21.313634765625
+        }
+      ],
+      [
+        {
+          "step": 1,
+          "seconds": 21.696716796875
+        },
+        {
+          "step": 2,
+          "seconds": 21.326416015625
+        },
+        {
+          "step": 3,
+          "seconds": 21.3040703125
+        },
+        {
+          "step": 4,
+          "seconds": 21.332494140625
+        },
+        {
+          "step": 5,
+          "seconds": 21.26598046875
+        },
+        {
+          "step": 6,
+          "seconds": 21.267458984375
+        },
+        {
+          "step": 7,
+          "seconds": 21.25258203125
+        },
+        {
+          "step": 8,
+          "seconds": 21.21969140625
+        },
+        {
+          "step": 9,
+          "seconds": 21.242830078125
+        },
+        {
+          "step": 10,
+          "seconds": 21.20158203125
+        },
+        {
+          "step": 11,
+          "seconds": 21.193841796875
+        },
+        {
+          "step": 12,
+          "seconds": 21.141802734375
+        }
+      ],
+      [
+        {
+          "step": 1,
+          "seconds": 21.116169921875
+        },
+        {
+          "step": 2,
+          "seconds": 21.237697265625
+        },
+        {
+          "step": 3,
+          "seconds": 21.2424921875
+        },
+        {
+          "step": 4,
+          "seconds": 21.273701171875
+        },
+        {
+          "step": 5,
+          "seconds": 21.275826171875
+        },
+        {
+          "step": 6,
+          "seconds": 21.303318359375
+        },
+        {
+          "step": 7,
+          "seconds": 21.3065546875
+        },
+        {
+          "step": 8,
+          "seconds": 21.32540234375
+        },
+        {
+          "step": 9,
+          "seconds": 21.35030078125
+        },
+        {
+          "step": 10,
+          "seconds": 21.258994140625
+        },
+        {
+          "step": 11,
+          "seconds": 21.249765625
+        },
+        {
+          "step": 12,
+          "seconds": 21.193939453125
+        }
+      ]
+    ],
+    "peak_allocated_bytes": [
+      44445830144,
+      44445830144,
+      44445830144
+    ],
+    "peak_reserved_bytes": [
+      48708452352,
+      48708452352,
+      48708452352
+    ],
+    "latent_checksums": {
+      "video_sha256": "c62d23a42972eab907ba42f93c50247ff17a9c454b4a53fe93d2e34f9fefe578",
+      "audio_sha256": "852005383770480a6503504e1ffec86dd1fb63a69c6400f92da18e39e0986de2",
+      "exact_across_measured_runs": true
+    },
+    "fc2_dispatch": {
+      "required_per_run": 600,
+      "runs": [
+        {
+          "attempts": 600,
+          "successes": 600,
+          "fallbacks": 0
+        },
+        {
+          "attempts": 600,
+          "successes": 600,
+          "fallbacks": 0
+        },
+        {
+          "attempts": 600,
+          "successes": 600,
+          "fallbacks": 0
+        }
+      ],
+      "all_passed": true
+    },
+    "measurement_policy": {
+      "canonical_warmup_runs": 1,
+      "measured_runs": 3,
+      "authoritative_timing": "median synchronized resident sampling_seconds",
+      "step_timing": "deferred CUDA event elapsed time; no per-step synchronization",
+      "profiling": false
+    }
+  },
+  "block_24": {
+    "uninstrumented_module_p50_ms": 427.4100805005219,
+    "uninstrumented_module_samples": 50,
+    "synchronized_decomposition_is_attribution_only": true,
+    "fc2_schedule_note": "profile_h3_block.py decomposes generic NVFP4 linear calls and bypasses forward_swiglu/guarded FC2; use resident NSYS and targeted NCU for production FC2"
+  },
+  "one_warmed_sampling_step_nsys": {
+    "elapsed_seconds": 20.90812512299999,
+    "gpu_span_seconds": 20.90717264,
+    "kernel_time_seconds": 20.895065376,
+    "gpu_operation_count": 2803,
+    "kernel_count": 2744,
+    "kernel_busy_percent_of_span": 99.94209038109325,
+    "launch_gaps": {
+      "positive_gap_count": 2743,
+      "total_ns": 11518912,
+      "average_ns": 4199.384615384615,
+      "maximum_ns": 4797728
+    },
+    "cpu_gpu_overlap": {
+      "scope": "CUDA kernel-launch API intervals intersected with GPU kernel intervals",
+      "launch_api_union_ns": 13065552528,
+      "launch_api_gpu_overlap_ns": 13062771136,
+      "launch_api_overlap_percent": 99.97871202160002
+    },
+    "components": {
+      "sage2": {
+        "milliseconds": 13029.23856,
+        "percent_of_kernel_time": 62.355576905566124,
+        "launches": 250
+      },
+      "nvfp4_gemms": {
+        "milliseconds": 4167.855808,
+        "percent_of_kernel_time": 19.946603339117498,
+        "launches": 200
+      },
+      "nvfp4_packing": {
+        "milliseconds": 2026.853664,
+        "percent_of_kernel_time": 9.700154689767265,
+        "launches": 800
+      },
+      "norm_and_rope": {
+        "milliseconds": 1084.3856,
+        "percent_of_kernel_time": 5.1896731619970025,
+        "launches": 202
+      },
+      "remaining_gate_add": {
+        "milliseconds": 549.298592,
+        "percent_of_kernel_time": 2.6288436150619683,
+        "launches": 100
+      },
+      "other": {
+        "milliseconds": 37.433152,
+        "percent_of_kernel_time": 0.17914828849014078,
+        "launches": 1192
+      }
+    }
+  },
+  "block_24_targeted_ncu": {
+    "source_csv": "benchmarks\\gb10-post-fc2-block24-targeted-20260826.csv",
+    "source_traffic_csv": "benchmarks\\gb10-post-fc2-block24-targeted-traffic-20260826.csv",
+    "launch_order_contract": [
+      "qkv",
+      "sage2",
+      "attention_output",
+      "fc1",
+      "fc2"
+    ],
+    "cache_control": "none (warmed/uncontrolled cache, as reported by NCU)",
+    "metrics": {
+      "qkv": {
+        "launch_id": 0,
+        "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+        "grid_size": "(296, 168, 1)",
+        "block_size": "(384, 1, 1)",
+        "duration_ns": 26376032.0,
+        "registers_per_thread": 168.0,
+        "achieved_occupancy_percent": 20.83,
+        "eligible_warps_per_scheduler": 0.24,
+        "issue_active_percent": 18.29,
+        "sm_throughput_percent": 78.99,
+        "tensor_pipe_active_percent": 78.99,
+        "l2_requested_bytes": 40128950176.0,
+        "l2_hit_rate_percent": 89.11,
+        "l2_throughput_percent": 77.1,
+        "memory_throughput_percent": 76.92,
+        "local_spilling_requests": 0.0,
+        "top_scheduler_stalls": [
+          {
+            "reason": "sleeping",
+            "warps_per_issue_active": 3.93
+          },
+          {
+            "reason": "math_pipe_throttle",
+            "warps_per_issue_active": 3.82
+          },
+          {
+            "reason": "wait",
+            "warps_per_issue_active": 3.78
+          },
+          {
+            "reason": "selected",
+            "warps_per_issue_active": 1.0
+          },
+          {
+            "reason": "long_scoreboard",
+            "warps_per_issue_active": 0.61
+          }
+        ]
+      },
+      "sage2": {
+        "launch_id": 1,
+        "kernel_name": "void qk_int_sv_f8_attn_kernel<128, 64, 32, 64, 128, 1, 2, 2, float, 1, __nv_bfloat16, 1, 0, 0, 1, 0, 1>(signed char *, signed char *, signed char *, T11 *, float *, float *, float *, float *, float *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, float)",
+        "grid_size": "(296, 56, 1)",
+        "block_size": "(32, 4, 1)",
+        "duration_ns": 258996000.0,
+        "registers_per_thread": 255.0,
+        "achieved_occupancy_percent": 16.65,
+        "eligible_warps_per_scheduler": 0.46,
+        "issue_active_percent": 36.49,
+        "sm_throughput_percent": 75.51,
+        "tensor_pipe_active_percent": 75.51,
+        "l2_requested_bytes": 161497982464.0,
+        "l2_hit_rate_percent": 98.85,
+        "l2_throughput_percent": 31.59,
+        "memory_throughput_percent": 31.74,
+        "local_spilling_requests": 1458688.0,
+        "top_scheduler_stalls": [
+          {
+            "reason": "wait",
+            "warps_per_issue_active": 2.01
+          },
+          {
+            "reason": "math_pipe_throttle",
+            "warps_per_issue_active": 1.24
+          },
+          {
+            "reason": "selected",
+            "warps_per_issue_active": 1.0
+          },
+          {
+            "reason": "not_selected",
+            "warps_per_issue_active": 0.26
+          },
+          {
+            "reason": "short_scoreboard",
+            "warps_per_issue_active": 0.25
+          }
+        ]
+      },
+      "attention_output": {
+        "launch_id": 2,
+        "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+        "grid_size": "(296, 42, 1)",
+        "block_size": "(384, 1, 1)",
+        "duration_ns": 8481152.0,
+        "registers_per_thread": 168.0,
+        "achieved_occupancy_percent": 20.83,
+        "eligible_warps_per_scheduler": 0.23,
+        "issue_active_percent": 17.51,
+        "sm_throughput_percent": 81.88,
+        "tensor_pipe_active_percent": 81.88,
+        "l2_requested_bytes": 13241354176.0,
+        "l2_hit_rate_percent": 89.89,
+        "l2_throughput_percent": 79.1,
+        "memory_throughput_percent": 79.51,
+        "local_spilling_requests": 0.0,
+        "top_scheduler_stalls": [
+          {
+            "reason": "sleeping",
+            "warps_per_issue_active": 4.08
+          },
+          {
+            "reason": "math_pipe_throttle",
+            "warps_per_issue_active": 3.9
+          },
+          {
+            "reason": "wait",
+            "warps_per_issue_active": 3.85
+          },
+          {
+            "reason": "selected",
+            "warps_per_issue_active": 1.0
+          },
+          {
+            "reason": "long_scoreboard",
+            "warps_per_issue_active": 0.6
+          }
+        ]
+      },
+      "fc1": {
+        "launch_id": 3,
+        "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+        "grid_size": "(296, 224, 1)",
+        "block_size": "(384, 1, 1)",
+        "duration_ns": 36172608.0,
+        "registers_per_thread": 168.0,
+        "achieved_occupancy_percent": 20.83,
+        "eligible_warps_per_scheduler": 0.23,
+        "issue_active_percent": 17.39,
+        "sm_throughput_percent": 76.85,
+        "tensor_pipe_active_percent": 76.85,
+        "l2_requested_bytes": 53505320928.0,
+        "l2_hit_rate_percent": 89.11,
+        "l2_throughput_percent": 74.99,
+        "memory_throughput_percent": 74.85,
+        "local_spilling_requests": 0.0,
+        "top_scheduler_stalls": [
+          {
+            "reason": "sleeping",
+            "warps_per_issue_active": 4.41
+          },
+          {
+            "reason": "math_pipe_throttle",
+            "warps_per_issue_active": 3.82
+          },
+          {
+            "reason": "wait",
+            "warps_per_issue_active": 3.78
+          },
+          {
+            "reason": "selected",
+            "warps_per_issue_active": 1.0
+          },
+          {
+            "reason": "long_scoreboard",
+            "warps_per_issue_active": 0.61
+          }
+        ]
+      },
+      "fc2": {
+        "launch_id": 4,
+        "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu",
+        "grid_size": "(296, 42, 1)",
+        "block_size": "(384, 1, 1)",
+        "duration_ns": 16198976.0,
+        "registers_per_thread": 168.0,
+        "achieved_occupancy_percent": 20.83,
+        "eligible_warps_per_scheduler": 0.23,
+        "issue_active_percent": 17.44,
+        "sm_throughput_percent": 85.88,
+        "tensor_pipe_active_percent": 85.88,
+        "l2_requested_bytes": 26072923840.0,
+        "l2_hit_rate_percent": 91.06,
+        "l2_throughput_percent": 81.58,
+        "memory_throughput_percent": 77.65,
+        "local_spilling_requests": 0.0,
+        "top_scheduler_stalls": [
+          {
+            "reason": "math_pipe_throttle",
+            "warps_per_issue_active": 4.06
+          },
+          {
+            "reason": "wait",
+            "warps_per_issue_active": 3.98
+          },
+          {
+            "reason": "sleeping",
+            "warps_per_issue_active": 3.81
+          },
+          {
+            "reason": "selected",
+            "warps_per_issue_active": 1.0
+          },
+          {
+            "reason": "long_scoreboard",
+            "warps_per_issue_active": 0.6
+          }
+        ]
+      }
+    },
+    "units": {
+      "duration_ns": "ns",
+      "l2_requested_bytes": "lts__t_bytes.sum",
+      "throughput_and_hit_rate": "%"
+    }
+  },
+  "comparison_to_pre_fc2_profile": {
+    "previous_summary": "benchmarks/gb10-fully-fused-fresh-nsight-summary.json",
+    "block_24_p50_ms": {
+      "previous": 458.7751985236537,
+      "current": 427.4100805005219,
+      "absolute_change": -31.365118023131856,
+      "percent_change": -6.83670741663136
+    },
+    "block_24_comparison_note": "same-script decomposition control only; it excludes guarded FC2 and must not be interpreted as the production FC2 gain",
+    "warmed_step_gpu_span_seconds": {
+      "previous": 23.706977664,
+      "current": 20.90717264,
+      "absolute_change": -2.799805024000001,
+      "percent_change": -11.810046239051452
+    },
+    "warmed_step_kernel_time_seconds": {
+      "previous": 23.693982688,
+      "current": 20.895065376,
+      "absolute_change": -2.798917311999997,
+      "percent_change": -11.812776893002164
+    },
+    "warmed_step_kernel_count": {
+      "previous": 2694,
+      "current": 2744,
+      "absolute_change": 50,
+      "percent_change": 1.8559762435040872
+    },
+    "components": {
+      "sage2": {
+        "previous": 13760.566304,
+        "current": 13029.23856,
+        "absolute_change": -731.3277440000002,
+        "percent_change": -5.314663131178065
+      },
+      "nvfp4_gemms": {
+        "previous": 6269.804512,
+        "current": 4167.855808,
+        "absolute_change": -2101.9487039999995,
+        "percent_change": -33.52494802632212
+      },
+      "nvfp4_packing": {
+        "previous": 2093.052672,
+        "current": 2026.853664,
+        "absolute_change": -66.19900799999982,
+        "percent_change": -3.1627970421185814
+      },
+      "norm_and_rope": {
+        "previous": 987.609696,
+        "current": 1084.3856,
+        "absolute_change": 96.77590400000008,
+        "percent_change": 9.79900302639396
+      },
+      "remaining_gate_add": {
+        "previous": 545.150976,
+        "current": 549.298592,
+        "absolute_change": 4.147615999999971,
+        "percent_change": 0.7608196963037273
+      },
+      "other": {
+        "previous": 37.798528,
+        "current": 37.433152,
+        "absolute_change": -0.3653759999999977,
+        "percent_change": -0.966640817335529
+      }
+    }
+  },
+  "bottleneck_ranking": [
+    {
+      "component": "sage2",
+      "milliseconds": 13029.23856,
+      "percent_of_kernel_time": 62.355576905566124
+    },
+    {
+      "component": "nvfp4_gemms",
+      "milliseconds": 4167.855808,
+      "percent_of_kernel_time": 19.946603339117498
+    },
+    {
+      "component": "nvfp4_packing",
+      "milliseconds": 2026.853664,
+      "percent_of_kernel_time": 9.700154689767265
+    },
+    {
+      "component": "norm_and_rope",
+      "milliseconds": 1084.3856,
+      "percent_of_kernel_time": 5.1896731619970025
+    },
+    {
+      "component": "remaining_gate_add",
+      "milliseconds": 549.298592,
+      "percent_of_kernel_time": 2.6288436150619683
+    },
+    {
+      "component": "other",
+      "milliseconds": 37.433152,
+      "percent_of_kernel_time": 0.17914828849014078
+    }
+  ],
+  "decision": {
+    "authoritative_exact_baseline": "255.446994 s median resident sampling",
+    "time_bottleneck": "Sage2 remains dominant at 62.36% of warmed-step kernel time; its mainloop is 241.91 ms average in NSYS and 259.00 ms in the NCU replay.",
+    "secondary_bottlenecks": "NVFP4 GEMMs are 19.95%, packing 9.70%, norm/RoPE 5.19%, and gate/add 2.63%.",
+    "next_optimization": "None started. Sage2 is the next-ranked investigation target; any implementation requires a separate approved experiment after this baseline is accepted."
+  },
+  "artifacts": [
+    "benchmarks/gb10-post-fc2-resident-baseline-20260826.json",
+    "benchmarks/gb10-post-fc2-block24-profile-20260826.json",
+    "benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json",
+    "benchmarks/gb10-post-fc2-warmed-step-20260826.nsys-rep",
+    "benchmarks/gb10-post-fc2-warmed-step-20260826.sqlite",
+    "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv",
+    "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv",
+    "benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv",
+    "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv",
+    "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv",
+    "benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json",
+    "benchmarks/gb10-post-fc2-block24-targeted-20260826.ncu-rep",
+    "benchmarks/gb10-post-fc2-block24-targeted-20260826.csv",
+    "benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json",
+    "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep",
+    "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.csv",
+    "benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json",
+    "benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json"
+  ]
+}

benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json

@@ -0,0 +1,67 @@
+{
+  "device": "NVIDIA GB10",
+  "torch": "2.9.1+cu130",
+  "attention": "sage2",
+  "resolution": [
+    1344,
+    768
+  ],
+  "frames": 124,
+  "steps": 1,
+  "seed": 440420,
+  "text_tokens": 100,
+  "warmup_runs": 1,
+  "cuda_profiler_capture": true,
+  "argv": [
+    "tools/profile_sampling_stages.py",
+    "--attention",
+    "sage2",
+    "--steps",
+    "1",
+    "--warmup-runs",
+    "1",
+    "--uninstrumented",
+    "--cuda-profiler-capture",
+    "--output",
+    "/output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json"
+  ],
+  "environment_switches": {
+    "CUDA_DEVICE_MAX_CONNECTIONS": "1",
+    "CUDA_DEVICE_MAX_COPY_CONNECTIONS": "4",
+    "CUDA_HOME": "/usr/local/cuda",
+    "CUDA_INC_PATH": "/usr/local/cuda/include",
+    "CUDA_INJECTION64_PATH": "/opt/nsys/target-linux-sbsa-armv8/libToolsInjection64.so",
+    "CUDA_MANAGED_FORCE_DEVICE_ALLOC": "1",
+    "CUDA_MODULE_LOADING": "EAGER",
+    "CUDA_VERSION": "13.0.2",
+    "H3_FUSED_ELEMENTWISE": "1",
+    "H3_MODEL_PATH": "/models/minimax_h3_ref2va_pruned_nvfp4.safetensors",
+    "H3_NVFP4_FC2_LT_SPLITK1": "1",
+    "H3_NVFP4_MODULATE_FUSION": "1",
+    "H3_NVFP4_SCALE_BACKEND": "vortex",
+    "H3_NVFP4_SCALE_VERSION": "1",
+    "H3_NVFP4_SWIGLU_FUSION": "1",
+    "H3_SAGE_QKV_LAYOUT": "strided_nhd",
+    "TORCH_COMPILE_DISABLE": "0",
+    "TORCH_CUDA_ARCH_LIST": "12.1a",
+    "TORCH_EXTENSIONS_DIR": "/opt/h3-blackwell-runtime/.torch_extensions"
+  },
+  "elapsed_seconds": 20.90812512299999,
+  "stage_trace": [],
+  "checksums": [
+    -197605.3125,
+    528.5980224609375
+  ],
+  "sha256": {
+    "video": "b5fc9bd43ff65f189797fd2377d5a48d8d57f64fb2774fb729dc2be478c39a4b",
+    "audio": "de6da2540b639a821be7d069efcbf067b209b20de3f7daa0bb9ee28e2624764c"
+  },
+  "fc2_dispatch_delta": {
+    "attempts": 50,
+    "successes": 50,
+    "fallbacks": 0
+  },
+  "peak_allocated_bytes": 15570401280,
+  "peak_reserved_bytes": 18538823680,
+  "measurement_policy": "uninstrumented sampling wall time"
+}

benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json

@@ -0,0 +1,61 @@
+{
+  "source_gpu_trace": "benchmarks\\gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv",
+  "source_kernel_exec_trace": "benchmarks\\gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv",
+  "gpu_span_ns": 20907172640,
+  "gpu_operation_count": 2803,
+  "kernel_count": 2744,
+  "kernel_time_ns": 20895065376,
+  "kernel_busy_percent_of_span": 99.94209038109325,
+  "launch_gaps": {
+    "positive_gap_count": 2743,
+    "total_ns": 11518912,
+    "average_ns": 4199.384615384615,
+    "maximum_ns": 4797728
+  },
+  "cpu_gpu_overlap": {
+    "scope": "CUDA kernel-launch API intervals intersected with GPU kernel intervals",
+    "launch_api_union_ns": 13065552528,
+    "launch_api_gpu_overlap_ns": 13062771136,
+    "launch_api_overlap_percent": 99.97871202160002
+  },
+  "components": {
+    "sage2": {
+      "milliseconds": 13029.23856,
+      "percent_of_kernel_time": 62.355576905566124,
+      "launches": 250
+    },
+    "nvfp4_gemms": {
+      "milliseconds": 4167.855808,
+      "percent_of_kernel_time": 19.946603339117498,
+      "launches": 200
+    },
+    "nvfp4_packing": {
+      "milliseconds": 2026.853664,
+      "percent_of_kernel_time": 9.700154689767265,
+      "launches": 800
+    },
+    "norm_and_rope": {
+      "milliseconds": 1084.3856,
+      "percent_of_kernel_time": 5.1896731619970025,
+      "launches": 202
+    },
+    "remaining_gate_add": {
+      "milliseconds": 549.298592,
+      "percent_of_kernel_time": 2.6288436150619683,
+      "launches": 100
+    },
+    "other": {
+      "milliseconds": 37.433152,
+      "percent_of_kernel_time": 0.17914828849014078,
+      "launches": 1192
+    }
+  },
+  "classification_policy": {
+    "sage2": "mainloop plus MeanScale/TransposePadPermute/QuantInt8 preparation",
+    "nvfp4_gemms": "SM120 block-scaled CUTLASS GEMMs",
+    "nvfp4_packing": "absmax, final-scale, zero-fill and NVFP4 quantization kernels",
+    "norm_and_rope": "layer norm, mean reduction and fused RMS/RoPE kernels",
+    "remaining_gate_add": "fused residual gate/add kernels",
+    "other": "all unmatched kernels"
+  }
+}

benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv

@@ -0,0 +1,11 @@
+Time (%),Total Time (ns),Num Calls,Avg (ns),Med (ns),Min (ns),Max (ns),StdDev (ns),Name
+60.5,12620672144,2441,5170287.6,4016.0,2384,243250816,27751294.3,cudaLaunchKernel
+36.5,7617076416,7,1088153773.7,7616.0,1712,7607043680,2874562566.0,cudaStreamSynchronize
+2.1,441237232,300,1470790.8,3744.0,2448,10733504,3261468.3,cuLaunchKernelEx
+0.8,163999584,52,3153838.2,5079656.0,128,5558272,2619562.7,cudaMemsetAsync
+0.0,3643152,3,1214384.0,5552.0,5456,3632144,2093841.6,cuLaunchKernel
+0.0,922240,1,922240.0,922240.0,922240,922240,0.0,cuProfilerStart
+0.0,280112,9,31123.6,31344.0,2848,58160,22382.3,cudaMemcpyAsync
+0.0,204368,2441,83.7,48.0,16,6592,205.4,cuKernelGetName
+0.0,10128,3,3376.0,2016.0,1648,6464,2680.6,cuKernelGetFunction
+0.0,5456,1,5456.0,5456.0,5456,5456,0.0,cudaDeviceSynchronize

benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv

@@ -0,0 +1,63 @@
+Time (%),Total Time (ns),Instances,Avg (ns),Med (ns),Min (ns),Max (ns),StdDev (ns),Name
+57.9,12095370432,50,241907408.6,241960816.0,238904576,245704832,1430443.8,"void qk_int_sv_f8_attn_kernel<(unsigned int)128, (unsigned int)64, (unsigned int)32, (unsigned int)64, (unsigned int)128, (DataType)1, (QuantGranularity)2, (QuantGranularity)2, float, (bool)1, __nv_bfloat16, (ComputeUnit)1, (MaskMode)0, (bool)0, (bool)1, (bool)0, (bool)1>(signed char *, signed char *, signed char *, T11 *, float *, float *, float *, float *, float *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, float)"
+19.9,4167855808,200,20839279.0,20371696.0,7702240,35071936,9954629.7,cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu
+3.0,616933984,50,12338679.7,12314672.0,11865984,12928448,384444.2,"<unnamed>::partial_absmax_swiglu_bf16_kernel(const unsigned short *, float *, long, long)"
+2.9,606392320,50,12127846.4,12134352.0,11797920,12481600,272664.4,"void comfy::<unnamed>::rope_kernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1, (bool)1, (bool)1, (bool)0>(const T1 *, const T1 *, const T2 *, const T3 *, const T3 *, T1 *, T1 *, long, long, long, int, int, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, float)"
+2.6,549298592,100,5492985.9,5476528.0,5449216,5824160,55130.6,_gate_add_kernel
+2.5,528098848,50,10561977.0,10468928.0,10341376,11079968,201841.8,"<unnamed>::quantize_nvfp4_swiglu_bf16_kernel(const unsigned short *, const float *, unsigned char *, unsigned char *, long, long, long, long, long)"
+1.8,371120608,100,3711206.1,3518080.0,3408320,4344608,319034.1,"void <unnamed>::partial_absmax_modulated_bf16_kernel<float>(const unsigned short *, const T1 *, const T1 *, const int *, float *, long, long)"
+1.7,358825984,102,3517901.8,3507296.0,45056,3844544,361846.1,"void at::native::<unnamed>::vectorized_layer_norm_kernel<c10::BFloat16, float, (bool)1>(int, T2, const T1 *, const T1 *, const T1 *, T2 *, T2 *, T1 *)"
+1.4,286555520,50,5731110.4,5711552.0,5660672,5912992,64887.3,"void MeanScaleKernel<(unsigned int)64, (bool)0, __nv_bfloat16>(T3 *, signed char *, float *, float *, float, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int)"
+1.3,264825568,50,5296511.4,5240688.0,5065568,5700544,172507.0,"void TransposePadPermuteKernel<(unsigned int)128, (unsigned int)64, (bool)1, __nv_bfloat16>(T4 *, T4 *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int)"
+1.2,248751328,100,2487513.3,2438752.0,2417696,2845728,108528.2,"void <unnamed>::quantize_nvfp4_modulated_bf16_kernel<float>(const unsigned short *, const T1 *, const T1 *, const int *, const float *, unsigned char *, unsigned char *, long, long, long, long, long, long)"
+0.9,192832480,50,3856649.6,3831584.0,3724512,4098912,116494.0,"void QuantInt8Kernel<(unsigned int)128, (unsigned int)64, (unsigned int)1, (bool)0, (bool)1, __nv_bfloat16>(T6 *, T6 *, signed char *, float *, float, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int)"
+0.9,189654560,50,3793091.2,3766976.0,3707360,4001984,83318.4,"void QuantInt8Kernel<(unsigned int)128, (unsigned int)32, (unsigned int)1, (bool)0, (bool)0, __nv_bfloat16>(T6 *, T6 *, signed char *, float *, float, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int)"
+0.7,138351232,50,2767024.6,2761616.0,2747456,2899360,25808.7,"void comfy::<unnamed>::quantize_nvfp4_kernel<__nv_bfloat16, __nv_fp4x2_e2m1, __nv_fp8_e4m3, (bool)1, (bool)1>(const T1 *, const float *, T2 *, T3 *, unsigned long, unsigned long, unsigned long, unsigned long, float)"
+0.6,119167296,50,2383345.9,2358464.0,2338976,2551328,51731.7,"void at::native::reduce_kernel<(int)128, (int)4, at::native::ReduceOp<c10::BFloat16, at::native::MeanOps<c10::BFloat16, float, float, c10::BFloat16>, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)"
+0.5,108606272,50,2172125.4,2172144.0,2156896,2190848,8057.6,"<unnamed>::partial_absmax_bf16_vec_kernel(const uint4 *, const unsigned short *, float *, long, long)"
+0.1,14493152,200,72465.8,48272.0,40608,206816,42011.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor<unsigned char>, std::array<char *, (unsigned long)1>>(int, T2, T3)"
+0.0,6883840,2,3441920.0,3441920.0,61632,6822208,4780449.1,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast<at::native::CUDAFunctor_add<float>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,5532736,60,92212.3,2912.0,1280,5035136,649588.6,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::bfloat16_copy_kernel_cuda(at::TensorIteratorBase &)::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,5508544,2,2754272.0,2754272.0,51296,5457248,3822585.3,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float>>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,4915936,1,4915936.0,4915936.0,4915936,4915936,0.0,void cutlass::Kernel2<cutlass_80_simt_sgemm_256x128_8x4_tn_align1>(T1::Params)
+0.0,4410976,1,4410976.0,4410976.0,4410976,4410976,0.0,"void magma_sgemmEx_kernel<float, float, float, (bool)1, (bool)0, (int)6, (int)4, (int)6, (int)3, (int)4>(int, int, int, Tensor, int, Tensor, int, Tensor, int, Tensor, int, int, int, const T1 *, const T1 *, T1, T1, int, cublasLtEpilogue_t, int, const void *, long)"
+0.0,3766816,1,3766816.0,3766816.0,3766816,3766816,0.0,"void at::native::<unnamed>::CatArrayBatchedCopy_alignedK_contig<at::native::<unnamed>::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)8>(T1 *, at::native::<unnamed>::CatArrInputTensorMetadata<T1, T2, T4, T5>, at::native::<unnamed>::TensorSizeStride<T2, (unsigned int)4>, int, T2)"
+0.0,1542400,51,30243.1,26560.0,4640,93376,13266.2,"std::enable_if<!T7, void>::type internal::gemvx::kernel<int, int, float, float, float, float, (bool)0, (bool)1, (bool)1, (bool)0, (int)6, (bool)0, cublasGemvParamsEx<int, cublasGemvTensorStridedBatched<const float>, cublasGemvTensorStridedBatched<const float>, cublasGemvTensorStridedBatched<float>, float>>(T13)"
+0.0,873696,1,873696.0,873696.0,873696,873696,0.0,"void at::native::<unnamed>::CatArrayBatchedCopy_alignedK_contig<at::native::<unnamed>::OpaqueType<(unsigned int)4>, unsigned int, (int)3, (int)128, (int)1, (int)16>(T1 *, at::native::<unnamed>::CatArrInputTensorMetadata<T1, T2, T4, T5>, at::native::<unnamed>::TensorSizeStride<T2, (unsigned int)4>, int, T2)"
+0.0,730688,201,3635.3,2336.0,1920,100448,7972.3,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 3)]::operator ()() const::[lambda() (instance 7)]::operator ()() const::[lambda(float) (instance 1)]>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,501280,6,83546.7,81136.0,1600,172960,89454.6,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add<float>, std::array<char *, (unsigned long)3>>(int, T2, T3)"
+0.0,405472,200,2027.4,1920.0,1824,7584,687.4,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float>>, std::array<char *, (unsigned long)3>>(int, T2, T3)"
+0.0,400288,150,2668.6,2176.0,1920,9696,1031.2,"<unnamed>::final_scale_bf16_compat_kernel(const float *, float *, long, float)"
+0.0,358944,102,3519.1,4304.0,1600,11488,2037.3,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel<void at::native::index_kernel_impl<at::native::OpaqueType<(int)4>>(at::TensorIteratorBase &, c10::ArrayRef<long>, c10::ArrayRef<long>)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef<long>, c10::ArrayRef<long>, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)"
+0.0,224736,55,4086.1,1536.0,1504,71520,12382.0,"void at::native::unrolled_elementwise_kernel<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 3)]::operator ()() const::[lambda() (instance 7)]::operator ()() const::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)"
+0.0,218368,5,43673.6,27680.0,2240,88064,35553.0,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float>>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,217312,51,4261.0,3456.0,3040,22080,3140.9,"void at::native::unrolled_elementwise_kernel<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 3)]::operator ()() const::[lambda() (instance 4)]::operator ()() const::[lambda(long) (instance 1)], std::array<char *, (unsigned long)2>, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)"
+0.0,126112,2,63056.0,63056.0,39712,86400,33013.4,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::neg_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 2)]::operator ()() const::[lambda() (instance 7)]::operator ()() const::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,114816,51,2251.3,1504.0,1472,14048,2514.1,"void at::native::unrolled_elementwise_kernel<at::native::CUDAFunctor_add<float>, std::array<char *, (unsigned long)3>, (int)4, TrivialOffsetCalculator<(int)2, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)2>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)"
+0.0,108896,52,2094.2,1952.0,1920,3904,385.3,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::<unnamed>::launch_clamp_scalar(at::TensorIteratorBase &, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::[lambda() (instance 1)]::operator ()() const::[lambda() (instance 7)]::operator ()() const::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,104288,51,2044.9,1312.0,1280,3744,1021.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::floor_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 1)]::operator ()() const::[lambda() (instance 2)]::operator ()() const::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,99968,1,99968.0,99968.0,99968,99968,0.0,void cutlass::Kernel2<cutlass_80_simt_sgemm_128x32_8x5_tn_align1>(T1::Params)
+0.0,97952,50,1959.0,1952.0,1920,2048,29.1,"<unnamed>::final_scale_warp_kernel(const float *, float *, long, float)"
+0.0,89600,50,1792.0,896.0,832,8672,1561.9,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor<float>, std::array<char *, (unsigned long)1>>(int, T2, T3)"
+0.0,87776,2,43888.0,43888.0,3904,83872,56545.9,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<float, float, float, at::native::binary_internal::DivFunctor<float>>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,87744,60,1462.4,1440.0,1088,3104,258.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float>>, std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,83392,51,1635.1,1280.0,1216,8192,1333.7,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast<at::native::<unnamed>::lerp_tensor_kernel(at::TensorIteratorBase &)::[lambda() (instance 2)]::operator ()() const::[lambda() (instance 2)]::operator ()() const::[lambda(float, float, float) (instance 1)]>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,78720,51,1543.5,1344.0,1312,6144,751.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::<unnamed>::launch_clamp_scalar(at::TensorIteratorBase &, c10::Scalar, c10::Scalar, at::native::detail::ClampLimits)::[lambda() (instance 1)]::operator ()() const::[lambda() (instance 4)]::operator ()() const::[lambda(long) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,73152,51,1434.4,1120.0,1088,7584,1226.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add<long>, std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,52384,1,52384.0,52384.0,52384,52384,0.0,"void at::native::<unnamed>::CatArrayBatchedCopy<at::native::<unnamed>::OpaqueType<(unsigned int)4>, unsigned int, (int)2, (int)64, (int)64>(T1 *, at::native::<unnamed>::CatArrInputTensorMetadata<T1, T2, T4, T5>, at::native::<unnamed>::TensorSizeStride<T2, (unsigned int)4>, int, T2)"
+0.0,48832,1,48832.0,48832.0,48832,48832,0.0,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast<at::native::direct_copy_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 3)]::operator ()() const::[lambda() (instance 12)]::operator ()() const::[lambda(c10::BFloat16) (instance 1)]>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,44096,1,44096.0,44096.0,44096,44096,0.0,void cutlass::Kernel2<cutlass_80_simt_sgemm_128x64_8x5_tt_align1>(T1::Params)
+0.0,39456,1,39456.0,39456.0,39456,39456,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::sin_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 2)]::operator ()() const::[lambda() (instance 2)]::operator ()() const::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,31872,1,31872.0,31872.0,31872,31872,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::cos_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 2)]::operator ()() const::[lambda() (instance 2)]::operator ()() const::[lambda(float) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,16448,7,2349.7,1376.0,1056,6688,2004.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctorOnSelf_add<float>, std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,11424,5,2284.8,2080.0,1824,3392,628.1,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BinaryFunctor<float, float, float, at::native::binary_internal::DivFunctor<float>>, std::array<char *, (unsigned long)3>>(int, T2, T3)"
+0.0,8960,2,4480.0,4480.0,3360,5600,1583.9,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast<at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float>>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,6112,2,3056.0,3056.0,1184,4928,2647.4,"void at::native::unrolled_elementwise_kernel<at::native::CUDAFunctor_add<float>, std::array<char *, (unsigned long)3>, (int)4, TrivialOffsetCalculator<(int)2, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, T1, T2, T4, T5, T6, T7)"
+0.0,5728,1,5728.0,5728.0,5728,5728,0.0,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl<at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float>>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,4672,2,2336.0,2336.0,1824,2848,724.1,"void at::native::unrolled_elementwise_kernel<at::native::AUnaryFunctor<float, float, bool, at::native::<unnamed>::CompareEqFunctor<float>>, std::array<char *, (unsigned long)2>, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithoutCast, at::native::memory::StoreWithoutCast>(int, T1, T2, T4, T5, T6, T7)"
+0.0,4128,1,4128.0,4128.0,4128,4128,0.0,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast<at::native::CUDAFunctor_add<c10::BFloat16>>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)"
+0.0,4000,1,4000.0,4000.0,4000,4000,0.0,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)1, (bool)0, (bool)0>(cublasLt::cublasSplitKParams<T6>, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)"
+0.0,3872,2,1936.0,1936.0,1024,2848,1289.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctorOnOther_add<float>, std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,1984,1,1984.0,1984.0,1984,1984,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor<float, float, float, at::native::binary_internal::MulFunctor<float>>, std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,1664,1,1664.0,1664.0,1664,1664,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::neg_kernel_cuda(at::TensorIteratorBase &)::[lambda() (instance 2)]::operator ()() const::[lambda() (instance 9)]::operator ()() const::[lambda(c10::BFloat16) (instance 1)], std::array<char *, (unsigned long)2>>(int, T2, T3)"
+0.0,1312,1,1312.0,1312.0,1312,1312,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float>>, std::array<char *, (unsigned long)2>>(int, T2, T3)"

benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv

@@ -0,0 +1,2 @@
+Name,Projected Start (ns),Projected Duration (ns),Orig Start (ns),Orig Duration (ns),Style,PID,TID,NumGPUOps,Lvl,NumChild,RangeId,ParentId,RangeStack
+:complete_sampling_run,6687024,20907172640,5969088,20907943008,PushPop,73,73,2803,0,0,1,,:1

research/artifact_manifest.json

@@ -1,37 +1,37 @@
 {
   "metadata": {
     "algorithm": "SHA-256",
-    "generated_date": "2026-08-25",
-    "scope_note": "Local selected scope is the established 167-file set plus 14 retained FC2 scheduling and integration artifacts. Spark records are the complete reproducible current top-level benchmark output set; the audit-retained 279-file/665950155-byte aggregate cannot be reconstructed because its path list was not retained.",
+    "generated_date": "2026-08-26",
+    "scope_note": "Local selected scope is the established 167-file set plus 14 retained FC2 scheduling and integration artifacts and 19 post-FC2 production-profile artifacts. Spark records are the complete reproducible current top-level benchmark output set; the audit-retained 279-file/665950155-byte aggregate cannot be reconstructed because its path list was not retained.",
     "summary": {
       "local": {
-        "record_count": 181,
-        "size_bytes": 230398680,
-        "expected_record_count": 181,
-        "expected_size_bytes": 230398680,
+        "record_count": 200,
+        "size_bytes": 257150989,
+        "expected_record_count": 200,
+        "expected_size_bytes": 257150989,
         "reconciled": true
       },
       "spark": {
-        "record_count": 294,
-        "size_bytes": 567718575,
+        "record_count": 313,
+        "size_bytes": 594470884,
         "expected_record_count": 279,
         "expected_size_bytes": 665950155,
         "reconciled": false,
-        "record_count_delta": 15,
-        "size_bytes_delta": -98231580
+        "record_count_delta": 34,
+        "size_bytes_delta": -71479271
       },
       "total": {
-        "record_count": 475,
-        "size_bytes": 798117255
+        "record_count": 513,
+        "size_bytes": 851621873
       }
     },
     "json_reconciliation": {
-      "identical": 118,
+      "identical": 126,
       "mismatches": 0,
       "local_only": 40,
       "spark_only": 107,
-      "local_total": 158,
-      "spark_total": 225
+      "local_total": 166,
+      "spark_total": 233
     }
   },
   "artifacts": [
@@ -3834,6 +3834,44 @@
       "sha256": "9b711de3301244891d04465249db49a9233951928607a50a6ef9823a5f0fab58",
       "artifact_class": "run_log",
       "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/warmup-direct-sage3-fp16-1s.log"
-    }
+    },
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-profile-20260826.json", "size_bytes": 40955, "sha256": "631da0f3780250e2cd34dd39c5a0dbecc0624c58fefa38048c961fb161f80c86", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-profile-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-20260826.csv", "size_bytes": 50742, "sha256": "6fa41faf2de5697106a9a90520c2f812ffeb900051ae4fdadd4a5ef36dc9fe5f", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-20260826.csv"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-20260826.ncu-rep", "size_bytes": 8696909, "sha256": "f07a5abcd275951096ae5c95522fdbe5670081711bec73a4e381fcee3066af0d", "artifact_class": "nsight_compute_report", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-20260826.ncu-rep"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json", "size_bytes": 487, "sha256": "cb97c3f3c80bf5bee185df51b5c4b0b82bf90a55bb026431b25b2faa7bdd5b15", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-capture-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json", "size_bytes": 7359, "sha256": "2d046da14d90bb97317da176e75491f887732afccd48e9703c9904dd0e0e57f2", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-summary-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.csv", "size_bytes": 21877, "sha256": "4d7bca04ed986897989e9bee5070461d9259b2797779ac74504603fa9a7c5005", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-traffic-20260826.csv"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep", "size_bytes": 8082484, "sha256": "27cd225e5ed94977d9437410b42d5a2228e74f01100a737091b0b637b07c42d5", "artifact_class": "nsight_compute_report", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json", "size_bytes": 487, "sha256": "cb97c3f3c80bf5bee185df51b5c4b0b82bf90a55bb026431b25b2faa7bdd5b15", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-block24-targeted-traffic-capture-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-production-profile-summary-20260826.json", "size_bytes": 20432, "sha256": "559f9a0ebbe9507fece3612949329632761e3f7e10b42ceb5c87854583b6f0af", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-production-profile-summary-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-resident-baseline-20260826.json", "size_bytes": 28079, "sha256": "0b73bed25e8e209101a63b5dde633b7eea38783e251b01dbf2e6cd59bf4ee679", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-resident-baseline-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-20260826.nsys-rep", "size_bytes": 856665, "sha256": "286b925c48e7bc2e89d2b68a5c0a07c07c1905419ca7b3f511cc5d7e289f3cbb", "artifact_class": "nsight_systems_report", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-20260826.nsys-rep"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-20260826.sqlite", "size_bytes": 7106560, "sha256": "b8f64ac3938b190ecb1ef8ecedfe37278dd779fa8196a5e7220fe0cb90e217fb", "artifact_class": "nsight_sqlite_database", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-20260826.sqlite"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json", "size_bytes": 1963, "sha256": "402f2f4b4aa0f987f5cf133cc3c09f54bfcee0769c37a4a9981cb0e318fbe71f", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-capture-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json", "size_bytes": 2094, "sha256": "8bb61cb70ade2d5f794a21c7cc3ccf58d2bc8d85e3d8abe596437eb9749aafdd", "artifact_class": "benchmark_json", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-nsys-summary-20260826.json"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv", "size_bytes": 788, "sha256": "e0be4be5700ee6cff7bedf0bc7e9168a2b41a0825034ec8996b7e98e4da8ac8d", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv", "size_bytes": 18564, "sha256": "f7459b6e3cf004f89b2c42df4e0bf752b25fcbd7b550962e4ea95673d9edbcf5", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv", "size_bytes": 853416, "sha256": "d7eafe6a3da20b5afa49bcb7a531c04c396f47f3ba76934e978857440ab2bd61", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv", "size_bytes": 962206, "sha256": "509c9ad616f0834759871f46e302137d979be21a540b6656173f876589c08273", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv"},
+    {"scope": "local", "path": "benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv", "size_bytes": 242, "sha256": "12bfbbbbe10467badbf6b4a6ca712241f0375223d2798ab0f3ea68051a2c7f8c", "artifact_class": "nsight_csv_export", "location": "S:\\PycharmProjects\\AuthorCompanion\\tmp\\h3-blackwell-runtime\\benchmarks\\gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-profile-20260826.json", "size_bytes": 40955, "sha256": "631da0f3780250e2cd34dd39c5a0dbecc0624c58fefa38048c961fb161f80c86", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-profile-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-20260826.csv", "size_bytes": 50742, "sha256": "6fa41faf2de5697106a9a90520c2f812ffeb900051ae4fdadd4a5ef36dc9fe5f", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-20260826.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-20260826.ncu-rep", "size_bytes": 8696909, "sha256": "f07a5abcd275951096ae5c95522fdbe5670081711bec73a4e381fcee3066af0d", "artifact_class": "nsight_compute_report", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-20260826.ncu-rep"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json", "size_bytes": 487, "sha256": "cb97c3f3c80bf5bee185df51b5c4b0b82bf90a55bb026431b25b2faa7bdd5b15", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json", "size_bytes": 7359, "sha256": "2d046da14d90bb97317da176e75491f887732afccd48e9703c9904dd0e0e57f2", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.csv", "size_bytes": 21877, "sha256": "4d7bca04ed986897989e9bee5070461d9259b2797779ac74504603fa9a7c5005", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep", "size_bytes": 8082484, "sha256": "27cd225e5ed94977d9437410b42d5a2228e74f01100a737091b0b637b07c42d5", "artifact_class": "nsight_compute_report", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json", "size_bytes": 487, "sha256": "cb97c3f3c80bf5bee185df51b5c4b0b82bf90a55bb026431b25b2faa7bdd5b15", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-production-profile-summary-20260826.json", "size_bytes": 20432, "sha256": "559f9a0ebbe9507fece3612949329632761e3f7e10b42ceb5c87854583b6f0af", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-production-profile-summary-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-resident-baseline-20260826.json", "size_bytes": 28079, "sha256": "0b73bed25e8e209101a63b5dde633b7eea38783e251b01dbf2e6cd59bf4ee679", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-resident-baseline-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-20260826.nsys-rep", "size_bytes": 856665, "sha256": "286b925c48e7bc2e89d2b68a5c0a07c07c1905419ca7b3f511cc5d7e289f3cbb", "artifact_class": "nsight_systems_report", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-20260826.nsys-rep"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-20260826.sqlite", "size_bytes": 7106560, "sha256": "b8f64ac3938b190ecb1ef8ecedfe37278dd779fa8196a5e7220fe0cb90e217fb", "artifact_class": "nsight_sqlite_database", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-20260826.sqlite"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json", "size_bytes": 1963, "sha256": "402f2f4b4aa0f987f5cf133cc3c09f54bfcee0769c37a4a9981cb0e318fbe71f", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json", "size_bytes": 2094, "sha256": "8bb61cb70ade2d5f794a21c7cc3ccf58d2bc8d85e3d8abe596437eb9749aafdd", "artifact_class": "benchmark_json", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv", "size_bytes": 788, "sha256": "e0be4be5700ee6cff7bedf0bc7e9168a2b41a0825034ec8996b7e98e4da8ac8d", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv", "size_bytes": 18564, "sha256": "f7459b6e3cf004f89b2c42df4e0bf752b25fcbd7b550962e4ea95673d9edbcf5", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv", "size_bytes": 853416, "sha256": "d7eafe6a3da20b5afa49bcb7a531c04c396f47f3ba76934e978857440ab2bd61", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv", "size_bytes": 962206, "sha256": "509c9ad616f0834759871f46e302137d979be21a540b6656173f876589c08273", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv"},
+    {"scope": "spark", "path": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv", "size_bytes": 242, "sha256": "12bfbbbbe10467badbf6b4a6ca712241f0375223d2798ab0f3ea68051a2c7f8c", "artifact_class": "nsight_csv_export", "location": "/home/daniel/StoryStudioAssets/H3-output/h3-blackwell-runtime/benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv"}
   ]
 }

src/h3_blackwell_runtime/runtime.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import os
 import subprocess
 import time
@@ -55,6 +56,11 @@ def _sync() -> None:
         torch.cuda.synchronize()
 
 
+def _tensor_sha256(value: torch.Tensor) -> str:
+    immutable = value.detach().contiguous().view(torch.uint16).cpu()
+    return hashlib.sha256(immutable.numpy().tobytes()).hexdigest()
+
+
 def _ffmpeg_command(loglevel: str, *parts: str) -> list[str]:
     return ["ffmpeg", "-hide_banner", "-loglevel", loglevel, *parts]
 
@@ -263,9 +269,12 @@ class H3HotRuntime:
         cache_start_percent: float = 0.0,
         cache_end_percent: float = 1.0,
         cache_subsample_factor: int = 2,
+        benchmark_text_tokens: int | None = None,
     ) -> dict:
         stages: list[dict] = []
         cache_stats: dict = {}
+        diagnostics_enabled = benchmark_text_tokens is not None
+        sampling_step_events: list[dict] | None = [] if diagnostics_enabled else None
 
         def timed(stage: str, fn):
             _sync()
@@ -313,7 +322,19 @@ class H3HotRuntime:
                 "frame_count": frame_count,
             }
         else:
-            text = timed("text_conditioned", lambda: self.refiner(self.conditioner(prompt)))
+            if benchmark_text_tokens is None:
+                text = timed("text_conditioned", lambda: self.refiner(self.conditioner(prompt)))
+            else:
+                text = timed(
+                    "text_conditioned",
+                    lambda: torch.randn(
+                        1,
+                        benchmark_text_tokens,
+                        5376,
+                        device=self.config.device,
+                        dtype=torch.bfloat16,
+                    ),
+                )
             pack_kwargs = {}
         if self.turbo is None:
             sample = lambda: sample_video_res_multistep(
@@ -321,6 +342,7 @@ class H3HotRuntime:
                 return_audio=mux_audio, cache_mode=cache_mode, cache_threshold=cache_threshold,
                 cache_start_percent=cache_start_percent, cache_end_percent=cache_end_percent,
                 cache_subsample_factor=cache_subsample_factor, cache_stats=cache_stats, **pack_kwargs,
+                sampling_step_events=sampling_step_events,
             )
         else:
             sample = lambda: sample_video_turbo(
@@ -328,11 +350,32 @@ class H3HotRuntime:
                 video_shift=TURBO_VARIANTS[self.turbo]["video_shift"], seed=seed,
                 return_audio=mux_audio, **pack_kwargs,
             )
+        from .fc2_lt import fc2_lt_status
+
+        fc2_before = fc2_lt_status()
+        if diagnostics_enabled:
+            torch.cuda.reset_peak_memory_stats()
         sampled = timed("sampled", sample)
+        sampling_peak_allocated = torch.cuda.max_memory_allocated() if diagnostics_enabled else None
+        sampling_peak_reserved = torch.cuda.max_memory_reserved() if diagnostics_enabled else None
+        fc2_after = fc2_lt_status()
+        sampling_steps = [
+            {
+                "step": row["step"],
+                "seconds": row["start"].elapsed_time(row["end"]) / 1000.0,
+            }
+            for row in sampling_step_events or ()
+        ]
         if mux_audio:
             latent, audio_latent = sampled
         else:
             latent, audio_latent = sampled, None
+        latent_checksums = None
+        if diagnostics_enabled:
+            latent_checksums = {
+                "video_sha256": _tensor_sha256(latent),
+                "audio_sha256": _tensor_sha256(audio_latent) if audio_latent is not None else None,
+            }
         source_width, source_height = width, height
         if upscale_scale is not None:
             latent = timed(
@@ -411,6 +454,18 @@ class H3HotRuntime:
             "keep_intermediates": keep_intermediates,
             "vae_dtype": self.config.vae_dtype,
             "vae_tile_size": self.config.vae_tile_size,
+            "benchmark_text_tokens": benchmark_text_tokens,
+            "sampling_seconds": next(
+                stage["seconds"] for stage in stages if stage["stage"] == "sampled"
+            ),
+            "sampling_steps": sampling_steps,
+            "sampling_peak_allocated_bytes": sampling_peak_allocated,
+            "sampling_peak_reserved_bytes": sampling_peak_reserved,
+            "latent_checksums": latent_checksums,
+            "fc2_dispatch_delta": {
+                name: fc2_after[name] - fc2_before[name]
+                for name in ("attempts", "successes", "fallbacks")
+            },
             "stages": stages,
             "cache": cache_stats,
             "request_seconds": sum(stage["seconds"] for stage in stages),

src/h3_blackwell_runtime/sampler.py

@@ -116,6 +116,7 @@ def sample_video_res_multistep(
     cache_stats: dict | None = None,
     audio_step_trace: list[dict] | None = None,
     sampling_stage_trace: list[dict] | None = None,
+    sampling_step_events: list[dict] | None = None,
 ) -> torch.Tensor:
     """Direct H3 beta/RES sampling with Comfy-equivalent joint AV carry semantics."""
     sigmas = beta_sigmas(steps, device=video.device)
@@ -138,6 +139,10 @@ def sample_video_res_multistep(
         "cumulative_rate": 0.0,
     }
     for index, sigma in enumerate(sigmas[:-1], start=1):
+        step_start_event = None
+        if sampling_step_events is not None:
+            step_start_event = torch.cuda.Event(enable_timing=True)
+            step_start_event.record()
         step_started = time.perf_counter()
         stage_row = None
         if sampling_stage_trace is not None:
@@ -256,6 +261,14 @@ def sample_video_res_multistep(
             })
         video_history, audio_history = video_denoised, audio_denoised
         video_history_sigma = audio_history_sigma = sigma_down
+        if sampling_step_events is not None:
+            step_end_event = torch.cuda.Event(enable_timing=True)
+            step_end_event.record()
+            sampling_step_events.append({
+                "step": index,
+                "start": step_start_event,
+                "end": step_end_event,
+            })
         if stage_row is not None:
             torch.cuda.synchronize(video.device)
             stage_row["step_total"] = time.perf_counter() - profiled_step_started

tests/test_turbo.py

@@ -115,6 +115,43 @@ class TurboLoraContracts(unittest.TestCase):
         torch.testing.assert_close(trace[0]["audio_after"], torch.full_like(audio, 4.0))
         torch.testing.assert_close(final_audio, torch.ones_like(audio))
 
+    def test_base_sampler_records_deferred_step_events_without_synchronizing(self):
+        video = torch.zeros(1, 1, 1, 1, 1)
+        audio = torch.zeros(1, 32, 2, 1)
+        events = []
+
+        class FakeEvent:
+            def __init__(self, *, enable_timing):
+                self.enable_timing = enable_timing
+                self.recorded = False
+
+            def record(self):
+                self.recorded = True
+
+        def packer(*args, **kwargs):
+            return (None, None, None, None, None, None)
+
+        def model(*args):
+            return torch.ones(1), torch.ones(1)
+
+        with (
+            patch("h3_blackwell_runtime.sampler.beta_sigmas", return_value=torch.tensor([1.0, 0.0])),
+            patch("h3_blackwell_runtime.sampler.unpatchify_video", return_value=torch.zeros_like(video)),
+            patch("h3_blackwell_runtime.sampler._unpack_audio", return_value=torch.ones_like(audio)),
+            patch("h3_blackwell_runtime.sampler.torch.cuda.Event", side_effect=FakeEvent),
+            patch("h3_blackwell_runtime.sampler.torch.cuda.synchronize") as synchronize,
+        ):
+            sample_video_res_multistep(
+                model, packer, torch.empty(0), video, audio,
+                steps=1, return_audio=True, sampling_step_events=events,
+            )
+
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0]["step"], 1)
+        self.assertTrue(events[0]["start"].recorded)
+        self.assertTrue(events[0]["end"].recorded)
+        synchronize.assert_not_called()
+
 
 if __name__ == "__main__":
     unittest.main()

tools/benchmark_hot_runtime.py

@@ -0,0 +1,164 @@
+"""Establish a repeated canonical baseline through the resident H3 API."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import statistics
+import time
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+
+DEFAULT_PROMPT = (
+    "A playful orange tabby cat starts in an ordinary cozy living room in a normal house, "
+    "afternoon light, sofa and rug. The cat crouches, jumps, and does one clean athletic "
+    "backflip in slow motion. As the backflip completes there is a sharp cinematic cut: "
+    "the cat lands perfectly on a glowing neon disco dance floor wearing oversized black "
+    "sunglasses. Mirror ball reflections, colorful lights, joyful party energy, stylish "
+    "and funny, clear before-and-after transformation."
+)
+EXPECTED_VIDEO_SHA256 = "c62d23a42972eab907ba42f93c50247ff17a9c454b4a53fe93d2e34f9fefe578"
+EXPECTED_AUDIO_SHA256 = "852005383770480a6503504e1ffec86dd1fb63a69c6400f92da18e39e0986de2"
+
+
+def get_json(url: str, timeout: float = 30.0) -> dict:
+    with urlopen(url, timeout=timeout) as response:
+        return json.loads(response.read().decode())
+
+
+def post_json(url: str, payload: dict, timeout: float) -> dict:
+    request = Request(
+        url,
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"},
+    )
+    with urlopen(request, timeout=timeout) as response:
+        return json.loads(response.read().decode())
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--server", default="http://127.0.0.1:8001")
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--runs", type=int, default=3)
+    parser.add_argument("--timeout", type=float, default=1200.0)
+    parser.add_argument("--source-commit", default="a29b8960b0f887c20e74dafa16a24c37d6508b4e")
+    parser.add_argument("--image", required=True)
+    args = parser.parse_args()
+    if args.runs < 3:
+        raise ValueError("the authoritative baseline requires at least three measured runs")
+
+    ready_before = get_json(f"{args.server}/ready")
+    if not ready_before.get("ready"):
+        raise RuntimeError("resident runtime is not ready")
+    if ready_before["runtime"]["current_attention"] != "sage2":
+        raise RuntimeError("resident runtime is not using Sage2")
+    if not ready_before["runtime"]["fc2_lt"]["enabled"]:
+        raise RuntimeError("guarded FC2 schedule is disabled")
+
+    def payload(label: str) -> dict:
+        return {
+            "prompt": DEFAULT_PROMPT,
+            "output": f"/output/h3-blackwell-runtime/post-fc2-{label}.mp4",
+            "width": 1344,
+            "height": 768,
+            "frames": 124,
+            "steps": 12,
+            "seed": 440420,
+            "attention": "sage2",
+            "turbo": None,
+            "cache_mode": None,
+            "mux_audio": True,
+            "keep_intermediates": False,
+            "benchmark_text_tokens": 100,
+        }
+
+    warmup = post_json(f"{args.server}/generate", payload("canonical-warmup"), args.timeout)
+    measured = []
+    for index in range(1, args.runs + 1):
+        started = time.perf_counter()
+        response = post_json(
+            f"{args.server}/generate", payload(f"canonical-run-{index}"), args.timeout,
+        )
+        response["client_wall_seconds"] = time.perf_counter() - started
+        measured.append(response)
+
+    for name, response in [("warmup", warmup), *[
+        (f"run_{index}", value) for index, value in enumerate(measured, start=1)
+    ]]:
+        dispatch = response["fc2_dispatch_delta"]
+        if dispatch != {"attempts": 600, "successes": 600, "fallbacks": 0}:
+            raise RuntimeError(f"{name} FC2 dispatch validation failed: {dispatch}")
+        if len(response["sampling_steps"]) != 12:
+            raise RuntimeError(f"{name} did not report 12 sampling steps")
+
+    checksum_pairs = [
+        (row["latent_checksums"]["video_sha256"], row["latent_checksums"]["audio_sha256"])
+        for row in measured
+    ]
+    exact_parity = len(set(checksum_pairs)) == 1
+    if not exact_parity:
+        raise RuntimeError(f"measured latent checksums differ: {checksum_pairs}")
+    expected_checksums = (EXPECTED_VIDEO_SHA256, EXPECTED_AUDIO_SHA256)
+    if checksum_pairs[0] != expected_checksums:
+        raise RuntimeError(
+            f"latent checksums differ from canonical reference: "
+            f"expected {expected_checksums}, got {checksum_pairs[0]}"
+        )
+
+    sampling_seconds = [row["sampling_seconds"] for row in measured]
+    report = {
+        "name": "gb10-post-fc2-resident-baseline",
+        "source_commit": args.source_commit,
+        "image": args.image,
+        "measurement_date": "2026-08-26",
+        "workload": {
+            "resolution": [1344, 768],
+            "frames": 124,
+            "steps": 12,
+            "seed": 440420,
+            "attention": "sage2",
+            "prompt": DEFAULT_PROMPT,
+            "prompt_sha256": hashlib.sha256(DEFAULT_PROMPT.encode()).hexdigest(),
+        },
+        "measurement_policy": {
+            "canonical_warmup_runs": 1,
+            "measured_runs": args.runs,
+            "authoritative_timing": "median synchronized resident sampling_seconds",
+            "step_timing": "deferred CUDA event elapsed time; no per-step synchronization",
+            "profiling": False,
+        },
+        "sampling_seconds": sampling_seconds,
+        "median_sampling_seconds": statistics.median(sampling_seconds),
+        "sampling_steps": [row["sampling_steps"] for row in measured],
+        "sampling_peak_allocated_bytes": [
+            row["sampling_peak_allocated_bytes"] for row in measured
+        ],
+        "sampling_peak_reserved_bytes": [
+            row["sampling_peak_reserved_bytes"] for row in measured
+        ],
+        "latent_checksums": {
+            "video_sha256": checksum_pairs[0][0],
+            "audio_sha256": checksum_pairs[0][1],
+            "exact_across_measured_runs": exact_parity,
+            "matches_canonical_reference": True,
+        },
+        "fc2_dispatch": {
+            "required_per_run": 600,
+            "runs": [row["fc2_dispatch_delta"] for row in measured],
+            "all_passed": True,
+        },
+        "canonical_warmup": warmup,
+        "measured_responses": measured,
+        "ready_before": ready_before,
+        "ready_after": get_json(f"{args.server}/ready"),
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(report, indent=2), flush=True)
+
+
+if __name__ == "__main__":
+    main()

tools/build_post_fc2_profile_summary.py

@@ -0,0 +1,193 @@
+"""Build the post-FC2 authoritative GB10 baseline and profile summary."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def load(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def delta(current: float, previous: float) -> dict:
+    return {
+        "previous": previous,
+        "current": current,
+        "absolute_change": current - previous,
+        "percent_change": (current / previous - 1.0) * 100.0,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    benchmarks = args.root / "benchmarks"
+    resident = load(benchmarks / "gb10-post-fc2-resident-baseline-20260826.json")
+    block = load(benchmarks / "gb10-post-fc2-block24-profile-20260826.json")
+    nsys = load(benchmarks / "gb10-post-fc2-warmed-step-nsys-summary-20260826.json")
+    ncu = load(benchmarks / "gb10-post-fc2-block24-targeted-summary-20260826.json")
+    previous = load(benchmarks / "gb10-fully-fused-fresh-nsight-summary.json")
+
+    current_components = nsys["components"]
+    previous_components = previous["one_warmed_sampling_step"]["components"]
+    component_comparison = {
+        name: delta(
+            current_components[name]["milliseconds"],
+            previous_components[name]["milliseconds"],
+        )
+        for name in current_components
+    }
+    ranking = sorted(
+        (
+            {
+                "component": name,
+                "milliseconds": values["milliseconds"],
+                "percent_of_kernel_time": values["percent_of_kernel_time"],
+            }
+            for name, values in current_components.items()
+        ),
+        key=lambda row: row["milliseconds"],
+        reverse=True,
+    )
+    previous_step = previous["one_warmed_sampling_step"]
+    report = {
+        "name": "gb10-post-fc2-production-profile",
+        "measurement_date": "2026-08-26",
+        "source_commit": resident["source_commit"],
+        "measurement_overlay": (
+            "the measurement image included deferred step events and latent hashes plus an opt-in "
+            "100-token synthetic-text request; the final production overlay gates diagnostics to "
+            "that benchmark request, and model math is unchanged"
+        ),
+        "image": resident["image"],
+        "device": "NVIDIA GB10",
+        "compute_capability": "SM121",
+        "tools": {
+            "torch": "2.9.1+cu130",
+            "cuda": "13.0",
+            "nsight_systems": "2025.3.2.474-253236389321v0",
+            "nsight_compute": "2025.3.1",
+        },
+        "workload": {
+            "resolution": [1344, 768],
+            "frames": 124,
+            "packed_tokens": 37810,
+            "text_tokens": 100,
+            "steps": 12,
+            "seed": 440420,
+            "attention": "sage2",
+        },
+        "configuration": {
+            "H3_NVFP4_SCALE_BACKEND": "vortex",
+            "H3_NVFP4_SCALE_VERSION": "1",
+            "H3_FUSED_ELEMENTWISE": "1",
+            "H3_NVFP4_MODULATE_FUSION": "1",
+            "H3_NVFP4_SWIGLU_FUSION": "1",
+            "H3_NVFP4_FC2_LT_SPLITK1": "1",
+            "H3_SAGE_QKV_LAYOUT": "strided_nhd",
+        },
+        "authoritative_resident_baseline": {
+            "sampling_seconds": resident["sampling_seconds"],
+            "median_sampling_seconds": resident["median_sampling_seconds"],
+            "per_step_seconds": resident["sampling_steps"],
+            "peak_allocated_bytes": resident["sampling_peak_allocated_bytes"],
+            "peak_reserved_bytes": resident["sampling_peak_reserved_bytes"],
+            "latent_checksums": resident["latent_checksums"],
+            "fc2_dispatch": resident["fc2_dispatch"],
+            "measurement_policy": resident["measurement_policy"],
+        },
+        "block_24": {
+            "uninstrumented_module_p50_ms": block["module_forward"]["p50_s"] * 1000.0,
+            "uninstrumented_module_samples": block["module_forward"]["count"],
+            "synchronized_decomposition_is_attribution_only": True,
+            "fc2_schedule_note": (
+                "profile_h3_block.py decomposes generic NVFP4 linear calls and bypasses "
+                "forward_swiglu/guarded FC2; use resident NSYS and targeted NCU for production FC2"
+            ),
+        },
+        "one_warmed_sampling_step_nsys": {
+            "elapsed_seconds": load(
+                benchmarks / "gb10-post-fc2-warmed-step-capture-20260826.json"
+            )["elapsed_seconds"],
+            "gpu_span_seconds": nsys["gpu_span_ns"] / 1.0e9,
+            "kernel_time_seconds": nsys["kernel_time_ns"] / 1.0e9,
+            "gpu_operation_count": nsys["gpu_operation_count"],
+            "kernel_count": nsys["kernel_count"],
+            "kernel_busy_percent_of_span": nsys["kernel_busy_percent_of_span"],
+            "launch_gaps": nsys["launch_gaps"],
+            "cpu_gpu_overlap": nsys["cpu_gpu_overlap"],
+            "components": current_components,
+        },
+        "block_24_targeted_ncu": ncu,
+        "comparison_to_pre_fc2_profile": {
+            "previous_summary": "benchmarks/gb10-fully-fused-fresh-nsight-summary.json",
+            "block_24_p50_ms": delta(
+                block["module_forward"]["p50_s"] * 1000.0,
+                previous["block_24"]["uninstrumented_module_p50_ms"],
+            ),
+            "block_24_comparison_note": (
+                "same-script decomposition control only; it excludes guarded FC2 and must not "
+                "be interpreted as the production FC2 gain"
+            ),
+            "warmed_step_gpu_span_seconds": delta(
+                nsys["gpu_span_ns"] / 1.0e9,
+                previous_step["gpu_span_seconds"],
+            ),
+            "warmed_step_kernel_time_seconds": delta(
+                nsys["kernel_time_ns"] / 1.0e9,
+                previous_step["kernel_time_seconds"],
+            ),
+            "warmed_step_kernel_count": delta(
+                nsys["kernel_count"], previous_step["kernel_launches"],
+            ),
+            "components": component_comparison,
+        },
+        "bottleneck_ranking": ranking,
+        "decision": {
+            "authoritative_exact_baseline": (
+                f"{resident['median_sampling_seconds']:.6f} s median resident sampling"
+            ),
+            "time_bottleneck": (
+                "Sage2 remains dominant at 62.36% of warmed-step kernel time; its mainloop "
+                "is 241.91 ms average in NSYS and 259.00 ms in the NCU replay."
+            ),
+            "secondary_bottlenecks": (
+                "NVFP4 GEMMs are 19.95%, packing 9.70%, norm/RoPE 5.19%, and gate/add 2.63%."
+            ),
+            "next_optimization": (
+                "None started. Sage2 is the next-ranked investigation target; any implementation "
+                "requires a separate approved experiment after this baseline is accepted."
+            ),
+        },
+        "artifacts": [
+            "benchmarks/gb10-post-fc2-resident-baseline-20260826.json",
+            "benchmarks/gb10-post-fc2-block24-profile-20260826.json",
+            "benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json",
+            "benchmarks/gb10-post-fc2-warmed-step-20260826.nsys-rep",
+            "benchmarks/gb10-post-fc2-warmed-step-20260826.sqlite",
+            "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv",
+            "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv",
+            "benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv",
+            "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv",
+            "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv",
+            "benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json",
+            "benchmarks/gb10-post-fc2-block24-targeted-20260826.ncu-rep",
+            "benchmarks/gb10-post-fc2-block24-targeted-20260826.csv",
+            "benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json",
+            "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep",
+            "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.csv",
+            "benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json",
+            "benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json",
+        ],
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(report, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tools/profile_sampling_stages.py

@@ -3,8 +3,11 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import inspect
 import json
+import os
+import sys
 import time
 from pathlib import Path
 
@@ -13,6 +16,7 @@ import torch
 from h3_blackwell_runtime.attention import AVAILABLE_BACKENDS
 from h3_blackwell_runtime.checkpoint import H3Checkpoint
 from h3_blackwell_runtime.denoiser import H3PackedDenoiser
+from h3_blackwell_runtime.fc2_lt import fc2_lt_status, prepare_fc2_lt
 from h3_blackwell_runtime.packing import H3PromptPacker
 from h3_blackwell_runtime.sampler import sample_video_res_multistep
 from h3_blackwell_runtime.t2v import random_av_latents
@@ -40,6 +44,11 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
+def tensor_sha256(value: torch.Tensor) -> str:
+    immutable = value.detach().contiguous().view(torch.uint16).cpu()
+    return hashlib.sha256(immutable.numpy().tobytes()).hexdigest()
+
+
 def main() -> None:
     args = parse_args()
     torch.manual_seed(args.seed)
@@ -47,6 +56,8 @@ def main() -> None:
     model = H3PackedDenoiser.from_checkpoint(
         checkpoint, output_dtype=torch.bfloat16, attention_backend=args.attention,
     ).eval()
+    if not prepare_fc2_lt():
+        raise RuntimeError("guarded FC2 extension preparation failed")
     packer = H3PromptPacker(checkpoint)
     if hasattr(checkpoint, "release_cache"):
         checkpoint.release_cache()
@@ -75,6 +86,7 @@ def main() -> None:
     torch.cuda.synchronize()
     if args.cuda_profiler_capture:
         torch.cuda.cudart().cudaProfilerStart()
+    fc2_before = fc2_lt_status()
     started = time.perf_counter()
     supports_stage_trace = "sampling_stage_trace" in inspect.signature(
         sample_video_res_multistep,
@@ -99,6 +111,9 @@ def main() -> None:
     elapsed = time.perf_counter() - started
     if args.cuda_profiler_capture:
         torch.cuda.cudart().cudaProfilerStop()
+    fc2_after = fc2_lt_status()
+    peak_allocated = torch.cuda.max_memory_allocated()
+    peak_reserved = torch.cuda.max_memory_reserved()
     report = {
         "device": torch.cuda.get_device_name(),
         "torch": torch.__version__,
@@ -110,11 +125,24 @@ def main() -> None:
         "text_tokens": args.text_tokens,
         "warmup_runs": args.warmup_runs,
         "cuda_profiler_capture": args.cuda_profiler_capture,
+        "argv": sys.argv,
+        "environment_switches": {
+            name: value for name, value in sorted(os.environ.items())
+            if name.startswith(("H3_", "CUDA_", "TORCH_"))
+        },
         "elapsed_seconds": elapsed,
         "stage_trace": trace,
         "checksums": [sampled_video.float().sum().item(), sampled_audio.float().sum().item()],
-        "peak_allocated_bytes": torch.cuda.max_memory_allocated(),
-        "peak_reserved_bytes": torch.cuda.max_memory_reserved(),
+        "sha256": {
+            "video": tensor_sha256(sampled_video),
+            "audio": tensor_sha256(sampled_audio),
+        },
+        "fc2_dispatch_delta": {
+            name: fc2_after[name] - fc2_before[name]
+            for name in ("attempts", "successes", "fallbacks")
+        },
+        "peak_allocated_bytes": peak_allocated,
+        "peak_reserved_bytes": peak_reserved,
         "measurement_policy": (
             "uninstrumented sampling wall time"
             if args.uninstrumented or not supports_stage_trace

tools/serve_hot_runtime.py

@@ -164,6 +164,12 @@ class Handler(BaseHTTPRequestHandler):
             cache_start_percent = float(payload.get("cache_start_percent", 0.0))
             cache_end_percent = float(payload.get("cache_end_percent", 1.0))
             cache_subsample_factor = int(payload.get("cache_subsample_factor", 2))
+            benchmark_text_tokens = payload.get("benchmark_text_tokens")
+            if benchmark_text_tokens is not None:
+                benchmark_text_tokens = int(benchmark_text_tokens)
+                if benchmark_text_tokens != 100 or first_frame is not None or last_frame is not None:
+                    write_json(self, 400, {"error": "benchmark_text_tokens requires canonical T2V value 100"})
+                    return
             started = time.perf_counter()
             with runtime_lock:
                 result = runtime.generate(
@@ -188,6 +194,7 @@ class Handler(BaseHTTPRequestHandler):
                     cache_start_percent=cache_start_percent,
                     cache_end_percent=cache_end_percent,
                     cache_subsample_factor=cache_subsample_factor,
+                    benchmark_text_tokens=benchmark_text_tokens,
                 )
             result["wall_seconds"] = time.perf_counter() - started
             write_json(self, 200, result)

tools/summarize_ncu_profile.py

@@ -0,0 +1,127 @@
+"""Extract targeted H3 block metrics from an Nsight Compute raw CSV export."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+
+
+ROLES = ("qkv", "sage2", "attention_output", "fc1", "fc2")
+EXPECTED_GRIDS = (
+    "(296, 168, 1)",
+    "(296, 56, 1)",
+    "(296, 42, 1)",
+    "(296, 224, 1)",
+    "(296, 42, 1)",
+)
+
+
+def number(row: dict[str, str], name: str) -> float | None:
+    value = row.get(name, "")
+    if value in {"", "no data", "n/a"}:
+        return None
+    return float(value)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--traffic", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+
+    with args.input.open(newline="", encoding="utf-8") as handle:
+        reader = csv.DictReader(handle)
+        units = next(reader)
+        rows = list(reader)
+    if len(rows) != len(ROLES):
+        raise RuntimeError(f"expected five targeted launches, found {len(rows)}")
+    with args.traffic.open(newline="", encoding="utf-8") as handle:
+        traffic_reader = csv.DictReader(handle)
+        next(traffic_reader)
+        traffic_rows = list(traffic_reader)
+    if len(traffic_rows) != len(ROLES):
+        raise RuntimeError(f"expected five traffic launches, found {len(traffic_rows)}")
+    for index, (role, row, traffic_row, expected_grid) in enumerate(
+        zip(ROLES, rows, traffic_rows, EXPECTED_GRIDS, strict=True)
+    ):
+        if int(row["ID"]) != index or int(traffic_row["ID"]) != index:
+            raise RuntimeError(f"{role} launch ID/order contract failed")
+        if row["Kernel Name"] != traffic_row["Kernel Name"]:
+            raise RuntimeError(f"{role} kernel differs between metric and traffic passes")
+        if row["Grid Size"] != expected_grid or traffic_row["Grid Size"] != expected_grid:
+            raise RuntimeError(f"{role} grid/order contract failed")
+        if role == "sage2" and "qk_int_sv_f8_attn_kernel" not in row["Kernel Name"]:
+            raise RuntimeError("Sage2 launch contract failed")
+        if role != "sage2" and "block_scaled" not in row["Kernel Name"]:
+            raise RuntimeError(f"{role} NVFP4 GEMM launch contract failed")
+
+    stall_prefix = "smsp__average_warps_issue_stalled_"
+    stall_suffix = "_per_issue_active.ratio"
+    output = {}
+    for role, row, traffic_row in zip(ROLES, rows, traffic_rows, strict=True):
+        stalls = []
+        for name in row:
+            if name.startswith(stall_prefix) and name.endswith(stall_suffix):
+                value = number(row, name)
+                if value is not None:
+                    stalls.append({
+                        "reason": name[len(stall_prefix):-len(stall_suffix)],
+                        "warps_per_issue_active": value,
+                    })
+        stalls.sort(key=lambda item: item["warps_per_issue_active"], reverse=True)
+        output[role] = {
+            "launch_id": int(row["ID"]),
+            "kernel_name": row["Kernel Name"],
+            "grid_size": row["Grid Size"],
+            "block_size": row["Block Size"],
+            "duration_ns": number(row, "gpu__time_duration.sum"),
+            "registers_per_thread": number(row, "launch__registers_per_thread"),
+            "achieved_occupancy_percent": number(
+                row, "sm__warps_active.avg.pct_of_peak_sustained_active"
+            ),
+            "eligible_warps_per_scheduler": number(
+                row, "smsp__warps_eligible.avg.per_cycle_active"
+            ),
+            "issue_active_percent": number(
+                row, "smsp__issue_active.avg.pct_of_peak_sustained_active"
+            ),
+            "sm_throughput_percent": number(
+                row, "sm__throughput.avg.pct_of_peak_sustained_elapsed"
+            ),
+            "tensor_pipe_active_percent": number(
+                row, "sm__pipe_tensor_cycles_active.avg.pct_of_peak_sustained_elapsed"
+            ),
+            "l2_requested_bytes": number(traffic_row, "lts__t_bytes.sum"),
+            "l2_hit_rate_percent": number(traffic_row, "lts__t_sector_hit_rate.pct"),
+            "l2_throughput_percent": number(
+                row, "lts__throughput.avg.pct_of_peak_sustained_elapsed"
+            ),
+            "memory_throughput_percent": number(
+                traffic_row, "gpu__compute_memory_throughput.avg.pct_of_peak_sustained_elapsed"
+            ),
+            "local_spilling_requests": number(row, "derived__local_spilling_requests"),
+            "top_scheduler_stalls": stalls[:5],
+        }
+
+    report = {
+        "source_csv": str(args.input),
+        "source_traffic_csv": str(args.traffic),
+        "launch_order_contract": list(ROLES),
+        "cache_control": "none (warmed/uncontrolled cache, as reported by NCU)",
+        "metrics": output,
+        "units": {
+            "duration_ns": "ns",
+            "l2_requested_bytes": "lts__t_bytes.sum",
+            "throughput_and_hit_rate": "%",
+        },
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(report, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tools/summarize_nsys_profile.py

@@ -0,0 +1,166 @@
+"""Summarize an Nsight Systems CUDA trace into stable H3 component categories."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+
+
+def category(name: str) -> str:
+    if any(token in name for token in (
+        "qk_int_sv_f8_attn_kernel",
+        "MeanScaleKernel",
+        "TransposePadPermuteKernel",
+        "QuantInt8Kernel",
+    )):
+        return "sage2"
+    if "cutlass3x_sm120_bstensorop" in name:
+        return "nvfp4_gemms"
+    if any(token in name for token in (
+        "partial_absmax_",
+        "quantize_nvfp4_",
+        "quantize_nvfp4_kernel",
+        "final_scale_",
+        "FillFunctor<unsigned char>",
+    )):
+        return "nvfp4_packing"
+    if any(token in name for token in (
+        "rope_kernel",
+        "vectorized_layer_norm_kernel",
+        "MeanOps<c10::BFloat16",
+    )):
+        return "norm_and_rope"
+    if "_gate_add_kernel" in name:
+        return "remaining_gate_add"
+    return "other"
+
+
+def merge(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:
+    output: list[list[int]] = []
+    for start, end in sorted(intervals):
+        if not output or start > output[-1][1]:
+            output.append([start, end])
+        else:
+            output[-1][1] = max(output[-1][1], end)
+    return [(start, end) for start, end in output]
+
+
+def total(intervals: list[tuple[int, int]]) -> int:
+    return sum(end - start for start, end in merge(intervals))
+
+
+def intersection_total(
+    left: list[tuple[int, int]], right: list[tuple[int, int]],
+) -> int:
+    left = merge(left)
+    right = merge(right)
+    i = j = result = 0
+    while i < len(left) and j < len(right):
+        start = max(left[i][0], right[j][0])
+        end = min(left[i][1], right[j][1])
+        result += max(0, end - start)
+        if left[i][1] <= right[j][1]:
+            i += 1
+        else:
+            j += 1
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--gpu-trace", type=Path, required=True)
+    parser.add_argument("--kernel-exec-trace", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+
+    with args.gpu_trace.open(newline="", encoding="utf-8") as handle:
+        gpu_rows = list(csv.DictReader(handle))
+    kernels = [row for row in gpu_rows if row["GrdX"]]
+    gpu_intervals = [
+        (int(row["Start (ns)"]), int(row["Start (ns)"]) + int(row["Duration (ns)"]))
+        for row in kernels
+    ]
+    all_gpu_intervals = [
+        (int(row["Start (ns)"]), int(row["Start (ns)"]) + int(row["Duration (ns)"]))
+        for row in gpu_rows
+    ]
+    first_gpu = min(start for start, _ in all_gpu_intervals)
+    last_gpu = max(end for _, end in all_gpu_intervals)
+    kernel_time = sum(int(row["Duration (ns)"]) for row in kernels)
+    component_ns = {
+        name: 0 for name in (
+            "sage2", "nvfp4_gemms", "nvfp4_packing", "norm_and_rope",
+            "remaining_gate_add", "other",
+        )
+    }
+    component_launches = component_ns.copy()
+    for row in kernels:
+        name = category(row["Name"])
+        component_ns[name] += int(row["Duration (ns)"])
+        component_launches[name] += 1
+
+    ordered = sorted(gpu_intervals)
+    gaps = [
+        max(0, ordered[index][0] - ordered[index - 1][1])
+        for index in range(1, len(ordered))
+    ]
+    positive_gaps = [gap for gap in gaps if gap]
+
+    with args.kernel_exec_trace.open(newline="", encoding="utf-8") as handle:
+        launch_rows = list(csv.DictReader(handle))
+    api_intervals = [
+        (int(row["API Start (ns)"]), int(row["API Start (ns)"]) + int(row["API Dur (ns)"]))
+        for row in launch_rows
+    ]
+    launch_api_time = total(api_intervals)
+    launch_api_gpu_overlap = intersection_total(api_intervals, gpu_intervals)
+    gpu_span = last_gpu - first_gpu
+
+    report = {
+        "source_gpu_trace": str(args.gpu_trace),
+        "source_kernel_exec_trace": str(args.kernel_exec_trace),
+        "gpu_span_ns": gpu_span,
+        "gpu_operation_count": len(gpu_rows),
+        "kernel_count": len(kernels),
+        "kernel_time_ns": kernel_time,
+        "kernel_busy_percent_of_span": kernel_time / gpu_span * 100.0,
+        "launch_gaps": {
+            "positive_gap_count": len(positive_gaps),
+            "total_ns": sum(positive_gaps),
+            "average_ns": sum(positive_gaps) / len(positive_gaps) if positive_gaps else 0,
+            "maximum_ns": max(positive_gaps, default=0),
+        },
+        "cpu_gpu_overlap": {
+            "scope": "CUDA kernel-launch API intervals intersected with GPU kernel intervals",
+            "launch_api_union_ns": launch_api_time,
+            "launch_api_gpu_overlap_ns": launch_api_gpu_overlap,
+            "launch_api_overlap_percent": (
+                launch_api_gpu_overlap / launch_api_time * 100.0 if launch_api_time else 0
+            ),
+        },
+        "components": {
+            name: {
+                "milliseconds": value / 1.0e6,
+                "percent_of_kernel_time": value / kernel_time * 100.0,
+                "launches": component_launches[name],
+            }
+            for name, value in component_ns.items()
+        },
+        "classification_policy": {
+            "sage2": "mainloop plus MeanScale/TransposePadPermute/QuantInt8 preparation",
+            "nvfp4_gemms": "SM120 block-scaled CUTLASS GEMMs",
+            "nvfp4_packing": "absmax, final-scale, zero-fill and NVFP4 quantization kernels",
+            "norm_and_rope": "layer norm, mean reduction and fused RMS/RoPE kernels",
+            "remaining_gate_add": "fused residual gate/add kernels",
+            "other": "all unmatched kernels",
+        },
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(report, indent=2))
+
+
+if __name__ == "__main__":
+    main()