{ "name": "gb10-post-fc2-production-profile", "measurement_date": "2026-08-26", "source_commit": "a29b8960b0f887c20e74dafa16a24c37d6508b4e", "measurement_overlay": "the measurement image included deferred step events and latent hashes plus an opt-in 100-token synthetic-text request; the final production overlay gates diagnostics to that benchmark request, and model math is unchanged", "image": "sha256:a15d0c09dd8cc82aaf2b564d3da760ea5ab8dec974f73075f7d30ac3a504815c", "device": "NVIDIA GB10", "compute_capability": "SM121", "tools": { "torch": "2.9.1+cu130", "cuda": "13.0", "nsight_systems": "2025.3.2.474-253236389321v0", "nsight_compute": "2025.3.1" }, "workload": { "resolution": [ 1344, 768 ], "frames": 124, "packed_tokens": 37810, "text_tokens": 100, "steps": 12, "seed": 440420, "attention": "sage2" }, "configuration": { "H3_NVFP4_SCALE_BACKEND": "vortex", "H3_NVFP4_SCALE_VERSION": "1", "H3_FUSED_ELEMENTWISE": "1", "H3_NVFP4_MODULATE_FUSION": "1", "H3_NVFP4_SWIGLU_FUSION": "1", "H3_NVFP4_FC2_LT_SPLITK1": "1", "H3_SAGE_QKV_LAYOUT": "strided_nhd" }, "authoritative_resident_baseline": { "sampling_seconds": [ 256.46369375299946, 255.44699439899978, 255.13481797700024 ], "median_sampling_seconds": 255.44699439899978, "per_step_seconds": [ [ { "step": 1, "seconds": 21.337705078125 }, { "step": 2, "seconds": 21.40503515625 }, { "step": 3, "seconds": 21.370228515625 }, { "step": 4, "seconds": 21.38410546875 }, { "step": 5, "seconds": 21.38205078125 }, { "step": 6, "seconds": 21.373046875 }, { "step": 7, "seconds": 21.376810546875 }, { "step": 8, "seconds": 21.367021484375 }, { "step": 9, "seconds": 21.386291015625 }, { "step": 10, "seconds": 21.36256640625 }, { "step": 11, "seconds": 21.404072265625 }, { "step": 12, "seconds": 21.313634765625 } ], [ { "step": 1, "seconds": 21.696716796875 }, { "step": 2, "seconds": 21.326416015625 }, { "step": 3, "seconds": 21.3040703125 }, { "step": 4, "seconds": 21.332494140625 }, { "step": 5, "seconds": 21.26598046875 }, { "step": 6, "seconds": 21.267458984375 }, { "step": 7, "seconds": 21.25258203125 }, { "step": 8, "seconds": 21.21969140625 }, { "step": 9, "seconds": 21.242830078125 }, { "step": 10, "seconds": 21.20158203125 }, { "step": 11, "seconds": 21.193841796875 }, { "step": 12, "seconds": 21.141802734375 } ], [ { "step": 1, "seconds": 21.116169921875 }, { "step": 2, "seconds": 21.237697265625 }, { "step": 3, "seconds": 21.2424921875 }, { "step": 4, "seconds": 21.273701171875 }, { "step": 5, "seconds": 21.275826171875 }, { "step": 6, "seconds": 21.303318359375 }, { "step": 7, "seconds": 21.3065546875 }, { "step": 8, "seconds": 21.32540234375 }, { "step": 9, "seconds": 21.35030078125 }, { "step": 10, "seconds": 21.258994140625 }, { "step": 11, "seconds": 21.249765625 }, { "step": 12, "seconds": 21.193939453125 } ] ], "peak_allocated_bytes": [ 44445830144, 44445830144, 44445830144 ], "peak_reserved_bytes": [ 48708452352, 48708452352, 48708452352 ], "latent_checksums": { "video_sha256": "c62d23a42972eab907ba42f93c50247ff17a9c454b4a53fe93d2e34f9fefe578", "audio_sha256": "852005383770480a6503504e1ffec86dd1fb63a69c6400f92da18e39e0986de2", "exact_across_measured_runs": true }, "fc2_dispatch": { "required_per_run": 600, "runs": [ { "attempts": 600, "successes": 600, "fallbacks": 0 }, { "attempts": 600, "successes": 600, "fallbacks": 0 }, { "attempts": 600, "successes": 600, "fallbacks": 0 } ], "all_passed": true }, "measurement_policy": { "canonical_warmup_runs": 1, "measured_runs": 3, "authoritative_timing": "median synchronized resident sampling_seconds", "step_timing": "deferred CUDA event elapsed time; no per-step synchronization", "profiling": false } }, "block_24": { "uninstrumented_module_p50_ms": 427.4100805005219, "uninstrumented_module_samples": 50, "synchronized_decomposition_is_attribution_only": true, "fc2_schedule_note": "profile_h3_block.py decomposes generic NVFP4 linear calls and bypasses forward_swiglu/guarded FC2; use resident NSYS and targeted NCU for production FC2" }, "one_warmed_sampling_step_nsys": { "elapsed_seconds": 20.90812512299999, "gpu_span_seconds": 20.90717264, "kernel_time_seconds": 20.895065376, "gpu_operation_count": 2803, "kernel_count": 2744, "kernel_busy_percent_of_span": 99.94209038109325, "launch_gaps": { "positive_gap_count": 2743, "total_ns": 11518912, "average_ns": 4199.384615384615, "maximum_ns": 4797728 }, "cpu_gpu_overlap": { "scope": "CUDA kernel-launch API intervals intersected with GPU kernel intervals", "launch_api_union_ns": 13065552528, "launch_api_gpu_overlap_ns": 13062771136, "launch_api_overlap_percent": 99.97871202160002 }, "components": { "sage2": { "milliseconds": 13029.23856, "percent_of_kernel_time": 62.355576905566124, "launches": 250 }, "nvfp4_gemms": { "milliseconds": 4167.855808, "percent_of_kernel_time": 19.946603339117498, "launches": 200 }, "nvfp4_packing": { "milliseconds": 2026.853664, "percent_of_kernel_time": 9.700154689767265, "launches": 800 }, "norm_and_rope": { "milliseconds": 1084.3856, "percent_of_kernel_time": 5.1896731619970025, "launches": 202 }, "remaining_gate_add": { "milliseconds": 549.298592, "percent_of_kernel_time": 2.6288436150619683, "launches": 100 }, "other": { "milliseconds": 37.433152, "percent_of_kernel_time": 0.17914828849014078, "launches": 1192 } } }, "block_24_targeted_ncu": { "source_csv": "benchmarks\\gb10-post-fc2-block24-targeted-20260826.csv", "source_traffic_csv": "benchmarks\\gb10-post-fc2-block24-targeted-traffic-20260826.csv", "launch_order_contract": [ "qkv", "sage2", "attention_output", "fc1", "fc2" ], "cache_control": "none (warmed/uncontrolled cache, as reported by NCU)", "metrics": { "qkv": { "launch_id": 0, "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu", "grid_size": "(296, 168, 1)", "block_size": "(384, 1, 1)", "duration_ns": 26376032.0, "registers_per_thread": 168.0, "achieved_occupancy_percent": 20.83, "eligible_warps_per_scheduler": 0.24, "issue_active_percent": 18.29, "sm_throughput_percent": 78.99, "tensor_pipe_active_percent": 78.99, "l2_requested_bytes": 40128950176.0, "l2_hit_rate_percent": 89.11, "l2_throughput_percent": 77.1, "memory_throughput_percent": 76.92, "local_spilling_requests": 0.0, "top_scheduler_stalls": [ { "reason": "sleeping", "warps_per_issue_active": 3.93 }, { "reason": "math_pipe_throttle", "warps_per_issue_active": 3.82 }, { "reason": "wait", "warps_per_issue_active": 3.78 }, { "reason": "selected", "warps_per_issue_active": 1.0 }, { "reason": "long_scoreboard", "warps_per_issue_active": 0.61 } ] }, "sage2": { "launch_id": 1, "kernel_name": "void qk_int_sv_f8_attn_kernel<128, 64, 32, 64, 128, 1, 2, 2, float, 1, __nv_bfloat16, 1, 0, 0, 1, 0, 1>(signed char *, signed char *, signed char *, T11 *, float *, float *, float *, float *, float *, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, float)", "grid_size": "(296, 56, 1)", "block_size": "(32, 4, 1)", "duration_ns": 258996000.0, "registers_per_thread": 255.0, "achieved_occupancy_percent": 16.65, "eligible_warps_per_scheduler": 0.46, "issue_active_percent": 36.49, "sm_throughput_percent": 75.51, "tensor_pipe_active_percent": 75.51, "l2_requested_bytes": 161497982464.0, "l2_hit_rate_percent": 98.85, "l2_throughput_percent": 31.59, "memory_throughput_percent": 31.74, "local_spilling_requests": 1458688.0, "top_scheduler_stalls": [ { "reason": "wait", "warps_per_issue_active": 2.01 }, { "reason": "math_pipe_throttle", "warps_per_issue_active": 1.24 }, { "reason": "selected", "warps_per_issue_active": 1.0 }, { "reason": "not_selected", "warps_per_issue_active": 0.26 }, { "reason": "short_scoreboard", "warps_per_issue_active": 0.25 } ] }, "attention_output": { "launch_id": 2, "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu", "grid_size": "(296, 42, 1)", "block_size": "(384, 1, 1)", "duration_ns": 8481152.0, "registers_per_thread": 168.0, "achieved_occupancy_percent": 20.83, "eligible_warps_per_scheduler": 0.23, "issue_active_percent": 17.51, "sm_throughput_percent": 81.88, "tensor_pipe_active_percent": 81.88, "l2_requested_bytes": 13241354176.0, "l2_hit_rate_percent": 89.89, "l2_throughput_percent": 79.1, "memory_throughput_percent": 79.51, "local_spilling_requests": 0.0, "top_scheduler_stalls": [ { "reason": "sleeping", "warps_per_issue_active": 4.08 }, { "reason": "math_pipe_throttle", "warps_per_issue_active": 3.9 }, { "reason": "wait", "warps_per_issue_active": 3.85 }, { "reason": "selected", "warps_per_issue_active": 1.0 }, { "reason": "long_scoreboard", "warps_per_issue_active": 0.6 } ] }, "fc1": { "launch_id": 3, "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu", "grid_size": "(296, 224, 1)", "block_size": "(384, 1, 1)", "duration_ns": 36172608.0, "registers_per_thread": 168.0, "achieved_occupancy_percent": 20.83, "eligible_warps_per_scheduler": 0.23, "issue_active_percent": 17.39, "sm_throughput_percent": 76.85, "tensor_pipe_active_percent": 76.85, "l2_requested_bytes": 53505320928.0, "l2_hit_rate_percent": 89.11, "l2_throughput_percent": 74.99, "memory_throughput_percent": 74.85, "local_spilling_requests": 0.0, "top_scheduler_stalls": [ { "reason": "sleeping", "warps_per_issue_active": 4.41 }, { "reason": "math_pipe_throttle", "warps_per_issue_active": 3.82 }, { "reason": "wait", "warps_per_issue_active": 3.78 }, { "reason": "selected", "warps_per_issue_active": 1.0 }, { "reason": "long_scoreboard", "warps_per_issue_active": 0.61 } ] }, "fc2": { "launch_id": 4, "kernel_name": "cutlass3x_sm120_bstensorop_s16864gemm_block_scaled_ue4m3xe2m1_ue4m3xe2m1_f32_bf16_bf16_128x128x256_1x1x1_0_tnn_align32_o_vs16_bias_bf16_relu", "grid_size": "(296, 42, 1)", "block_size": "(384, 1, 1)", "duration_ns": 16198976.0, "registers_per_thread": 168.0, "achieved_occupancy_percent": 20.83, "eligible_warps_per_scheduler": 0.23, "issue_active_percent": 17.44, "sm_throughput_percent": 85.88, "tensor_pipe_active_percent": 85.88, "l2_requested_bytes": 26072923840.0, "l2_hit_rate_percent": 91.06, "l2_throughput_percent": 81.58, "memory_throughput_percent": 77.65, "local_spilling_requests": 0.0, "top_scheduler_stalls": [ { "reason": "math_pipe_throttle", "warps_per_issue_active": 4.06 }, { "reason": "wait", "warps_per_issue_active": 3.98 }, { "reason": "sleeping", "warps_per_issue_active": 3.81 }, { "reason": "selected", "warps_per_issue_active": 1.0 }, { "reason": "long_scoreboard", "warps_per_issue_active": 0.6 } ] } }, "units": { "duration_ns": "ns", "l2_requested_bytes": "lts__t_bytes.sum", "throughput_and_hit_rate": "%" } }, "comparison_to_pre_fc2_profile": { "previous_summary": "benchmarks/gb10-fully-fused-fresh-nsight-summary.json", "block_24_p50_ms": { "previous": 458.7751985236537, "current": 427.4100805005219, "absolute_change": -31.365118023131856, "percent_change": -6.83670741663136 }, "block_24_comparison_note": "same-script decomposition control only; it excludes guarded FC2 and must not be interpreted as the production FC2 gain", "warmed_step_gpu_span_seconds": { "previous": 23.706977664, "current": 20.90717264, "absolute_change": -2.799805024000001, "percent_change": -11.810046239051452 }, "warmed_step_kernel_time_seconds": { "previous": 23.693982688, "current": 20.895065376, "absolute_change": -2.798917311999997, "percent_change": -11.812776893002164 }, "warmed_step_kernel_count": { "previous": 2694, "current": 2744, "absolute_change": 50, "percent_change": 1.8559762435040872 }, "components": { "sage2": { "previous": 13760.566304, "current": 13029.23856, "absolute_change": -731.3277440000002, "percent_change": -5.314663131178065 }, "nvfp4_gemms": { "previous": 6269.804512, "current": 4167.855808, "absolute_change": -2101.9487039999995, "percent_change": -33.52494802632212 }, "nvfp4_packing": { "previous": 2093.052672, "current": 2026.853664, "absolute_change": -66.19900799999982, "percent_change": -3.1627970421185814 }, "norm_and_rope": { "previous": 987.609696, "current": 1084.3856, "absolute_change": 96.77590400000008, "percent_change": 9.79900302639396 }, "remaining_gate_add": { "previous": 545.150976, "current": 549.298592, "absolute_change": 4.147615999999971, "percent_change": 0.7608196963037273 }, "other": { "previous": 37.798528, "current": 37.433152, "absolute_change": -0.3653759999999977, "percent_change": -0.966640817335529 } } }, "bottleneck_ranking": [ { "component": "sage2", "milliseconds": 13029.23856, "percent_of_kernel_time": 62.355576905566124 }, { "component": "nvfp4_gemms", "milliseconds": 4167.855808, "percent_of_kernel_time": 19.946603339117498 }, { "component": "nvfp4_packing", "milliseconds": 2026.853664, "percent_of_kernel_time": 9.700154689767265 }, { "component": "norm_and_rope", "milliseconds": 1084.3856, "percent_of_kernel_time": 5.1896731619970025 }, { "component": "remaining_gate_add", "milliseconds": 549.298592, "percent_of_kernel_time": 2.6288436150619683 }, { "component": "other", "milliseconds": 37.433152, "percent_of_kernel_time": 0.17914828849014078 } ], "decision": { "authoritative_exact_baseline": "255.446994 s median resident sampling", "time_bottleneck": "Sage2 remains dominant at 62.36% of warmed-step kernel time; its mainloop is 241.91 ms average in NSYS and 259.00 ms in the NCU replay.", "secondary_bottlenecks": "NVFP4 GEMMs are 19.95%, packing 9.70%, norm/RoPE 5.19%, and gate/add 2.63%.", "next_optimization": "None started. Sage2 is the next-ranked investigation target; any implementation requires a separate approved experiment after this baseline is accepted." }, "artifacts": [ "benchmarks/gb10-post-fc2-resident-baseline-20260826.json", "benchmarks/gb10-post-fc2-block24-profile-20260826.json", "benchmarks/gb10-post-fc2-warmed-step-capture-20260826.json", "benchmarks/gb10-post-fc2-warmed-step-20260826.nsys-rep", "benchmarks/gb10-post-fc2-warmed-step-20260826.sqlite", "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_trace.csv", "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_kern_exec_trace.csv", "benchmarks/gb10-post-fc2-warmed-step-stats_nvtx_gpu_proj_trace.csv", "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_gpu_kern_sum.csv", "benchmarks/gb10-post-fc2-warmed-step-stats_cuda_api_sum.csv", "benchmarks/gb10-post-fc2-block24-targeted-capture-20260826.json", "benchmarks/gb10-post-fc2-block24-targeted-20260826.ncu-rep", "benchmarks/gb10-post-fc2-block24-targeted-20260826.csv", "benchmarks/gb10-post-fc2-block24-targeted-traffic-capture-20260826.json", "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.ncu-rep", "benchmarks/gb10-post-fc2-block24-targeted-traffic-20260826.csv", "benchmarks/gb10-post-fc2-warmed-step-nsys-summary-20260826.json", "benchmarks/gb10-post-fc2-block24-targeted-summary-20260826.json" ] }