306 lines
7.9 KiB
JSON
306 lines
7.9 KiB
JSON
|
{
|
||
|
|
"metadata": {
|
||
|
|
"width": 1344,
|
||
|
|
"height": 768,
|
||
|
|
"frames": 124,
|
||
|
|
"steps": 12,
|
||
|
|
"sampler_step": 1,
|
||
|
|
"seed": 440420,
|
||
|
|
"text_tokens": 100,
|
||
|
|
"block_index": 24,
|
||
|
|
"attention": "sage2",
|
||
|
|
"hidden_shape": [
|
||
|
|
37810,
|
||
|
|
5376
|
||
|
|
],
|
||
|
|
"h_msa_shape": [
|
||
|
|
37810,
|
||
|
|
5376
|
||
|
|
],
|
||
|
|
"segments": [
|
||
|
|
[
|
||
|
|
0,
|
||
|
|
100,
|
||
|
|
1
|
||
|
|
],
|
||
|
|
[
|
||
|
|
100,
|
||
|
|
514,
|
||
|
|
2
|
||
|
|
],
|
||
|
|
[
|
||
|
|
514,
|
||
|
|
37810,
|
||
|
|
0
|
||
|
|
]
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"q_shape": [
|
||
|
|
1,
|
||
|
|
37810,
|
||
|
|
56,
|
||
|
|
128
|
||
|
|
],
|
||
|
|
"k_shape": [
|
||
|
|
1,
|
||
|
|
37810,
|
||
|
|
56,
|
||
|
|
128
|
||
|
|
],
|
||
|
|
"v_shape": [
|
||
|
|
1,
|
||
|
|
37810,
|
||
|
|
56,
|
||
|
|
128
|
||
|
|
],
|
||
|
|
"warmup": 3,
|
||
|
|
"iterations": 10,
|
||
|
|
"phase_timings": {
|
||
|
|
"k_mean_and_smoothing_preparation": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.002420537567138672,
|
||
|
|
"p50_s": 0.0023651360273361207,
|
||
|
|
"p90_s": 0.0025749952793121334,
|
||
|
|
"p95_s": 0.0026686816215515134,
|
||
|
|
"p99_s": 0.0027436306953430175,
|
||
|
|
"min_s": 0.002331295967102051,
|
||
|
|
"max_s": 0.0027623679637908934
|
||
|
|
},
|
||
|
|
"qk_int8_quantization": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.007748214387893677,
|
||
|
|
"p50_s": 0.0076711041927337645,
|
||
|
|
"p90_s": 0.007954323053359986,
|
||
|
|
"p95_s": 0.007966649413108826,
|
||
|
|
"p99_s": 0.007976510500907898,
|
||
|
|
"min_s": 0.007604640007019043,
|
||
|
|
"max_s": 0.007978975772857666
|
||
|
|
},
|
||
|
|
"q_int8_quantization": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.0037990527868270877,
|
||
|
|
"p50_s": 0.0037993919849395753,
|
||
|
|
"p90_s": 0.0038029504060745237,
|
||
|
|
"p95_s": 0.0038038431525230405,
|
||
|
|
"p99_s": 0.003804557349681854,
|
||
|
|
"min_s": 0.003791712045669556,
|
||
|
|
"max_s": 0.0038047358989715574
|
||
|
|
},
|
||
|
|
"k_int8_subtract_mean_quantization": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.0038265248060226436,
|
||
|
|
"p50_s": 0.0038259359598159792,
|
||
|
|
"p90_s": 0.0038348320722579954,
|
||
|
|
"p95_s": 0.0038351920247077943,
|
||
|
|
"p99_s": 0.003835479986667633,
|
||
|
|
"min_s": 0.003819711923599243,
|
||
|
|
"max_s": 0.0038355519771575927
|
||
|
|
},
|
||
|
|
"v_fp8_transpose_scale_quantization": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.011513513278961182,
|
||
|
|
"p50_s": 0.010691807746887208,
|
||
|
|
"p90_s": 0.011705465221405026,
|
||
|
|
"p95_s": 0.015195276212692253,
|
||
|
|
"p99_s": 0.017987125005722047,
|
||
|
|
"min_s": 0.010680031776428223,
|
||
|
|
"max_s": 0.018685087203979493
|
||
|
|
},
|
||
|
|
"v_transpose_pad_permute": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.0051148447513580325,
|
||
|
|
"p50_s": 0.005081151962280274,
|
||
|
|
"p90_s": 0.005127699327468873,
|
||
|
|
"p95_s": 0.005270201659202576,
|
||
|
|
"p99_s": 0.005384203524589539,
|
||
|
|
"min_s": 0.005073599815368653,
|
||
|
|
"max_s": 0.00541270399093628
|
||
|
|
},
|
||
|
|
"v_scale_fp8_quantization": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.005551833629608155,
|
||
|
|
"p50_s": 0.0055316162109375,
|
||
|
|
"p90_s": 0.0055659454822540285,
|
||
|
|
"p95_s": 0.005644108700752258,
|
||
|
|
"p99_s": 0.005706639275550842,
|
||
|
|
"min_s": 0.00552732801437378,
|
||
|
|
"max_s": 0.005722271919250488
|
||
|
|
},
|
||
|
|
"fused_mainloop": {
|
||
|
|
"count": 10,
|
||
|
|
"mean_s": 0.23669681091308595,
|
||
|
|
"p50_s": 0.23708939361572268,
|
||
|
|
"p90_s": 0.23775543670654295,
|
||
|
|
"p95_s": 0.2378476737976074,
|
||
|
|
"p99_s": 0.23792146347045898,
|
||
|
|
"min_s": 0.23529600524902344,
|
||
|
|
"max_s": 0.23793991088867186
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fused_mainloop_phases": {
|
||
|
|
"int8_qk": "fused inside qk_int_sv_f8_attn_kernel",
|
||
|
|
"scale_application": "fused inside qk_int_sv_f8_attn_kernel",
|
||
|
|
"online_softmax": "fused inside qk_int_sv_f8_attn_kernel",
|
||
|
|
"pv_accumulation": "fused inside qk_int_sv_f8_attn_kernel",
|
||
|
|
"final_normalization_and_output": "fused inside qk_int_sv_f8_attn_kernel",
|
||
|
|
"timing_policy": "Do not assign independent wall time without changing the exact kernel schedule; use source-correlated hardware counters."
|
||
|
|
},
|
||
|
|
"scheduler": {
|
||
|
|
"cta_q": 128,
|
||
|
|
"cta_k": 64,
|
||
|
|
"warp_q": 32,
|
||
|
|
"warp_k": 64,
|
||
|
|
"warps_per_cta": 4,
|
||
|
|
"threads_per_cta": 128,
|
||
|
|
"dynamic_shared_memory_bytes": 32768,
|
||
|
|
"q_ctas_per_head": 296,
|
||
|
|
"heads": 56,
|
||
|
|
"grid_ctas": 16576,
|
||
|
|
"k_iterations_per_cta": 591,
|
||
|
|
"q_tail_rows": 50,
|
||
|
|
"k_tail_rows": 50,
|
||
|
|
"explicit_pipeline_stages": 2
|
||
|
|
},
|
||
|
|
"manual_decomposition_vs_public_sage2": {
|
||
|
|
"equal": true,
|
||
|
|
"max_abs": 0.0,
|
||
|
|
"mean_abs": 0.0
|
||
|
|
},
|
||
|
|
"manual_checksum": 9190064.0,
|
||
|
|
"reference_checksum": 9190064.0,
|
||
|
|
"tail_study": [
|
||
|
|
{
|
||
|
|
"sweep": "q_tail_fixed_kv",
|
||
|
|
"q_len": 37760,
|
||
|
|
"kv_len": 37810,
|
||
|
|
"q_ctas_per_head": 295,
|
||
|
|
"k_iterations_per_cta": 591,
|
||
|
|
"q_tail_rows": 0,
|
||
|
|
"k_tail_rows": 50,
|
||
|
|
"scheduled_q_rows": 37760,
|
||
|
|
"q_row_efficiency": 1.0,
|
||
|
|
"mainloop": {
|
||
|
|
"count": 5,
|
||
|
|
"mean_s": 0.2392295654296875,
|
||
|
|
"p50_s": 0.23901161193847656,
|
||
|
|
"p90_s": 0.24024520874023436,
|
||
|
|
"p95_s": 0.24033575439453123,
|
||
|
|
"p99_s": 0.24040819091796872,
|
||
|
|
"min_s": 0.23829190063476563,
|
||
|
|
"max_s": 0.24042630004882812
|
||
|
|
},
|
||
|
|
"checksum": 9183940.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"sweep": "q_tail_fixed_kv",
|
||
|
|
"q_len": 37761,
|
||
|
|
"kv_len": 37810,
|
||
|
|
"q_ctas_per_head": 296,
|
||
|
|
"k_iterations_per_cta": 591,
|
||
|
|
"q_tail_rows": 1,
|
||
|
|
"k_tail_rows": 50,
|
||
|
|
"scheduled_q_rows": 37888,
|
||
|
|
"q_row_efficiency": 0.9966480152027027,
|
||
|
|
"mainloop": {
|
||
|
|
"count": 5,
|
||
|
|
"mean_s": 0.23963267211914063,
|
||
|
|
"p50_s": 0.23961180114746095,
|
||
|
|
"p90_s": 0.23997588500976563,
|
||
|
|
"p95_s": 0.24002579956054687,
|
||
|
|
"p99_s": 0.2400657312011719,
|
||
|
|
"min_s": 0.23915267944335938,
|
||
|
|
"max_s": 0.24007571411132814
|
||
|
|
},
|
||
|
|
"checksum": 9183946.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"sweep": "q_tail_fixed_kv",
|
||
|
|
"q_len": 37810,
|
||
|
|
"kv_len": 37810,
|
||
|
|
"q_ctas_per_head": 296,
|
||
|
|
"k_iterations_per_cta": 591,
|
||
|
|
"q_tail_rows": 50,
|
||
|
|
"k_tail_rows": 50,
|
||
|
|
"scheduled_q_rows": 37888,
|
||
|
|
"q_row_efficiency": 0.9979413006756757,
|
||
|
|
"mainloop": {
|
||
|
|
"count": 5,
|
||
|
|
"mean_s": 0.23999984436035157,
|
||
|
|
"p50_s": 0.23978463745117187,
|
||
|
|
"p90_s": 0.24042403869628906,
|
||
|
|
"p95_s": 0.24055045166015623,
|
||
|
|
"p99_s": 0.24065158203124998,
|
||
|
|
"min_s": 0.2397151641845703,
|
||
|
|
"max_s": 0.24067686462402343
|
||
|
|
},
|
||
|
|
"checksum": 9190064.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"sweep": "kv_tail_fixed_q",
|
||
|
|
"q_len": 37810,
|
||
|
|
"kv_len": 37760,
|
||
|
|
"q_ctas_per_head": 296,
|
||
|
|
"k_iterations_per_cta": 590,
|
||
|
|
"q_tail_rows": 50,
|
||
|
|
"k_tail_rows": 0,
|
||
|
|
"scheduled_q_rows": 37888,
|
||
|
|
"q_row_efficiency": 0.9979413006756757,
|
||
|
|
"mainloop": {
|
||
|
|
"count": 5,
|
||
|
|
"mean_s": 0.23982306518554686,
|
||
|
|
"p50_s": 0.23983763122558593,
|
||
|
|
"p90_s": 0.2401393035888672,
|
||
|
|
"p95_s": 0.24022885437011718,
|
||
|
|
"p99_s": 0.2403004949951172,
|
||
|
|
"min_s": 0.2395267791748047,
|
||
|
|
"max_s": 0.24031840515136718
|
||
|
|
},
|
||
|
|
"checksum": 9175197.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"sweep": "kv_tail_fixed_q",
|
||
|
|
"q_len": 37810,
|
||
|
|
"kv_len": 37761,
|
||
|
|
"q_ctas_per_head": 296,
|
||
|
|
"k_iterations_per_cta": 591,
|
||
|
|
"q_tail_rows": 50,
|
||
|
|
"k_tail_rows": 1,
|
||
|
|
"scheduled_q_rows": 37888,
|
||
|
|
"q_row_efficiency": 0.9979413006756757,
|
||
|
|
"mainloop": {
|
||
|
|
"count": 5,
|
||
|
|
"mean_s": 0.2405945220947266,
|
||
|
|
"p50_s": 0.24039689636230469,
|
||
|
|
"p90_s": 0.24119658203125,
|
||
|
|
"p95_s": 0.24143297729492186,
|
||
|
|
"p99_s": 0.24162209350585936,
|
||
|
|
"min_s": 0.2401432342529297,
|
||
|
|
"max_s": 0.24166937255859375
|
||
|
|
},
|
||
|
|
"checksum": 9176321.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"sweep": "kv_tail_fixed_q",
|
||
|
|
"q_len": 37810,
|
||
|
|
"kv_len": 37810,
|
||
|
|
"q_ctas_per_head": 296,
|
||
|
|
"k_iterations_per_cta": 591,
|
||
|
|
"q_tail_rows": 50,
|
||
|
|
"k_tail_rows": 50,
|
||
|
|
"scheduled_q_rows": 37888,
|
||
|
|
"q_row_efficiency": 0.9979413006756757,
|
||
|
|
"mainloop": {
|
||
|
|
"count": 5,
|
||
|
|
"mean_s": 0.24062385864257813,
|
||
|
|
"p50_s": 0.24038169860839845,
|
||
|
|
"p90_s": 0.24108528442382812,
|
||
|
|
"p95_s": 0.2411941619873047,
|
||
|
|
"p99_s": 0.24128126403808592,
|
||
|
|
"min_s": 0.2403317108154297,
|
||
|
|
"max_s": 0.24130303955078125
|
||
|
|
},
|
||
|
|
"checksum": 9190064.0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|