From d1cdd4b6b3e34b3a7a2b8f59f84594e713e38b4c Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Fri, 14 Aug 2026 20:02:14 +0700 Subject: [PATCH] Record Sage3 cat benchmark --- PLAN.md | 11 +++++++++++ benchmarks/ref2va-960x544-124f.json | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/PLAN.md b/PLAN.md index 0ee46d4..3400ae1 100644 --- a/PLAN.md +++ b/PLAN.md @@ -48,6 +48,17 @@ The direct runner must first match the model contract and output quality. Beatin 5. Implement ragged Ulysses Sage3 with transport-identity and distributed-versus-single-Sage3 tests. 6. Sweep Ulysses/tensor-parallel layouts on 2/4/6/8 GPUs in an NVLink/NVSwitch domain. +## Performance Backend Plan + +Prompt-only FL2VA is now at warm Comfy parity with the direct Sage2 baseline. Feature and performance work should proceed in this order: + +1. Validate and benchmark the existing `sage3` backend against the same cat prompt, seed, dimensions, and FP16 VAE runtime path used for Sage2 parity. + - First cat benchmark result: Sage3 runs successfully but is slower than Sage2 in this direct path. Sampling was `123.675s` versus Sage2 `114.414s`; warm after text conditioning was `158.111s` versus Sage2 `149.304s`. Same-seed MP4 frame diff versus Sage2 was mean `46.563`, max `255`, so keep Sage3 experimental pending human visual review and stricter tensor gates. +2. Add exact memory/lifetime optimizations next: `kj_head_sliced` and `kj_chunked_ffn`. These must preserve the validated direct outputs before being kept. +3. Evaluate prior H3-tested attention candidates as standalone adapters: `sol_attn` and `kj_sage`. +4. Evaluate approximate denoiser caches only after exact baselines are recorded: `easycache` and `h3_cache`. +5. Keep every backend explicit per run, with separate quality and timing records for sampling, VAE, audio, and end-to-end output. + ## Non-Negotiable Validation - Never silently pad semantic H3 tokens for unmasked attention. diff --git a/benchmarks/ref2va-960x544-124f.json b/benchmarks/ref2va-960x544-124f.json index 1fdc159..2bd4e9a 100644 --- a/benchmarks/ref2va-960x544-124f.json +++ b/benchmarks/ref2va-960x544-124f.json @@ -33,5 +33,20 @@ "direct_log": "\\\\192.168.1.162\\StoryStudioAssets\\H3-output\\h3-blackwell-runtime\\benchmarks\\direct-cat-benchmark-960x544-124f-seed440407-fp16-vae.log", "comfy_output": "\\\\192.168.1.162\\StoryStudioAssets\\H3-output\\h3-blackwell-runtime\\comfy-cat-benchmark-960x544-124f-seed440407_00001_.mp4", "comfy_metrics": "\\\\192.168.1.162\\StoryStudioAssets\\H3-output\\h3-blackwell-runtime\\benchmarks\\comfy-cat-benchmark-960x544-124f-seed440407.json" + }, + "prompt_only_fl2va_cat_sage3_benchmark": { + "seed": 440407, + "attention_backend": "sage3", + "direct_cold_through_audio_decode_seconds": 184.646, + "direct_warm_after_text_conditioned_seconds": 158.111, + "direct_warm_including_text_conditioning_seconds": 160.38, + "direct_sample_seconds": 123.675, + "direct_video_vae_dtype": "float16", + "direct_video_vae_tile_size": 256, + "direct_video_vae_decode_seconds": 25.024, + "sage2_mp4_frame_diff_max": 255, + "sage2_mp4_frame_diff_mean": 46.56300230273561, + "direct_output": "\\\\192.168.1.162\\StoryStudioAssets\\H3-output\\h3-blackwell-runtime\\direct-cat-benchmark-960x544-124f-seed440407-sage3-fp16-vae.mp4", + "direct_log": "\\\\192.168.1.162\\StoryStudioAssets\\H3-output\\h3-blackwell-runtime\\benchmarks\\direct-cat-benchmark-960x544-124f-seed440407-sage3-fp16-vae.log" } }