From bee998d8403aea18f9636b3773a8e7719ff3817c Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Sat, 22 Aug 2026 16:42:09 +0700 Subject: [PATCH] Record RTX Pro 6000 multi-GPU scaling --- CURRENT_STATE.md | 6 +- DISTRIBUTED.md | 10 +- README.md | 7 + ...00-server-1v2-sdpa-scaling-seed440420.json | 122 ++++++++++++++++++ .../rtxpro6000-server-2gpu-topology.txt | 46 +++++++ ...-1344x768-124f-base12-sdpa-seed440420.json | 12 ++ ...-1344x768-243f-base12-sdpa-seed440420.json | 12 ++ 7 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 benchmarks/rtxpro6000-server-1v2-sdpa-scaling-seed440420.json create mode 100644 benchmarks/rtxpro6000-server-2gpu-topology.txt create mode 100644 benchmarks/t2va-dialogue-quoted-1344x768-124f-base12-sdpa-seed440420.json create mode 100644 benchmarks/t2va-dialogue-quoted-1344x768-243f-base12-sdpa-seed440420.json diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index 6d10417..0eda2c2 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -32,6 +32,10 @@ Historical handoffs in `PLAN.md` and `PARITY.md` may describe older states. - Matched one-GPU RTX PRO 6000 Blackwell Server SDPA sampling averaged `28.50s` over two runs versus `126.66s` for the same tensor runner on GB10 (`4.44x`). RTX repeat variance was `0.49%` and checksums were identical between repeats. +- Two-GPU Ulysses SDPA speedup grows with sequence size: `1.21x` at + 864x480/141 frames, `1.55x` at 1344x768/124 frames, and `1.72x` at + 1344x768/243 frames. The tested cards have no NVLink; P2P read/write is + available and NCCL uses `P2P/CUMEM`. ## Primary Missing Scope @@ -51,7 +55,7 @@ Historical handoffs in `PLAN.md` and `PARITY.md` may describe older states. ### Distributed Execution -- Real NCCL transport and output parity above one rank. +- Output parity above one rank; real two-rank NCCL transport is validated. - 2/4/6/8-GPU topology and performance sweeps on one Blackwell machine. - Distributed resident-service orchestration; the current launcher is batch generation through `torchrun`. diff --git a/DISTRIBUTED.md b/DISTRIBUTED.md index 6dae89c..35ecbbf 100644 --- a/DISTRIBUTED.md +++ b/DISTRIBUTED.md @@ -141,9 +141,17 @@ runs took `28.43s` and `28.57s` (mean `28.50s`). The same tensor runner took during RTX sampling was 14,049,528,832 bytes. Both RTX repeats produced identical checksums; cross-device latent parity was not tested because no latent was saved. +Two-GPU SDPA results show that Ulysses is faster than TP at all measured shapes. +Against one GPU, Ulysses reached `1.21x` at 864x480/141 frames, `1.55x` at +1344x768/124 frames, and `1.72x` at 1344x768/243 frames. The cards expose no +NVLink. `nvidia-smi topo -p2p r` and `-p2p w` report `OK` in both directions, +and NCCL 2.27.7 selected `P2P/CUMEM` with GDR enabled and zero NVLS channels. +RunPod allocations varied between cross-NUMA `SYS` and same-NUMA `NODE` +topologies, so topology must be recorded with every scaling result. + ## Remaining GPU Gates -- Real NCCL identity at 2, 4, 6, and 8 GPUs. +- Real NCCL identity at 4, 6, and 8 GPUs; two-GPU transport is complete. - Distributed-versus-single latent parity above one rank. - Full 2/4/6/8 timing, transport, and memory reports; the one-GPU SDPA baseline is complete. diff --git a/README.md b/README.md index f6fe735..51bf678 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,13 @@ GB10, a `4.44x` speedup. The two RTX runs were within `0.49%` and produced identical checksums. See `benchmarks/rtxpro6000-server-vs-gb10-sdpa-864x480-141f-base12-seed440420.json`. +Two-GPU Ulysses scaling improves as the packed sequence grows: `1.21x` at +864x480/141 frames, `1.55x` at 1344x768/124 frames, and `1.72x` at +1344x768/243 frames. These RTX PRO 6000 Server cards have no NVLink. CUDA P2P +read/write is available and NCCL selected `P2P/CUMEM`; topology varied between +`SYS` and `NODE` across RunPod allocations. See +`benchmarks/rtxpro6000-server-1v2-sdpa-scaling-seed440420.json`. + Matched GB10 backend results and parity evidence are recorded in [`PLAN.md`](PLAN.md) and [`PARITY.md`](PARITY.md). Standalone `tools/compare_*`, `tools/trace_*`, `tools/inspect_*`, and `tools/patch_comfy_*` commands are diagnostic utilities, diff --git a/benchmarks/rtxpro6000-server-1v2-sdpa-scaling-seed440420.json b/benchmarks/rtxpro6000-server-1v2-sdpa-scaling-seed440420.json new file mode 100644 index 0000000..0240fcc --- /dev/null +++ b/benchmarks/rtxpro6000-server-1v2-sdpa-scaling-seed440420.json @@ -0,0 +1,122 @@ +{ + "device": "NVIDIA RTX PRO 6000 Blackwell Server Edition", + "attention": "sdpa", + "steps": 12, + "seed": 440420, + "hourly_usd": { + "one_gpu": 2.09, + "two_gpu": 4.18 + }, + "results": [ + { + "resolution": [ + 864, + 480 + ], + "frames": 141, + "video_seconds_at_24fps": 5.875, + "one_gpu": { + "mode": "tensor", + "sampling_seconds": 28.500914809992537, + "sampling_cost_usd": 0.016546364431356778 + }, + "two_gpu": { + "mode": "ulysses", + "sampling_seconds": 23.494563594460487, + "sampling_cost_usd": 0.027279798840234674, + "speedup": 1.2130855163750494, + "scaling_efficiency": 0.6065427581875247, + "cost_premium": 0.6486883842916646 + } + }, + { + "resolution": [ + 1344, + 768 + ], + "frames": 124, + "video_seconds_at_24fps": 5.166666666666667, + "one_gpu": { + "mode": "tensor", + "sampling_seconds": 97.94385590963066, + "sampling_cost_usd": 0.05686184968086891 + }, + "two_gpu": { + "mode": "ulysses", + "sampling_seconds": 63.132430186495185, + "sampling_cost_usd": 0.07330376616098608, + "speedup": 1.5514032268408078, + "scaling_efficiency": 0.7757016134204039, + "cost_premium": 0.2891554983243718 + } + }, + { + "resolution": [ + 1344, + 768 + ], + "frames": 243, + "video_seconds_at_24fps": 10.125, + "one_gpu": { + "mode": "tensor", + "sampling_seconds": 311.01632468774915, + "sampling_cost_usd": 0.18056225516594326 + }, + "two_gpu": { + "mode": "ulysses", + "sampling_seconds": 180.68109439313412, + "sampling_cost_usd": 0.2097908262675835, + "speedup": 1.7213551076407902, + "scaling_efficiency": 0.8606775538203951, + "cost_premium": 0.16187531040071534 + } + } + ], + "two_gpu_mode_comparison": [ + { + "resolution": [ + 864, + 480 + ], + "frames": 141, + "ulysses_sampling_seconds": 23.946259677002672, + "tensor_sampling_seconds": 28.037104761999217 + }, + { + "resolution": [ + 1344, + 768 + ], + "frames": 124, + "ulysses_sampling_seconds": 66.31268315999478, + "tensor_sampling_seconds": 74.59926067800552 + }, + { + "resolution": [ + 1344, + 768 + ], + "frames": 243, + "ulysses_sampling_seconds": 190.96160273600253, + "tensor_sampling_seconds": 203.40585646100226 + } + ], + "topology": { + "benchmark_host": "SYS between GPUs, separate NUMA nodes, no NVLink", + "diagnostic_host": "NODE between GPUs, same NUMA node, no NVLink", + "p2p_read": "OK in both directions", + "p2p_write": "OK in both directions", + "nccl_version": "2.27.7+cuda13.0", + "nccl_transport": "P2P/CUMEM", + "nccl_gdr": true, + "nccl_nvls_channels": 0, + "nccl_log": "/runpod-volume/h3-benchmarks/rtxpro6000-server-2gpu-nccl-transport.log" + }, + "cost_scope": "Sampling wall time only. Model/Qwen loading, text conditioning, VAE decode, media encoding, muxing, and pod startup are excluded.", + "notes": [ + "Two-GPU Ulysses timing in the primary results uses the better NODE topology diagnostic host.", + "The separate two-GPU mode comparison was measured on the SYS topology host.", + "Ulysses was faster than tensor parallelism at every tested shape.", + "Cross-mode and cross-device latent parity was not measured because latents were not retained." + ] +} diff --git a/benchmarks/rtxpro6000-server-2gpu-topology.txt b/benchmarks/rtxpro6000-server-2gpu-topology.txt new file mode 100644 index 0000000..f4432ac --- /dev/null +++ b/benchmarks/rtxpro6000-server-2gpu-topology.txt @@ -0,0 +1,46 @@ +Diagnostic host topology (RunPod EUR-IS-1, CUDA 13.0) + +$ nvidia-smi topo -m + GPU0 GPU1 NIC0 NIC1 CPU Affinity NUMA Affinity GPU NUMA ID +GPU0 X NODE SYS SYS 64-127,192-255 1 N/A +GPU1 NODE X SYS SYS 64-127,192-255 1 N/A +NIC0 SYS SYS X PIX +NIC1 SYS SYS PIX X + +$ nvidia-smi topo -p2p r + GPU0 GPU1 +GPU0 X OK +GPU1 OK X + +$ nvidia-smi topo -p2p w + GPU0 GPU1 +GPU0 X OK +GPU1 OK X + +$ nvidia-smi nvlink -s +(no output; no NVLink devices or links exposed) + +NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=INIT,GRAPH,P2P,COLL summary: + +- NCCL version 2.27.7+cuda13.0 +- GPU path classified as PHB at 48.0 GB/s by NCCL topology +- 4 collective channels, 4 P2P channels, 2 P2P channels per peer +- 0 NVLS channels +- P2P chunksize 131072 +- Check P2P Type isAllDirectP2p 1 +- GPU 0 -> GPU 1 via P2P/CUMEM +- GPU 1 -> GPU 0 via P2P/CUMEM +- Connected all rings, PXN 0, GDR 1 + +The first benchmark allocation differed: + +$ nvidia-smi topo -m + GPU0 GPU1 CPU Affinity NUMA Affinity GPU NUMA ID +GPU0 X SYS 0-55,112-167 0 N/A +GPU1 SYS X 56-111,168-223 1 N/A + +$ nvidia-smi nvlink -s +GPU 0: NVIDIA RTX PRO 6000 Blackwell Server Edition +Device does not have or support Nvlink +GPU 1: NVIDIA RTX PRO 6000 Blackwell Server Edition +Device does not have or support Nvlink diff --git a/benchmarks/t2va-dialogue-quoted-1344x768-124f-base12-sdpa-seed440420.json b/benchmarks/t2va-dialogue-quoted-1344x768-124f-base12-sdpa-seed440420.json new file mode 100644 index 0000000..3efd4fa --- /dev/null +++ b/benchmarks/t2va-dialogue-quoted-1344x768-124f-base12-sdpa-seed440420.json @@ -0,0 +1,12 @@ +{ + "name": "t2va-dialogue-quoted-1344x768-124f-base12-sdpa-seed440420", + "mode": "t2va", + "prompt": "Medium two-shot inside a quiet late-night cafe during rain. Warm practical lamps, realistic skin texture, shallow depth of field, restrained natural movement, stable faces, accurate lip sync. A tired woman in a dark green coat looks directly at the man across from her and says, \"I kept the porch light on every night.\" He pauses, meets her eyes, and says, \"I know. I just did not know how to come home.\" Soft rain against the windows, low room tone, faint ceramic cup movement, no music.", + "resolution": [ + 1344, + 768 + ], + "frames": 124, + "steps": 12, + "seed": 440420 +} diff --git a/benchmarks/t2va-dialogue-quoted-1344x768-243f-base12-sdpa-seed440420.json b/benchmarks/t2va-dialogue-quoted-1344x768-243f-base12-sdpa-seed440420.json new file mode 100644 index 0000000..30849e2 --- /dev/null +++ b/benchmarks/t2va-dialogue-quoted-1344x768-243f-base12-sdpa-seed440420.json @@ -0,0 +1,12 @@ +{ + "name": "t2va-dialogue-quoted-1344x768-243f-base12-sdpa-seed440420", + "mode": "t2va", + "prompt": "Medium two-shot inside a quiet late-night cafe during rain. Warm practical lamps, realistic skin texture, shallow depth of field, restrained natural movement, stable faces, accurate lip sync. A tired woman in a dark green coat looks directly at the man across from her and says, \"I kept the porch light on every night.\" He pauses, meets her eyes, and says, \"I know. I just did not know how to come home.\" Soft rain against the windows, low room tone, faint ceramic cup movement, no music.", + "resolution": [ + 1344, + 768 + ], + "frames": 243, + "steps": 12, + "seed": 440420 +}