From 6d8a1c3c2b69c3577dce9f0982b0f035f2546b23 Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Sat, 22 Aug 2026 15:38:09 +0700 Subject: [PATCH] Record RTX Pro 6000 SDPA baseline --- CURRENT_STATE.md | 3 + DISTRIBUTED.md | 18 +++++- README.md | 6 ++ ...0-sdpa-864x480-141f-base12-seed440420.json | 58 +++++++++++++++++++ tools/runpod_api.py | 36 ++++++++++-- 5 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 benchmarks/rtxpro6000-server-vs-gb10-sdpa-864x480-141f-base12-seed440420.json diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index 8afd1bf..6d10417 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -29,6 +29,9 @@ Historical handoffs in `PLAN.md` and `PARITY.md` may describe older states. - Automatic visible-GPU launchers and 1/2/4/6/8 benchmark matrix tooling. - Real-checkpoint one-rank Ulysses-versus-TP identity at 864x480, 141 frames, and 12 steps, including exact video and audio latent equality. +- Matched one-GPU RTX PRO 6000 Blackwell Server SDPA sampling averaged `28.50s` + over two runs versus `126.66s` for the same tensor runner on GB10 (`4.44x`). + RTX repeat variance was `0.49%` and checksums were identical between repeats. ## Primary Missing Scope diff --git a/DISTRIBUTED.md b/DISTRIBUTED.md index 1d4d35d..6dae89c 100644 --- a/DISTRIBUTED.md +++ b/DISTRIBUTED.md @@ -107,6 +107,14 @@ After choosing an available data center, create the pod explicitly: python .\tools\runpod_api.py create --count 8 --datacenter US-XX-N --yes ``` +Attach an existing network volume and inject an SSH key when required: + +```powershell +python .\tools\runpod_api.py create --count 1 --datacenter EUR-IS-1 ` + --network-volume VOLUME_ID --volume-mount-path /runpod-volume ` + --ssh-public-key $HOME\.ssh\id_ed25519.pub --yes +``` + The client defaults to the server-edition RTX PRO 6000 Blackwell and RunPod's x86_64 CUDA 13.0, Torch 2.9.1 image. It creates persistent workspace storage and enables SSH. Creation and termination require `--yes` to avoid accidental spend @@ -126,9 +134,17 @@ SageAttention2 is not currently packaged in the generic x86 image, so cloud correctness and scaling start with SDPA. Sage2 can be measured after an x86 wheel is added without changing the distributed layout. +The first matched one-GPU Server Edition run completed on CUDA 13.2 and Torch +2.9.1+cu130. At 864x480, 141 frames, 12 steps, seed 440420, two SDPA sampling +runs took `28.43s` and `28.57s` (mean `28.50s`). The same tensor runner took +`126.66s` on GB10, making the RTX PRO 6000 `4.44x` faster. Peak allocated memory +during RTX sampling was 14,049,528,832 bytes. Both RTX repeats produced identical +checksums; cross-device latent parity was not tested because no latent was saved. + ## Remaining GPU Gates - Real NCCL identity at 2, 4, 6, and 8 GPUs. - Distributed-versus-single latent parity above one rank. -- Full 1/2/4/6/8 timing, transport, and memory reports. +- Full 2/4/6/8 timing, transport, and memory reports; the one-GPU SDPA baseline + is complete. - Quality comparison after selecting an x86 attention backend. diff --git a/README.md b/README.md index ff82958..f6fe735 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,12 @@ NVFP4 shard layout, and 2/6-rank TP attention/MLP math. On GB10, real-checkpoint one-rank Ulysses and tensor paths produced exactly equal video and audio latents at 864x480, 141 frames, 12 steps, seed 440420. +A matched one-GPU RunPod RTX PRO 6000 Blackwell Server SDPA test averaged +`28.50s` sampling over two runs versus `126.66s` for the same tensor runner on +GB10, a `4.44x` speedup. The two RTX runs were within `0.49%` and produced +identical checksums. See +`benchmarks/rtxpro6000-server-vs-gb10-sdpa-864x480-141f-base12-seed440420.json`. + Matched GB10 backend results and parity evidence are recorded in [`PLAN.md`](PLAN.md) and [`PARITY.md`](PARITY.md). Standalone `tools/compare_*`, `tools/trace_*`, `tools/inspect_*`, and `tools/patch_comfy_*` commands are diagnostic utilities, diff --git a/benchmarks/rtxpro6000-server-vs-gb10-sdpa-864x480-141f-base12-seed440420.json b/benchmarks/rtxpro6000-server-vs-gb10-sdpa-864x480-141f-base12-seed440420.json new file mode 100644 index 0000000..f41471b --- /dev/null +++ b/benchmarks/rtxpro6000-server-vs-gb10-sdpa-864x480-141f-base12-seed440420.json @@ -0,0 +1,58 @@ +{ + "benchmark": "t2va-dialogue-quoted-864x480-141f-base12-sage2-seed440420.json", + "actual_attention": "sdpa", + "mode": "tensor", + "world_size": 1, + "resolution": [ + 864, + 480 + ], + "frames": 141, + "steps": 12, + "seed": 440420, + "rtx_pro_6000_blackwell_server": { + "cloud": "RunPod Secure", + "data_center": "EUR-IS-1", + "gpu_memory_mib": 97887, + "driver": "595.91.07", + "host_cuda": "13.2", + "torch": "2.9.1+cu130", + "hourly_usd": 2.09, + "sampling_seconds": [ + 28.431582752993563, + 28.57024686699151 + ], + "sampling_mean_seconds": 28.500914809992537, + "sampling_range_percent": 0.48652513409615622, + "model_load_seconds": [ + 4.332073616009438, + 4.819102452020161 + ], + "conditioning_seconds": [ + 8.492386644007638, + 8.663792312989244 + ], + "peak_sampling_allocated_bytes": 14049528832, + "checksums": [ + 98329.34375, + -345.9080505371094 + ], + "persistent_reports": [ + "/runpod-volume/h3-benchmarks/rtxpro6000-server-1gpu-sdpa-864x480-141f-base12-seed440420.json", + "/runpod-volume/h3-benchmarks/rtxpro6000-server-1gpu-sdpa-864x480-141f-base12-seed440420-repeat2.json" + ] + }, + "gb10": { + "direct_sdpa_sampling_seconds": 125.3, + "same_tensor_runner_sampling_seconds": 126.66326454500086 + }, + "speedup": { + "versus_gb10_direct_sdpa": 4.396350111403066, + "versus_gb10_same_tensor_runner": 4.444182419737356 + }, + "notes": [ + "Both RTX PRO 6000 runs produced identical checksums.", + "No latent was retained, so this is a performance and execution-stability gate rather than a cross-device numerical parity gate.", + "The billable pod was terminated after the repeat run; the network volume and reports were preserved." + ] +} diff --git a/tools/runpod_api.py b/tools/runpod_api.py index ae4160a..8d48fcc 100644 --- a/tools/runpod_api.py +++ b/tools/runpod_api.py @@ -7,6 +7,7 @@ import sys import urllib.error import urllib.parse import urllib.request +from pathlib import Path API = "https://api.runpod.io/v2" @@ -34,7 +35,8 @@ def request(method: str, path: str, body=None, query=None): call.add_header("Content-Type", "application/json") try: with urllib.request.urlopen(call, timeout=60) as response: - return json.load(response) + payload = response.read() + return json.loads(payload) if payload else {"status": response.status} except urllib.error.HTTPError as error: detail = error.read().decode("utf-8", errors="replace") raise SystemExit(f"RunPod API returned HTTP {error.code}: {detail}") from error @@ -55,6 +57,9 @@ create_parser.add_argument("--datacenter") create_parser.add_argument("--image", default=DEFAULT_IMAGE) create_parser.add_argument("--disk", type=int, default=100) create_parser.add_argument("--volume", type=int, default=100) +create_parser.add_argument("--network-volume") +create_parser.add_argument("--volume-mount-path", default="/runpod-volume") +create_parser.add_argument("--ssh-public-key", type=Path) create_parser.add_argument("--yes", action="store_true") get_parser = commands.add_parser("get") @@ -64,6 +69,10 @@ terminate_parser = commands.add_parser("terminate") terminate_parser.add_argument("pod_id") terminate_parser.add_argument("--yes", action="store_true") +ssh_keys_parser = commands.add_parser("ssh-keys") +ssh_keys_parser.add_argument("--replace-with", type=Path) +ssh_keys_parser.add_argument("--yes", action="store_true") + args = parser.parse_args() if args.command == "catalog": response = request("GET", "/catalog/gpus", query={ @@ -84,18 +93,37 @@ elif args.command == "create": "cloud": args.cloud, "disk": args.disk, "ports": ["22/tcp"], - "mounts": {"persistent": {"size": args.volume, "path": "/workspace"}}, "startSsh": True, } + if args.network_volume: + body["mounts"] = { + "network": [{"volumeId": args.network_volume, "path": args.volume_mount_path}], + } + else: + body["mounts"] = { + "persistent": {"size": args.volume, "path": args.volume_mount_path}, + } + if args.ssh_public_key: + body["env"] = { + "PUBLIC_KEY": args.ssh_public_key.read_text(encoding="utf-8").strip(), + } if args.datacenter: body["dataCenterIds"] = [args.datacenter] response = request("POST", "/pods", body=body) elif args.command == "get": response = request("GET", f"/pods/{args.pod_id}") -else: +elif args.command == "terminate": if not args.yes: raise SystemExit("termination is irreversible; repeat with --yes") - response = request("POST", f"/pods/{args.pod_id}/actions", body={"action": "terminate"}) + response = request("POST", f"/pods/{args.pod_id}/action", body={"action": "terminate"}) +else: + if args.replace_with: + if not args.yes: + raise SystemExit("replacing account SSH keys requires --yes") + public_key = args.replace_with.read_text(encoding="utf-8").strip() + response = request("PUT", "/account/ssh-keys", body={"keys": [public_key]}) + else: + response = request("GET", "/account/ssh-keys") json.dump(response, sys.stdout, indent=2) sys.stdout.write("\n")