Make runtime diagnostics opt in
This commit is contained in:
Daniel Maddern
parent
fea69673cd
commit
53cd8bd4f4
5 changed files with 54 additions and 20 deletions
11
README.md
11
README.md
|
|
@ -36,3 +36,14 @@ git pull --ff-only origin master
|
|||
```
|
||||
|
||||
The private key remains on Spark at `~/.ssh/id_ed25519_forgejo_h3`; only its public key is registered in Forgejo.
|
||||
|
||||
## Runtime Output
|
||||
|
||||
Generation and latent-decode tools are quiet by default: they suppress ffmpeg banners and only print compact JSON summaries. Use these flags when debugging:
|
||||
|
||||
- `--progress`: print per-step sampler timing in `tools/direct_t2v_preview.py`.
|
||||
- `--profile-memory`: print memory checkpoints in `tools/direct_t2v_preview.py`.
|
||||
- `--ffmpeg-loglevel info`: show ffmpeg details instead of the default `error` level.
|
||||
- `--quiet`: suppress JSON summary lines.
|
||||
|
||||
Standalone `tools/compare_*`, `tools/trace_*`, `tools/inspect_*`, and `tools/patch_comfy_*` scripts are debugging utilities and remain opt-in by being separate commands.
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ def sample_video_res_multistep(
|
|||
steps: int = 12,
|
||||
model_timesteps: list[torch.Tensor] | tuple[torch.Tensor, ...] | None = None,
|
||||
return_audio: bool = False,
|
||||
progress: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Direct H3 beta/RES sampling with Comfy-equivalent joint AV carry semantics."""
|
||||
sigmas = beta_sigmas(steps, device=video.device)
|
||||
|
|
@ -100,13 +101,14 @@ def sample_video_res_multistep(
|
|||
audio_carried = res_multistep_update(audio_carried, audio_denoised, sigma, sigma_down, audio_history, audio_history_sigma, previous_sigma)
|
||||
video_history, audio_history = video_denoised, audio_denoised
|
||||
video_history_sigma = audio_history_sigma = sigma_down
|
||||
elapsed = time.perf_counter() - started
|
||||
eta = elapsed / index * (total_steps - index)
|
||||
print(
|
||||
f"sampling step {index}/{total_steps}: "
|
||||
f"{time.perf_counter() - step_started:.1f}s, elapsed {elapsed:.1f}s, eta {eta:.1f}s",
|
||||
flush=True,
|
||||
)
|
||||
if progress:
|
||||
elapsed = time.perf_counter() - started
|
||||
eta = elapsed / index * (total_steps - index)
|
||||
print(
|
||||
f"sampling step {index}/{total_steps}: "
|
||||
f"{time.perf_counter() - step_started:.1f}s, elapsed {elapsed:.1f}s, eta {eta:.1f}s",
|
||||
flush=True,
|
||||
)
|
||||
return (video, _decode_audio_latent(audio_carried)) if return_audio else video
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from h3_blackwell_runtime.audio_vae_decoder import MiniMaxH3AudioVAE
|
|||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--latent", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--ffmpeg-loglevel", default="error")
|
||||
parser.add_argument("--quiet", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
state = torch.load(args.latent, map_location="cuda", weights_only=False)
|
||||
|
|
@ -31,8 +33,10 @@ args.output.parent.mkdir(parents=True, exist_ok=True)
|
|||
raw = args.output.with_suffix(".f32le")
|
||||
waveform.transpose(0, 1).contiguous().numpy().tofile(raw)
|
||||
subprocess.run([
|
||||
"ffmpeg", "-y", "-f", "f32le", "-ar", "32000", "-ac", "2",
|
||||
"ffmpeg", "-hide_banner", "-loglevel", args.ffmpeg_loglevel,
|
||||
"-y", "-f", "f32le", "-ar", "32000", "-ac", "2",
|
||||
"-i", str(raw), str(args.output),
|
||||
], check=True)
|
||||
raw.unlink()
|
||||
print({"output": str(args.output), "sample_rate": 32000, "shape": tuple(waveform.shape)})
|
||||
if not args.quiet:
|
||||
print({"output": str(args.output), "sample_rate": 32000, "shape": tuple(waveform.shape)})
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ parser.add_argument("--output", type=Path, required=True)
|
|||
parser.add_argument("--no-tiling", action="store_true")
|
||||
parser.add_argument("--implementation", choices=("direct", "upstream"), default="direct")
|
||||
parser.add_argument("--frames-dir", type=Path)
|
||||
parser.add_argument("--ffmpeg-loglevel", default="error")
|
||||
parser.add_argument("--quiet", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
state = torch.load(args.latent, map_location="cuda", weights_only=False)
|
||||
|
|
@ -35,7 +37,8 @@ else:
|
|||
vae = UpstreamMiniMaxH3VideoVAE(tiling=not args.no_tiling).to("cuda").eval()
|
||||
checkpoint = load_file("/vae/minimax_h3_video_vae_fp16.safetensors", device="cuda")
|
||||
missing, unexpected = vae.load_state_dict(checkpoint, strict=False)
|
||||
print({"upstream_missing": len(missing), "upstream_unexpected": len(unexpected)}, flush=True)
|
||||
if not args.quiet:
|
||||
print({"upstream_missing": len(missing), "upstream_unexpected": len(unexpected)}, flush=True)
|
||||
with torch.inference_mode():
|
||||
pixels = vae.decode(latent.to(next(vae.parameters()).dtype))[:, :, :frames]
|
||||
pixels = ((pixels[0].permute(1, 2, 3, 0).clamp(-1, 1) + 1) * 127.5).to(torch.uint8).cpu()
|
||||
|
|
@ -51,9 +54,11 @@ args.output.parent.mkdir(parents=True, exist_ok=True)
|
|||
raw = args.output.with_suffix(".rgb")
|
||||
pixels.numpy().tofile(raw)
|
||||
subprocess.run([
|
||||
"ffmpeg", "-y", "-f", "rawvideo", "-pixel_format", "rgb24",
|
||||
"ffmpeg", "-hide_banner", "-loglevel", args.ffmpeg_loglevel,
|
||||
"-y", "-f", "rawvideo", "-pixel_format", "rgb24",
|
||||
"-video_size", f"{pixels.shape[2]}x{pixels.shape[1]}", "-framerate", "24",
|
||||
"-i", str(raw), "-an", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(args.output),
|
||||
], check=True)
|
||||
raw.unlink()
|
||||
print({"output": str(args.output), "frames": frames, "shape": tuple(pixels.shape), "tiling": not args.no_tiling, "implementation": args.implementation})
|
||||
if not args.quiet:
|
||||
print({"output": str(args.output), "frames": frames, "shape": tuple(pixels.shape), "tiling": not args.no_tiling, "implementation": args.implementation})
|
||||
|
|
|
|||
|
|
@ -33,7 +33,10 @@ parser.add_argument("--steps", type=int, default=12)
|
|||
parser.add_argument("--seed", type=int, default=440204)
|
||||
parser.add_argument("--attention", choices=("sage2", "sdpa", "sage3"), default="sage2")
|
||||
parser.add_argument("--model-timesteps-capture", type=Path, help="Directory containing captured input_XX.pt H3 timesteps for strict parity checks.")
|
||||
parser.add_argument("--progress", action="store_true", help="Print per-step sampler progress.")
|
||||
parser.add_argument("--profile-memory", action="store_true")
|
||||
parser.add_argument("--ffmpeg-loglevel", default="error", help="ffmpeg loglevel, e.g. error, warning, info.")
|
||||
parser.add_argument("--quiet", action="store_true", help="Suppress JSON summary lines.")
|
||||
parser.add_argument("--save-latent", type=Path)
|
||||
parser.add_argument("--save-audio-latent", type=Path)
|
||||
parser.add_argument("--audio-output", type=Path)
|
||||
|
|
@ -63,6 +66,15 @@ def report_memory(stage: str) -> None:
|
|||
print({"stage": stage, "ts": datetime.now(timezone.utc).isoformat(), "epoch_s": round(time.time(), 3), "elapsed_s": round(now - started, 3), "delta_s": round(now - last_report, 3), "rss_gb": round(rss_kb / 1024**2, 3), "cuda_alloc_gb": round(cuda_alloc, 3), "cuda_reserved_gb": round(cuda_reserved, 3), "fast_safetensors": os.getenv("H3_FAST_SAFETENSORS", ""), "disable_mmap": os.getenv("H3_DISABLE_MMAP", "")}, flush=True)
|
||||
last_report = now
|
||||
|
||||
|
||||
def report(payload: dict) -> None:
|
||||
if not args.quiet:
|
||||
print(payload, flush=True)
|
||||
|
||||
|
||||
def ffmpeg_command(*parts: str) -> list[str]:
|
||||
return ["ffmpeg", "-hide_banner", "-loglevel", args.ffmpeg_loglevel, *parts]
|
||||
|
||||
report_memory("start")
|
||||
checkpoint = H3Checkpoint("/models/minimax_h3_fl2va_pruned_nvfp4.safetensors")
|
||||
conditioner = Qwen3VLPromptConditioner(
|
||||
|
|
@ -82,7 +94,7 @@ if args.model_timesteps_capture is not None:
|
|||
for index in range(args.steps)
|
||||
]
|
||||
want_audio = args.save_audio_latent is not None or args.audio_output is not None or args.mux_audio
|
||||
sampled = sample_video_res_multistep(model, H3PromptPacker(checkpoint), text, video, audio, steps=args.steps, model_timesteps=model_timesteps, return_audio=want_audio)
|
||||
sampled = sample_video_res_multistep(model, H3PromptPacker(checkpoint), text, video, audio, steps=args.steps, model_timesteps=model_timesteps, return_audio=want_audio, progress=args.progress)
|
||||
if want_audio:
|
||||
latent, audio_latent = sampled
|
||||
else:
|
||||
|
|
@ -94,13 +106,13 @@ if args.save_latent is not None:
|
|||
if audio_latent is not None:
|
||||
state["audio_latent"] = audio_latent.detach().cpu()
|
||||
torch.save(state, args.save_latent)
|
||||
print({"latent": str(args.save_latent)}, flush=True)
|
||||
report({"latent": str(args.save_latent)})
|
||||
if args.save_audio_latent is not None:
|
||||
if audio_latent is None:
|
||||
raise RuntimeError("audio latent was not sampled")
|
||||
args.save_audio_latent.parent.mkdir(parents=True, exist_ok=True)
|
||||
torch.save({"audio_latent": audio_latent.detach().cpu(), "frames": frames, "prompt": args.prompt, "seed": args.seed}, args.save_audio_latent)
|
||||
print({"audio_latent": str(args.save_audio_latent)}, flush=True)
|
||||
report({"audio_latent": str(args.save_audio_latent)})
|
||||
if args.skip_decode:
|
||||
raise SystemExit(0)
|
||||
vae = MiniMaxH3VideoVAE.from_safetensors("/vae/minimax_h3_video_vae_fp16.safetensors", device="cuda").eval()
|
||||
|
|
@ -116,7 +128,7 @@ args.output.parent.mkdir(parents=True, exist_ok=True)
|
|||
raw = args.output.with_suffix(".rgb")
|
||||
video_output = args.output.with_name(args.output.stem + ".video.mp4") if args.mux_audio else args.output
|
||||
pixels.numpy().tofile(raw)
|
||||
subprocess.run(["ffmpeg", "-y", "-f", "rawvideo", "-pixel_format", "rgb24", "-video_size", f"{pixels.shape[2]}x{pixels.shape[1]}", "-framerate", "24", "-i", str(raw), "-an", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(video_output)], check=True)
|
||||
subprocess.run(ffmpeg_command("-y", "-f", "rawvideo", "-pixel_format", "rgb24", "-video_size", f"{pixels.shape[2]}x{pixels.shape[1]}", "-framerate", "24", "-i", str(raw), "-an", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(video_output)), check=True)
|
||||
raw.unlink()
|
||||
audio_path = args.audio_output
|
||||
if args.mux_audio and audio_path is None:
|
||||
|
|
@ -132,10 +144,10 @@ if audio_path is not None:
|
|||
audio_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
audio_raw = audio_path.with_suffix(".f32le")
|
||||
waveform.transpose(0, 1).contiguous().numpy().tofile(audio_raw)
|
||||
subprocess.run(["ffmpeg", "-y", "-f", "f32le", "-ar", "32000", "-ac", "2", "-i", str(audio_raw), str(audio_path)], check=True)
|
||||
subprocess.run(ffmpeg_command("-y", "-f", "f32le", "-ar", "32000", "-ac", "2", "-i", str(audio_raw), str(audio_path)), check=True)
|
||||
audio_raw.unlink()
|
||||
print({"audio_output": str(audio_path), "sample_rate": 32000, "audio_shape": tuple(waveform.shape)}, flush=True)
|
||||
report({"audio_output": str(audio_path), "sample_rate": 32000, "audio_shape": tuple(waveform.shape)})
|
||||
if args.mux_audio:
|
||||
subprocess.run(["ffmpeg", "-y", "-i", str(video_output), "-i", str(audio_path), "-c:v", "copy", "-c:a", "aac", "-shortest", str(args.output)], check=True)
|
||||
subprocess.run(ffmpeg_command("-y", "-i", str(video_output), "-i", str(audio_path), "-c:v", "copy", "-c:a", "aac", "-shortest", str(args.output)), check=True)
|
||||
video_output.unlink()
|
||||
print({"output": str(args.output), "frames": frames, "shape": tuple(pixels.shape)})
|
||||
report({"output": str(args.output), "frames": frames, "shape": tuple(pixels.shape)})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue