Use packaged Qwen tokenizer assets

src/h3_blackwell_runtime/qwen3vl_text.py

@@ -155,9 +155,10 @@ class Qwen3VL32BTextEncoder(nn.Module):
 
 class Qwen3VLPromptConditioner:
     """Tokenize raw H3 prompt text and produce Qwen layer-50 conditioning."""
-    def __init__(self, checkpoint_path: str | Path, tokenizer_dir: str | Path, *, device: str | torch.device = "cuda", dtype: torch.dtype = torch.bfloat16):
+    def __init__(self, checkpoint_path: str | Path, tokenizer_dir: str | Path | None = None, *, device: str | torch.device = "cuda", dtype: torch.dtype = torch.bfloat16):
         from .conditioning import H3PromptTokenizer
 
+        tokenizer_dir = tokenizer_dir or Path(__file__).with_name("qwen25_tokenizer")
         self.tokenizer = H3PromptTokenizer(tokenizer_dir)
         self.encoder = Qwen3VL32BTextEncoder(checkpoint_path, device=device, dtype=dtype)
         self.device = device

tools/compare_fl2va_steps.py

@@ -17,7 +17,7 @@ initial = torch.load(f"{root}/initial.pt", map_location="cuda", weights_only=Fal
 steps = [torch.load(path, map_location="cuda", weights_only=False) for path in sorted(glob.glob(f"{root}/step_*.pt"))]
 sigmas = initial["sigmas"].to("cuda")
 checkpoint = H3Checkpoint("/models/minimax_h3_fl2va_pruned_nvfp4.safetensors")
-text = H3TokenRefiner(checkpoint)(Qwen3VLPromptConditioner("/text-encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", "/opt/h3-blackwell-runtime/qwen25_tokenizer")("A brass-and-paper dragon flies above a rain-washed old city at blue hour."))
+text = H3TokenRefiner(checkpoint)(Qwen3VLPromptConditioner("/text-encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors")("A brass-and-paper dragon flies above a rain-washed old city at blue hour."))
 model = H3PackedDenoiser.from_checkpoint(checkpoint, attention_backend="sage2").eval()
 packer = H3PromptPacker(checkpoint)