From f90801ea8aad845d51ddc2f38d8f63133ea07546 Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Wed, 12 Aug 2026 14:35:04 +0700 Subject: [PATCH] Restore local Qwen AutoTokenizer config --- src/h3_blackwell_runtime/conditioning.py | 8 ++------ src/h3_blackwell_runtime/qwen25_tokenizer/config.json | 5 +++++ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 src/h3_blackwell_runtime/qwen25_tokenizer/config.json diff --git a/src/h3_blackwell_runtime/conditioning.py b/src/h3_blackwell_runtime/conditioning.py index 8103875..cc7cb05 100644 --- a/src/h3_blackwell_runtime/conditioning.py +++ b/src/h3_blackwell_runtime/conditioning.py @@ -9,13 +9,9 @@ class H3PromptTokenizer: """Tokenize raw H3 prompt text without Qwen chat-template tokens.""" def __init__(self, tokenizer_dir: str | Path): - from transformers import Qwen2Tokenizer + from transformers import AutoTokenizer - tokenizer_dir = Path(tokenizer_dir) - self.tokenizer = Qwen2Tokenizer( - vocab_file=str(tokenizer_dir / "vocab.json"), - merges_file=str(tokenizer_dir / "merges.txt"), - ) + self.tokenizer = AutoTokenizer.from_pretrained(str(Path(tokenizer_dir)), local_files_only=True) def __call__(self, prompt: str, *, device: torch.device | str = "cuda") -> torch.Tensor: if not prompt: diff --git a/src/h3_blackwell_runtime/qwen25_tokenizer/config.json b/src/h3_blackwell_runtime/qwen25_tokenizer/config.json new file mode 100644 index 0000000..8863755 --- /dev/null +++ b/src/h3_blackwell_runtime/qwen25_tokenizer/config.json @@ -0,0 +1,5 @@ +{ + "model_type": "qwen2", + "tokenizer_class": "Qwen2Tokenizer", + "vocab_size": 151936 +}