diff --git a/src/h3_blackwell_runtime/conditioning.py b/src/h3_blackwell_runtime/conditioning.py index 8103875..cc7cb05 100644 --- a/src/h3_blackwell_runtime/conditioning.py +++ b/src/h3_blackwell_runtime/conditioning.py @@ -9,13 +9,9 @@ class H3PromptTokenizer: """Tokenize raw H3 prompt text without Qwen chat-template tokens.""" def __init__(self, tokenizer_dir: str | Path): - from transformers import Qwen2Tokenizer + from transformers import AutoTokenizer - tokenizer_dir = Path(tokenizer_dir) - self.tokenizer = Qwen2Tokenizer( - vocab_file=str(tokenizer_dir / "vocab.json"), - merges_file=str(tokenizer_dir / "merges.txt"), - ) + self.tokenizer = AutoTokenizer.from_pretrained(str(Path(tokenizer_dir)), local_files_only=True) def __call__(self, prompt: str, *, device: torch.device | str = "cuda") -> torch.Tensor: if not prompt: diff --git a/src/h3_blackwell_runtime/qwen25_tokenizer/config.json b/src/h3_blackwell_runtime/qwen25_tokenizer/config.json new file mode 100644 index 0000000..8863755 --- /dev/null +++ b/src/h3_blackwell_runtime/qwen25_tokenizer/config.json @@ -0,0 +1,5 @@ +{ + "model_type": "qwen2", + "tokenizer_class": "Qwen2Tokenizer", + "vocab_size": 151936 +}