daniel/h3-blackwell-runtime
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Load Qwen tokenizer from local vocab

Browse source
This commit is contained in:
Daniel Maddern 2026-08-12 14:28:41 +07:00
parent b188002199
commit 7e2062ee38
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/h3_blackwell_runtime/conditioning.py
View file

@ -9,9 +9,13 @@ class H3PromptTokenizer:
"""Tokenize raw H3 prompt text without Qwen chat-template tokens."""
def __init__(self, tokenizer_dir: str | Path):
from transformers import AutoTokenizer
from transformers import Qwen2Tokenizer
self.tokenizer = AutoTokenizer.from_pretrained(str(tokenizer_dir), local_files_only=True)
tokenizer_dir = Path(tokenizer_dir)
self.tokenizer = Qwen2Tokenizer(
vocab_file=str(tokenizer_dir / "vocab.json"),
merges_file=str(tokenizer_dir / "merges.txt"),
)
def __call__(self, prompt: str, *, device: torch.device | str = "cuda") -> torch.Tensor:
if not prompt: