daniel/h3-blackwell-runtime
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Restore local Qwen AutoTokenizer config

Browse source
This commit is contained in:
Daniel Maddern 2026-08-12 14:35:04 +07:00
parent 7e2062ee38
commit f90801ea8a
2 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/h3_blackwell_runtime/conditioning.py
View file

@ -9,13 +9,9 @@ class H3PromptTokenizer:
"""Tokenize raw H3 prompt text without Qwen chat-template tokens.""" """Tokenize raw H3 prompt text without Qwen chat-template tokens."""
def __init__(self, tokenizer_dir: str | Path): def __init__(self, tokenizer_dir: str | Path):
from transformers import Qwen2Tokenizer from transformers import AutoTokenizer
tokenizer_dir = Path(tokenizer_dir) self.tokenizer = AutoTokenizer.from_pretrained(str(Path(tokenizer_dir)), local_files_only=True)
self.tokenizer = Qwen2Tokenizer(
vocab_file=str(tokenizer_dir / "vocab.json"),
merges_file=str(tokenizer_dir / "merges.txt"),
)
def __call__(self, prompt: str, *, device: torch.device | str = "cuda") -> torch.Tensor: def __call__(self, prompt: str, *, device: torch.device | str = "cuda") -> torch.Tensor:
if not prompt: if not prompt:

5
src/h3_blackwell_runtime/qwen25_tokenizer/config.json Normal file
View file

@ -0,0 +1,5 @@
{
"model_type": "qwen2",
"tokenizer_class": "Qwen2Tokenizer",
"vocab_size": 151936
}