From 9b584742c4c7acb6ce0cc0129c364c89658437a1 Mon Sep 17 00:00:00 2001 From: Tom Lu Date: Wed, 1 Jul 2026 15:32:42 +0800 Subject: [PATCH] Support Laoyao BPE tokenizer in SFT heldout builder --- dataset_building/README.md | 6 ++++-- .../build_training_and_test_mix_v3.py | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/dataset_building/README.md b/dataset_building/README.md index 8173c98..6f52ebb 100644 --- a/dataset_building/README.md +++ b/dataset_building/README.md @@ -56,8 +56,10 @@ dataset_building/generated/training_mix_v4_train1m_test2p8k_noupsample_nobbh_202 ``` Use `OUT=/path/to/output` and `TOKENIZER=/path/to/tokenizer` to place the -validation set outside the repo or build it with a specific tokenizer. Generated -outputs stay ignored by git. +validation set outside the repo or build it with a specific tokenizer. For +Laoyao custom BPE tokenizers, set `LAOYAO_MODEL_ROOT=/path/to/laoyao/repo` so +the builder can import `laoyaomodel.tokenization.bpe`. Generated outputs stay +ignored by git. ## SFT Builders diff --git a/dataset_building/build_training_and_test_mix_v3.py b/dataset_building/build_training_and_test_mix_v3.py index d5f4872..94a7f1e 100644 --- a/dataset_building/build_training_and_test_mix_v3.py +++ b/dataset_building/build_training_and_test_mix_v3.py @@ -7,6 +7,7 @@ import json import os import random import re +import sys from collections import Counter, defaultdict from pathlib import Path @@ -145,15 +146,30 @@ def prompt_hash(user): def load_tokenizer(path): + path_obj = Path(path) try: return AutoTokenizer.from_pretrained(path, trust_remote_code=True) except Exception: - return Tokenizer.from_file(str(Path(path) / "tokenizer.json")) + tokenizer_path = path_obj if path_obj.is_file() else path_obj / "tokenizer.json" + try: + return Tokenizer.from_file(str(tokenizer_path)) + except Exception: + payload = json.loads(tokenizer_path.read_text(encoding="utf-8")) + if payload.get("implementation") != "laoyaomodel.bytes_bpe": + raise + model_root = os.environ.get("LAOYAO_MODEL_ROOT") + if model_root: + sys.path.insert(0, model_root) + from laoyaomodel.tokenization.bpe import BPETokenizer + + return BPETokenizer.from_tokenizer_json(tokenizer_path) def token_len(tok, text): if isinstance(tok, Tokenizer): return len(tok.encode(text, add_special_tokens=False).ids) + if tok.__class__.__name__ == "BPETokenizer": + return len(tok.encode(text)) return len(tok(text, add_special_tokens=False)["input_ids"])