Clean tokenizer swap migration

2026-06-18 10:10:57 +00:00
commit 5d1e9c4bc3
37 changed files with 9574 additions and 0 deletions
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=${ROOT:-/ssd/yi/Tokenizer_Swap}
+BASE_MODEL=${BASE_MODEL:-/ssd/yi/tokenizer_swap_cepe/models/Qwen3-0.6B}
+DSV_TOKENIZER=${DSV_TOKENIZER:-/ssd/yi/tokenizer_swap_cepe/models/tokenizers/dsv4_flash}
+OUT=${OUT:-$ROOT/model_building/generated_models/Qwen3-0.6B-DSV4-tokenizer-remap-v2}
+
+python "$ROOT/model_building/build_qwen3_dsv4_remap_checkpoint_v2.py" \
+  --qwen-model "$BASE_MODEL" \
+  --dsv-tokenizer "$DSV_TOKENIZER" \
+  --out "$OUT"