Clean tokenizer swap migration

2026-06-18 10:10:57 +00:00
commit 5d1e9c4bc3
37 changed files with 9574 additions and 0 deletions
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=${ROOT:-/ssd/yi/Tokenizer_Swap}
+NPROC=${NPROC:-8}
+MODEL=${MODEL:-$ROOT/model_building/generated_models/Qwen3-0.6B-DSV4-tokenizer-remap-v2}
+DATA=${DATA:-$ROOT/dataset_building/generated/cpt_packed_1b_seq8192_seed42_stratified}
+OUT=${OUT:-$ROOT/model_training/checkpoints/qwen3_06b_dsv4_remap_v2_cpt_1b_seed42_seq8192_bsz3acc3}
+
+mkdir -p "$(dirname "$OUT")"
+torchrun --nproc_per_node "$NPROC" "$ROOT/model_training/train_cpt_packed_full.py" \
+  --model "$MODEL" \
+  --data "$DATA" \
+  --out "$OUT" \
+  --epochs 1 \
+  --batch-size 3 \
+  --grad-accum 3 \
+  --lr 2e-5 \
+  --eval-steps 500 \
+  --save-steps 500 \
+  --num-workers 2