22 lines
671 B
Bash
Executable File
22 lines
671 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT=${ROOT:-/ssd/yi/Tokenizer_Swap}
|
|
NPROC=${NPROC:-8}
|
|
MODEL=${MODEL:-$ROOT/model_building/generated_models/Qwen3-0.6B-DSV4-tokenizer-remap-v2}
|
|
DATA=${DATA:-$ROOT/dataset_building/generated/cpt_packed_5b_seq8192_seed42_stratified}
|
|
OUT=${OUT:-$ROOT/model_training/checkpoints/qwen3_06b_dsv4_remap_v2_cpt_5b_seed42_seq8192_bsz3acc3}
|
|
|
|
mkdir -p "$(dirname "$OUT")"
|
|
torchrun --nproc_per_node "$NPROC" "$ROOT/model_training/train_cpt_packed_full.py" \
|
|
--model "$MODEL" \
|
|
--data "$DATA" \
|
|
--out "$OUT" \
|
|
--epochs 1 \
|
|
--batch-size 3 \
|
|
--grad-accum 3 \
|
|
--lr 2e-5 \
|
|
--eval-steps 2000 \
|
|
--save-steps 2000 \
|
|
--num-workers 2
|