25 lines
903 B
Bash
Executable File
25 lines
903 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT=${ROOT:-/ssd/yi/Tokenizer_Swap}
|
|
NPROC=${NPROC:-8}
|
|
MODEL=${MODEL:-$ROOT/model_training/checkpoints/qwen3_06b_dsv4_remap_v2_cpt_5b_seed42_seq8192_bsz3acc3}
|
|
TRAIN=${TRAIN:-$ROOT/dataset_building/generated/dsv4_chat_tokenized_v4_noupsample_nobbh_921k/train_dsv4_chat_tokenized.jsonl.gz}
|
|
EVAL=${EVAL:-$ROOT/dataset_building/generated/dsv4_chat_tokenized_v4_noupsample_nobbh_921k/validation_dsv4_chat_tokenized.jsonl.gz}
|
|
OUT=${OUT:-$ROOT/model_training/checkpoints/qwen3_06b_dsv4_remap_v2_cpt5b_then_sft1m_5epoch}
|
|
|
|
mkdir -p "$(dirname "$OUT")"
|
|
torchrun --nproc_per_node "$NPROC" "$ROOT/model_training/train_dsv4_tokenized_full_sft.py" \
|
|
--model "$MODEL" \
|
|
--train "$TRAIN" \
|
|
--eval "$EVAL" \
|
|
--out "$OUT" \
|
|
--max-length 2048 \
|
|
--epochs 5 \
|
|
--batch-size 8 \
|
|
--grad-accum 16 \
|
|
--lr 5e-5 \
|
|
--eval-steps 500 \
|
|
--save-steps 300 \
|
|
--num-workers 2
|