Initial Commit

scripts/kaiyuan2b-training/params/qwen3_1p7b/data_phase1_smoke.sh

@@ -0,0 +1,21 @@
+
+DATA_DIR=/ssd/yi/converted_data/megatron_phase1
+
+START=0
+END=210
+
+DATA_PATHS=""
+for idx in $(seq -f "%05g" $START $END); do
+  DATA_PATHS+=" 1 ${DATA_DIR}/phase1_part-${idx}_text_document"
+done
+
+DATA_ARGS="
+  --data-path ${DATA_PATHS}
+  --split 999,1,0
+  --tokenizer-type HuggingFaceTokenizer
+  --tokenizer-model /apps/yi/model_training/data/tokenizer
+"
+
+#   --vocab-file /apps/yi/model_training/data/tokenizer/vocab.json
+#   --merge-file /apps/yi/model_training/data/tokenizer/merges.txt
+#   --vocab-size 151936

scripts/kaiyuan2b-training/params/qwen3_1p7b/hparams.sh

@@ -0,0 +1,13 @@
+HPARAMS="
+  --micro-batch-size 16
+  --global-batch-size 2048
+  --train-iters 87000
+  --eval-iters 10
+  --eval-interval 1000
+  --save-interval 1000
+  --log-interval 1
+  --lr 5e-3
+  --min-lr 5e-3
+  --lr-decay-style constant
+  --lr-warmup-iters 10
+"

scripts/kaiyuan2b-training/params/qwen3_1p7b/model.sh

@@ -0,0 +1,19 @@
+# note: official qwen3 training uses qk norm while megatron has no official support
+
+MODEL_ARGS="
+  --seq-length 4096
+  --hidden-size 2048
+  --ffn-hidden-size 6144
+  --num-layers 28
+  --num-attention-heads 16
+  --num-query-groups 8
+  --rotary-base 10000
+  --init-method-std 0.018
+  --group-query-attention
+  --max-position-embeddings 4096
+  --position-embedding-type rope
+  --swiglu
+  --disable-bias-linear
+  --normalization RMSNorm
+  --untie-embeddings-and-output-weights
+"