Tune default SFT and LoRA hyperparameters

README.md

@@ -147,7 +147,35 @@ export QWEN36_27B_MODEL_ID=<actual-27b-id>
 - `report_to=tensorboard`
 - `max_length=262144`
 - `warmup_ratio=0.1`
-- `learning_rate=1e-5`
+- `lr_scheduler_type=cosine`
+- full SFT: `learning_rate=1e-5`
+- LoRA: `learning_rate=5e-5`
+- 默认每卡训练 batch size 为 `1`
+
+B300/g0049 当前是 8 张 B300，每卡约 275GB 显存。由于本实验默认上下文长度是 `262144`，预设 batch size 保守取 `1`，防止 27B/full 或长样本直接 OOM。吞吐测试时可以从脚本外部调大，建议先从 LoRA/9B 开始试。
+
+常用覆盖方式：
+
+```bash
+# 全部训练统一覆盖
+export PER_DEVICE_BATCH_SIZE=2
+export GRAD_ACCUM_STEPS=2
+
+# 只覆盖 LoRA 或 full
+export LORA_PER_DEVICE_BATCH_SIZE=2
+export FULL_PER_DEVICE_BATCH_SIZE=1
+export LORA_LEARNING_RATE=5e-5
+export FULL_LEARNING_RATE=1e-5
+
+# 只覆盖某一个 run；优先级最高
+export QWEN35_9B_LORA_R32_PER_DEVICE_BATCH_SIZE=2
+export QWEN35_9B_LORA_R32_GRAD_ACCUM_STEPS=2
+export QWEN36_27B_FULL_BF16_PER_DEVICE_BATCH_SIZE=1
+
+# scheduler/warmup
+export WARMUP_RATIO=0.1
+export LR_SCHEDULER_TYPE=cosine
+```
 
 命令：
 

SKILL.md

@@ -89,11 +89,29 @@ The default experiment uses:
 - 1 epoch
 - LoRA rank 32 for LoRA runs
 - bf16 full fine-tuning for full runs
+- full SFT learning rate `1e-5`
+- LoRA learning rate `5e-5`
+- warmup ratio `0.1`
+- explicit cosine LR scheduler via `--lr_scheduler_type cosine`
 - `max_length=262144`
+- conservative per-device train batch size `1`
 - checkpoint save every 1000 steps
 - validation every 1000 steps
 - TensorBoard logging under `runs/`
 
+Batch size is intentionally conservative because B300/g0049 has ~275GB per GPU but the default context length is 262144 tokens. Increase batch size from the shell only after checking memory:
+
+```bash
+export PER_DEVICE_BATCH_SIZE=2
+export GRAD_ACCUM_STEPS=2
+export LORA_PER_DEVICE_BATCH_SIZE=2
+export FULL_PER_DEVICE_BATCH_SIZE=1
+export QWEN35_9B_LORA_R32_PER_DEVICE_BATCH_SIZE=2
+export QWEN36_27B_FULL_BF16_PER_DEVICE_BATCH_SIZE=1
+```
+
+Run-specific variables have the highest precedence, then global `PER_DEVICE_BATCH_SIZE` / `GRAD_ACCUM_STEPS`, then train-type defaults, then the safe default of 1.
+
 Override model IDs or paths with:
 
 ```bash

scripts/swift_train_common.sh

@@ -25,12 +25,27 @@ MAX_LENGTH="${MAX_LENGTH:-262144}"
 SAVE_STEPS="${SAVE_STEPS:-1000}"
 EVAL_STEPS="${EVAL_STEPS:-1000}"
 LOGGING_STEPS="${LOGGING_STEPS:-1}"
-GRAD_ACCUM_STEPS="${GRAD_ACCUM_STEPS:-1}"
-PER_DEVICE_BATCH_SIZE="${PER_DEVICE_BATCH_SIZE:-1}"
 NUM_EPOCHS="${NUM_EPOCHS:-1}"
-LEARNING_RATE="${LEARNING_RATE:-1e-5}"
 WARMUP_RATIO="${WARMUP_RATIO:-0.1}"
+LR_SCHEDULER_TYPE="${LR_SCHEDULER_TYPE:-cosine}"
 LORA_RANK="${LORA_RANK:-32}"
+DEFAULT_PER_DEVICE_BATCH_SIZE="${DEFAULT_PER_DEVICE_BATCH_SIZE:-1}"
+DEFAULT_GRAD_ACCUM_STEPS="${DEFAULT_GRAD_ACCUM_STEPS:-1}"
+DEFAULT_EVAL_BATCH_SIZE="${DEFAULT_EVAL_BATCH_SIZE:-1}"
+
+env_key() {
+  printf '%s' "$1" | tr '[:lower:]-' '[:upper:]_' | sed 's/[^A-Z0-9_]/_/g'
+}
+
+env_or_default() {
+  local name="$1"
+  local fallback="$2"
+  if [[ -n "${!name:-}" ]]; then
+    printf '%s' "${!name}"
+  else
+    printf '%s' "${fallback}"
+  fi
+}
 
 require_file() {
   if [[ ! -f "$1" ]]; then
@@ -46,6 +61,28 @@ run_swift_train() {
   local output_dir="outputs/${run_name}"
   local tb_dir="runs/${run_name}"
   local log_file="logs/${run_name}.log"
+  local run_key
+  run_key="$(env_key "${run_name}")"
+
+  local default_lr
+  if [[ "${train_type}" == "lora" ]]; then
+    default_lr="${LORA_LEARNING_RATE:-5e-5}"
+  else
+    default_lr="${FULL_LEARNING_RATE:-1e-5}"
+  fi
+  local learning_rate
+  learning_rate="$(env_or_default "${run_key}_LEARNING_RATE" "${LEARNING_RATE:-${default_lr}}")"
+
+  local type_key
+  type_key="$(env_key "${train_type}")"
+  local type_bsz_var="${type_key}_PER_DEVICE_BATCH_SIZE"
+  local type_accum_var="${type_key}_GRAD_ACCUM_STEPS"
+  local per_device_batch_size
+  local grad_accum_steps
+  local eval_batch_size
+  per_device_batch_size="$(env_or_default "${run_key}_PER_DEVICE_BATCH_SIZE" "${PER_DEVICE_BATCH_SIZE:-${!type_bsz_var:-${DEFAULT_PER_DEVICE_BATCH_SIZE}}}")"
+  grad_accum_steps="$(env_or_default "${run_key}_GRAD_ACCUM_STEPS" "${GRAD_ACCUM_STEPS:-${!type_accum_var:-${DEFAULT_GRAD_ACCUM_STEPS}}}")"
+  eval_batch_size="$(env_or_default "${run_key}_EVAL_BATCH_SIZE" "${EVAL_PER_DEVICE_BATCH_SIZE:-${DEFAULT_EVAL_BATCH_SIZE}}")"
 
   require_file "${TRAIN_JSONL}"
   require_file "${VAL_JSONL}"
@@ -59,11 +96,12 @@ run_swift_train() {
     --train_type "${train_type}"
     --torch_dtype bfloat16
     --num_train_epochs "${NUM_EPOCHS}"
-    --per_device_train_batch_size "${PER_DEVICE_BATCH_SIZE}"
+    --per_device_train_batch_size "${per_device_batch_size}"
-    --per_device_eval_batch_size 1
+    --per_device_eval_batch_size "${eval_batch_size}"
-    --gradient_accumulation_steps "${GRAD_ACCUM_STEPS}"
+    --gradient_accumulation_steps "${grad_accum_steps}"
-    --learning_rate "${LEARNING_RATE}"
+    --learning_rate "${learning_rate}"
     --warmup_ratio "${WARMUP_RATIO}"
+    --lr_scheduler_type "${LR_SCHEDULER_TYPE}"
     --max_length "${MAX_LENGTH}"
     --save_steps "${SAVE_STEPS}"
     --eval_steps "${EVAL_STEPS}"