Add eight GPU SWIFT and Megatron training scripts

scripts/swift_train_common.sh

@@ -10,6 +10,7 @@ export HTTP_PROXY="${HTTP_PROXY:-${http_proxy}}"
 export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy}}"
 export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
 export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
 
 if [[ -f .venv/bin/activate ]]; then
   source .venv/bin/activate
@@ -26,6 +27,9 @@ SAVE_STEPS="${SAVE_STEPS:-1000}"
 EVAL_STEPS="${EVAL_STEPS:-1000}"
 LOGGING_STEPS="${LOGGING_STEPS:-1}"
 NUM_EPOCHS="${NUM_EPOCHS:-1}"
+NPROC_PER_NODE="${NPROC_PER_NODE:-8}"
+CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
+DEEPSPEED="${DEEPSPEED:-zero2}"
 WARMUP_RATIO="${WARMUP_RATIO:-0.1}"
 LR_SCHEDULER_TYPE="${LR_SCHEDULER_TYPE:-cosine}"
 LORA_RANK="${LORA_RANK:-32}"
@@ -93,7 +97,7 @@ run_swift_train() {
     --model "${model_path}"
     --dataset "${TRAIN_JSONL}"
     --val_dataset "${VAL_JSONL}"
-    --train_type "${train_type}"
+    --tuner_type "${train_type}"
     --torch_dtype bfloat16
     --num_train_epochs "${NUM_EPOCHS}"
     --per_device_train_batch_size "${per_device_batch_size}"
@@ -111,16 +115,21 @@ run_swift_train() {
     --output_dir "${output_dir}"
     --save_total_limit "${SAVE_TOTAL_LIMIT:-3}"
     --dataloader_num_workers "${DATALOADER_NUM_WORKERS:-4}"
+    --deepspeed "${DEEPSPEED}"
   )
 
   if [[ "${train_type}" == "lora" ]]; then
     cmd+=(--lora_rank "${LORA_RANK}")
   fi
+  if [[ -n "${MAX_STEPS:-}" ]]; then
+    cmd+=(--max_steps "${MAX_STEPS}")
+  fi
 
-  printf '%q ' "${cmd[@]}" | tee "${log_file}.cmd"
-  echo
+  printf 'CUDA_VISIBLE_DEVICES=%q NPROC_PER_NODE=%q ' "${CUDA_VISIBLE_DEVICES}" "${NPROC_PER_NODE}" | tee "${log_file}.cmd"
+  printf '%q ' "${cmd[@]}" | tee -a "${log_file}.cmd"
+  echo | tee -a "${log_file}.cmd"
   if [[ "${DRY_RUN:-0}" == "1" ]]; then
     return 0
   fi
-  "${cmd[@]}" 2>&1 | tee "${log_file}"
+  CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" NPROC_PER_NODE="${NPROC_PER_NODE}" "${cmd[@]}" 2>&1 | tee "${log_file}"
 }