Update Hugging Face model download CLI

This commit is contained in:
Codex
2026-06-25 15:34:22 +08:00
parent e79c7ea9bf
commit 4f9e0a30a9
3 changed files with 44 additions and 2 deletions

View File

@@ -135,6 +135,20 @@ export QWEN36_27B_MODEL_ID=<actual-27b-id>
./scripts/download_models.sh
```
模型下载脚本使用新版 Hugging Face CLI
```bash
hf download <model-id> --local-dir <dir> --max-workers <n>
```
旧版 `huggingface-cli download``--local-dir-use-symlinks` 参数已经不再使用。默认并发是 `HF_DOWNLOAD_MAX_WORKERS=8`,如果网络或镜像不稳定可以调小:
```bash
export HF_DOWNLOAD_MAX_WORKERS=4
export HF_TOKEN=<optional-token>
./scripts/download_models.sh
```
## 单步训练
每个训练脚本默认:

View File

@@ -59,6 +59,14 @@ Download base models:
./scripts/download_models.sh
```
Model downloads use the current Hugging Face CLI:
```bash
hf download <model-id> --local-dir <dir> --max-workers <n>
```
Do not use the deprecated `huggingface-cli download` or the removed `--local-dir-use-symlinks` option. Tune download concurrency with `HF_DOWNLOAD_MAX_WORKERS`; pass an optional token with `HF_TOKEN`.
Run the full ordered experiment:
```bash

View File

@@ -15,6 +15,26 @@ mkdir -p models
QWEN35_9B_MODEL_ID="${QWEN35_9B_MODEL_ID:-Qwen/Qwen3.5-9B}"
QWEN36_27B_MODEL_ID="${QWEN36_27B_MODEL_ID:-Qwen/Qwen3.6-27B}"
HF_DOWNLOAD_MAX_WORKERS="${HF_DOWNLOAD_MAX_WORKERS:-8}"
huggingface-cli download "${QWEN35_9B_MODEL_ID}" --local-dir "models/qwen3.5-9b" --local-dir-use-symlinks False
huggingface-cli download "${QWEN36_27B_MODEL_ID}" --local-dir "models/qwen3.6-27b" --local-dir-use-symlinks False
download_model() {
local model_id="$1"
local local_dir="$2"
local cmd=(
hf download "${model_id}"
--local-dir "${local_dir}"
--max-workers "${HF_DOWNLOAD_MAX_WORKERS}"
)
if [[ -n "${HF_TOKEN:-}" ]]; then
cmd+=(--token "${HF_TOKEN}")
fi
printf '%q ' "${cmd[@]}"
echo
if [[ "${DRY_RUN:-0}" == "1" ]]; then
return 0
fi
"${cmd[@]}"
}
download_model "${QWEN35_9B_MODEL_ID}" "models/qwen3.5-9b"
download_model "${QWEN36_27B_MODEL_ID}" "models/qwen3.6-27b"