Update Hugging Face model download CLI
This commit is contained in:
@@ -135,6 +135,20 @@ export QWEN36_27B_MODEL_ID=<actual-27b-id>
|
|||||||
./scripts/download_models.sh
|
./scripts/download_models.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
模型下载脚本使用新版 Hugging Face CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hf download <model-id> --local-dir <dir> --max-workers <n>
|
||||||
|
```
|
||||||
|
|
||||||
|
旧版 `huggingface-cli download` 和 `--local-dir-use-symlinks` 参数已经不再使用。默认并发是 `HF_DOWNLOAD_MAX_WORKERS=8`,如果网络或镜像不稳定可以调小:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export HF_DOWNLOAD_MAX_WORKERS=4
|
||||||
|
export HF_TOKEN=<optional-token>
|
||||||
|
./scripts/download_models.sh
|
||||||
|
```
|
||||||
|
|
||||||
## 单步训练
|
## 单步训练
|
||||||
|
|
||||||
每个训练脚本默认:
|
每个训练脚本默认:
|
||||||
|
|||||||
@@ -59,6 +59,14 @@ Download base models:
|
|||||||
./scripts/download_models.sh
|
./scripts/download_models.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Model downloads use the current Hugging Face CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hf download <model-id> --local-dir <dir> --max-workers <n>
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not use the deprecated `huggingface-cli download` or the removed `--local-dir-use-symlinks` option. Tune download concurrency with `HF_DOWNLOAD_MAX_WORKERS`; pass an optional token with `HF_TOKEN`.
|
||||||
|
|
||||||
Run the full ordered experiment:
|
Run the full ordered experiment:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -15,6 +15,26 @@ mkdir -p models
|
|||||||
|
|
||||||
QWEN35_9B_MODEL_ID="${QWEN35_9B_MODEL_ID:-Qwen/Qwen3.5-9B}"
|
QWEN35_9B_MODEL_ID="${QWEN35_9B_MODEL_ID:-Qwen/Qwen3.5-9B}"
|
||||||
QWEN36_27B_MODEL_ID="${QWEN36_27B_MODEL_ID:-Qwen/Qwen3.6-27B}"
|
QWEN36_27B_MODEL_ID="${QWEN36_27B_MODEL_ID:-Qwen/Qwen3.6-27B}"
|
||||||
|
HF_DOWNLOAD_MAX_WORKERS="${HF_DOWNLOAD_MAX_WORKERS:-8}"
|
||||||
|
|
||||||
huggingface-cli download "${QWEN35_9B_MODEL_ID}" --local-dir "models/qwen3.5-9b" --local-dir-use-symlinks False
|
download_model() {
|
||||||
huggingface-cli download "${QWEN36_27B_MODEL_ID}" --local-dir "models/qwen3.6-27b" --local-dir-use-symlinks False
|
local model_id="$1"
|
||||||
|
local local_dir="$2"
|
||||||
|
local cmd=(
|
||||||
|
hf download "${model_id}"
|
||||||
|
--local-dir "${local_dir}"
|
||||||
|
--max-workers "${HF_DOWNLOAD_MAX_WORKERS}"
|
||||||
|
)
|
||||||
|
if [[ -n "${HF_TOKEN:-}" ]]; then
|
||||||
|
cmd+=(--token "${HF_TOKEN}")
|
||||||
|
fi
|
||||||
|
printf '%q ' "${cmd[@]}"
|
||||||
|
echo
|
||||||
|
if [[ "${DRY_RUN:-0}" == "1" ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
"${cmd[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
download_model "${QWEN35_9B_MODEL_ID}" "models/qwen3.5-9b"
|
||||||
|
download_model "${QWEN36_27B_MODEL_ID}" "models/qwen3.6-27b"
|
||||||
|
|||||||
Reference in New Issue
Block a user