41 lines
921 B
Bash
41 lines
921 B
Bash
#!/bin/bash
|
|
|
|
source /apps/yi/.venv/bin/activate
|
|
|
|
export HF_HUB_ENABLE_HF_TRANSFER=1
|
|
export HF_HUB_DISABLE_SYMLINKS_WARNING=1
|
|
|
|
# activate mirror if necessary
|
|
|
|
export HF_ENDPOINT="https://hf-mirror.com/"
|
|
|
|
unset http_proxy
|
|
unset https_proxy
|
|
export http_proxy=http://10.29.1.201:8888
|
|
export https_proxy=http://10.29.1.201:8888
|
|
|
|
unset HTTP_PROXY
|
|
unset HTTPS_PROXY
|
|
export HTTP_PROXY=http://10.29.1.201:8888
|
|
export HTTPS_PROXY=http://10.29.1.201:8888
|
|
|
|
LOCAL_DIR="/apps/yi/kaiyuan_pretraining/"
|
|
PHASE="phase2"
|
|
|
|
echo "Starting download of $PHASE data to $LOCAL_DIR..."
|
|
|
|
|
|
while true; do
|
|
hf download thu-pacman/PCMind-2.1-Kaiyuan-2B \
|
|
--repo-type dataset \
|
|
--local-dir "$LOCAL_DIR" \
|
|
--include "$PHASE/*"
|
|
|
|
if [ $? -eq 0 ]; then
|
|
echo "Download completed successfully!"
|
|
break
|
|
else
|
|
echo "Download interrupted or failed. Retrying in 10 seconds..."
|
|
sleep 10
|
|
fi
|
|
done |