41 lines
924 B
Bash
41 lines
924 B
Bash
#!/bin/bash
|
|
|
|
# source /apps/yi/.venv/bin/activate
|
|
|
|
export HF_HUB_ENABLE_HF_TRANSFER=1
|
|
export HF_HUB_DISABLE_SYMLINKS_WARNING=1
|
|
|
|
# activate mirror if necessary
|
|
|
|
export HF_ENDPOINT="https://hf-mirror.com/"
|
|
|
|
# unset http_proxy
|
|
# unset https_proxy
|
|
# export http_proxy=http://10.29.1.201:8888
|
|
# export https_proxy=http://10.29.1.201:8888
|
|
|
|
# unset HTTP_PROXY
|
|
# unset HTTPS_PROXY
|
|
# export HTTP_PROXY=http://10.29.1.201:8888
|
|
# export HTTPS_PROXY=http://10.29.1.201:8888
|
|
|
|
LOCAL_DIR="/ssd1/yi/data/"
|
|
PHASE="phase1"
|
|
|
|
echo "Starting download of $PHASE data to $LOCAL_DIR..."
|
|
|
|
|
|
while true; do
|
|
hf download thu-pacman/PCMind-2.1-Kaiyuan-2B \
|
|
--repo-type dataset \
|
|
--local-dir "$LOCAL_DIR" \
|
|
--include "$PHASE/*"
|
|
|
|
if [ $? -eq 0 ]; then
|
|
echo "Download completed successfully!"
|
|
break
|
|
else
|
|
echo "Download interrupted or failed. Retrying in 10 seconds..."
|
|
sleep 10
|
|
fi
|
|
done |