#!/bin/bash

source /apps/yi/.venv/bin/activate

export HF_HUB_ENABLE_HF_TRANSFER=1
export HF_HUB_DISABLE_SYMLINKS_WARNING=1

# activate mirror if necessary

export HF_ENDPOINT="https://hf-mirror.com/"

unset http_proxy
unset https_proxy
export http_proxy=http://10.29.1.201:8888
export https_proxy=http://10.29.1.201:8888

unset HTTP_PROXY
unset HTTPS_PROXY
export HTTP_PROXY=http://10.29.1.201:8888
export HTTPS_PROXY=http://10.29.1.201:8888

LOCAL_DIR="/apps/yi/kaiyuan_pretraining/"
PHASE="phase2"

echo "Starting download of $PHASE data to $LOCAL_DIR..."


while true; do
    hf download thu-pacman/PCMind-2.1-Kaiyuan-2B \
        --repo-type dataset \
        --local-dir "$LOCAL_DIR" \
        --include "$PHASE/*"

    if [ $? -eq 0 ]; then
        echo "Download completed successfully!"
        break
    else
        echo "Download interrupted or failed. Retrying in 10 seconds..."
        sleep 10
    fi
done