Initial Commit

scripts/download_kaiyuan.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+source /apps/yi/.venv/bin/activate
+
+export HF_HUB_ENABLE_HF_TRANSFER=1
+export HF_HUB_DISABLE_SYMLINKS_WARNING=1
+
+# activate mirror if necessary
+
+export HF_ENDPOINT="https://hf-mirror.com/"
+
+unset http_proxy
+unset https_proxy
+export http_proxy=http://10.29.1.201:8888
+export https_proxy=http://10.29.1.201:8888
+
+unset HTTP_PROXY
+unset HTTPS_PROXY
+export HTTP_PROXY=http://10.29.1.201:8888
+export HTTPS_PROXY=http://10.29.1.201:8888
+
+LOCAL_DIR="/apps/yi/kaiyuan_pretraining/"
+PHASE="phase2"
+
+echo "Starting download of $PHASE data to $LOCAL_DIR..."
+
+
+while true; do
+    hf download thu-pacman/PCMind-2.1-Kaiyuan-2B \
+        --repo-type dataset \
+        --local-dir "$LOCAL_DIR" \
+        --include "$PHASE/*"
+
+    if [ $? -eq 0 ]; then
+        echo "Download completed successfully!"
+        break
+    else
+        echo "Download interrupted or failed. Retrying in 10 seconds..."
+        sleep 10
+    fi
+done