diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index a3c2133cd..24fbdcd2b 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -182,7 +182,7 @@ jobs: - name: Run test timeout-minutes: 10 run: | - docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-a-test-1 + bash scripts/ci/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1 stage-b-test-small-1-gpu-amd: needs: [check-changes, stage-a-test-1-amd] @@ -222,7 +222,7 @@ jobs: - name: Run test timeout-minutes: 30 run: | - docker exec -w /sglang-checkout/test ci_sglang python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 + bash scripts/ci/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-small-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 multimodal-gen-test-1-gpu-amd: needs: [check-changes] diff --git a/scripts/ci/amd_ci_install_dependency.sh b/scripts/ci/amd_ci_install_dependency.sh index f8c1d5fc1..d2081e573 100755 --- a/scripts/ci/amd_ci_install_dependency.sh +++ b/scripts/ci/amd_ci_install_dependency.sh @@ -60,6 +60,41 @@ install_with_retry() { return 1 } +# Helper function to git clone with retries +git_clone_with_retry() { + local repo_url="$1" + local dest_dir="${2:-}" + local branch_args="${3:-}" + local max_attempts=3 + + for attempt in $(seq 1 $max_attempts); do + echo "Git clone attempt $attempt/$max_attempts: $repo_url" + + # prevent from partial clone + if [ -n "$dest_dir" ] && [ -d "$dest_dir" ]; then + rm -rf "$dest_dir" + fi + + if git \ + -c http.lowSpeedLimit=1000 \ + -c http.lowSpeedTime=30 \ + clone --depth 1 ${branch_args:+$branch_args} "$repo_url" "$dest_dir"; then + echo "Git clone succeeded." + return 0 + fi + + if [ $attempt -lt $max_attempts ]; then + echo "Git clone failed, retrying in 5 seconds..." + sleep 5 + fi + done + + echo "Git clone failed after $max_attempts attempts: $repo_url" + return 1 +} + + + case "${GPU_ARCH}" in mi35x) echo "Runner uses ${GPU_ARCH}; will fetch mi35x image." @@ -82,7 +117,9 @@ case "${GPU_ARCH}" in ;; esac -docker exec -w / ci_sglang git clone https://github.com/merrymercy/human-eval.git +#docker exec -w / ci_sglang git clone https://github.com/merrymercy/human-eval.git +git_clone_with_retry https://github.com/merrymercy/human-eval.git human-eval +docker cp human-eval ci_sglang:/ install_with_retry docker exec -w /human-eval ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e . docker exec -w / ci_sglang mkdir -p /dummy-grok