CI: fix NFS EBUSY error in PR test workflow (#13460)
Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com> Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
alisonshao
99
.github/workflows/pr-test.yml
vendored
99
.github/workflows/pr-test.yml
vendored
@@ -21,6 +21,9 @@ concurrency:
|
||||
group: pr-test-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
|
||||
jobs:
|
||||
# =============================================== check changes ====================================================
|
||||
check-changes:
|
||||
@@ -232,6 +235,11 @@ jobs:
|
||||
cd sgl-kernel
|
||||
pytest tests/
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
sgl-kernel-mla-test:
|
||||
needs: [check-changes, sgl-kernel-build-wheels]
|
||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
@@ -263,6 +271,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 test_mla_deepseek_v3.py
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
sgl-kernel-benchmark-test:
|
||||
needs: [check-changes, sgl-kernel-build-wheels]
|
||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
||||
@@ -307,24 +320,35 @@ jobs:
|
||||
|
||||
echo "All benchmark tests completed!"
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
# =============================================== multimodal_gen ====================================================
|
||||
# multimodal-gen-test:
|
||||
# needs: [check-changes]
|
||||
# if: needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
# runs-on: 1-gpu-runner
|
||||
# steps:
|
||||
# - name: Checkout code
|
||||
# uses: actions/checkout@v4
|
||||
multimodal-gen-test:
|
||||
needs: [check-changes]
|
||||
if: needs.check-changes.outputs.multimodal_gen == 'true'
|
||||
runs-on: 1-gpu-runner
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
# CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
|
||||
|
||||
# - name: Run diffusion server tests
|
||||
# timeout-minutes: 60
|
||||
# run: |
|
||||
# cd python
|
||||
# pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
|
||||
- name: Run diffusion server tests
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
cd python
|
||||
pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
# Remove logs directory to prevent NFS lock files
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
# Adding a single CUDA13 smoke test to verify that the kernel builds and runs
|
||||
# TODO: Add back this test when it can pass on CI
|
||||
@@ -390,6 +414,11 @@ jobs:
|
||||
# temporarily put backend-independent cpu tests here
|
||||
python3 run_suite.py --hw cpu --suite default
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
unit-test-backend-1-gpu:
|
||||
needs: [check-changes, stage-a-test-1, sgl-kernel-build-wheels]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -424,6 +453,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
unit-test-backend-2-gpu:
|
||||
needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -457,6 +491,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
unit-test-backend-4-gpu:
|
||||
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -618,6 +657,11 @@ jobs:
|
||||
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency
|
||||
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
performance-test-1-gpu-part-2:
|
||||
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -671,6 +715,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
performance-test-1-gpu-part-3:
|
||||
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -718,6 +767,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_batch_scaling
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
performance-test-2-gpu:
|
||||
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -777,6 +831,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
accuracy-test-1-gpu:
|
||||
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -809,6 +868,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 test_eval_accuracy_large.py
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
accuracy-test-2-gpu:
|
||||
needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
@@ -841,6 +905,11 @@ jobs:
|
||||
cd test/srt
|
||||
python3 test_moe_eval_accuracy_large.py
|
||||
|
||||
- name: Cleanup logs directory
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf python/sglang/logs || true
|
||||
|
||||
unit-test-deepep-4-gpu:
|
||||
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
|
||||
if: always() && !failure() && !cancelled() &&
|
||||
|
||||
Reference in New Issue
Block a user