From c0d02cf4d16761ce16d4d3e50a03914cb867dcdb Mon Sep 17 00:00:00 2001 From: Kaixi Hou Date: Thu, 30 Oct 2025 14:32:03 -0700 Subject: [PATCH] [NVIDIA] Add CI workloads for GB200 (#12242) --- .github/workflows/pr-test.yml | 30 +++++++++++++++++ scripts/ci/ci_install_dependency.sh | 40 ++++++++++++++++++++--- scripts/ci_monitor/ci_analyzer.py | 4 +++ scripts/ci_monitor/ci_analyzer_balance.py | 1 + test/srt/run_suite.py | 3 ++ 5 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index e99f6aa79..dd161c810 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -784,6 +784,35 @@ jobs: cd test/srt python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 + unit-test-backend-4-gpu-gb200: + needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels] + if: always() && !failure() && !cancelled() && + ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) + runs-on: 4-gpu-gb200 + strategy: + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + if: needs.check-changes.outputs.sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 45 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 + pr-test-finish: needs: [ @@ -808,6 +837,7 @@ jobs: unit-test-deepep-4-gpu, unit-test-deepep-8-gpu, unit-test-backend-4-gpu-b200, + unit-test-backend-4-gpu-gb200, ] if: always() runs-on: ubuntu-latest diff --git a/scripts/ci/ci_install_dependency.sh b/scripts/ci/ci_install_dependency.sh index a1d1a0c65..2e6664d12 100755 --- a/scripts/ci/ci_install_dependency.sh +++ b/scripts/ci/ci_install_dependency.sh @@ -5,6 +5,10 @@ set -euxo pipefail IS_BLACKWELL=${IS_BLACKWELL:-0} CU_VERSION="cu129" +# Detect system architecture +ARCH=$(uname -m) +echo "Detected architecture: ${ARCH}" + if [ "$CU_VERSION" = "cu130" ]; then NVRTC_SPEC="nvidia-cuda-nvrtc" else @@ -23,8 +27,20 @@ rm -rf /root/.cache/flashinfer # Install apt packages apt install -y git libnuma-dev libssl-dev pkg-config +# Check if protoc of correct architecture is already installed +if command -v protoc >/dev/null 2>&1; then + if protoc --version >/dev/null 2>&1; then + echo "protoc already installed: $(protoc --version)" + else + echo "protoc found but not runnable, reinstalling..." + INSTALL_PROTOC=1 + fi +else + INSTALL_PROTOC=1 +fi + # Install protoc for router build (gRPC protobuf compilation) -if ! command -v protoc &> /dev/null; then +if [ "${INSTALL_PROTOC:-0}" = "1" ]; then echo "Installing protoc..." if command -v apt-get &> /dev/null; then # Ubuntu/Debian @@ -37,9 +53,16 @@ if ! command -v protoc &> /dev/null; then fi cd /tmp - wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip - unzip protoc-32.0-linux-x86_64.zip -d /usr/local - rm protoc-32.0-linux-x86_64.zip + # Determine protoc architecture + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + PROTOC_ARCH="aarch_64" + else + PROTOC_ARCH="x86_64" + fi + PROTOC_ZIP="protoc-32.0-linux-${PROTOC_ARCH}.zip" + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/${PROTOC_ZIP} + unzip -o ${PROTOC_ZIP} -d /usr/local + rm ${PROTOC_ZIP} protoc --version cd - else @@ -52,6 +75,7 @@ if [ "$IS_BLACKWELL" = "1" ]; then # so we can only use pip with `--break-system-packages` PIP_CMD="pip" PIP_INSTALL_SUFFIX="--break-system-packages" + $PIP_CMD install --upgrade pip # Clean up existing installations $PIP_CMD uninstall -y sgl-kernel sglang $PIP_INSTALL_SUFFIX || true @@ -81,7 +105,13 @@ echo "SGL_KERNEL_VERSION_FROM_KERNEL=${SGL_KERNEL_VERSION_FROM_KERNEL} SGL_KERNE if [ "${CUSTOM_BUILD_SGL_KERNEL:-}" = "true" ]; then ls -alh sgl-kernel/dist - $PIP_CMD install sgl-kernel/dist/sgl_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}-cp310-abi3-manylinux2014_x86_64.whl --force-reinstall $PIP_INSTALL_SUFFIX + # Determine wheel architecture + if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then + WHEEL_ARCH="aarch64" + else + WHEEL_ARCH="x86_64" + fi + $PIP_CMD install sgl-kernel/dist/sgl_kernel-${SGL_KERNEL_VERSION_FROM_KERNEL}-cp310-abi3-manylinux2014_${WHEEL_ARCH}.whl --force-reinstall $PIP_INSTALL_SUFFIX else $PIP_CMD install sgl-kernel==${SGL_KERNEL_VERSION_FROM_SRT} --force-reinstall $PIP_INSTALL_SUFFIX fi diff --git a/scripts/ci_monitor/ci_analyzer.py b/scripts/ci_monitor/ci_analyzer.py index ddd084e04..d3c726ba4 100755 --- a/scripts/ci_monitor/ci_analyzer.py +++ b/scripts/ci_monitor/ci_analyzer.py @@ -112,6 +112,9 @@ class SGLangCIAnalyzer: "b200": [ "unit-test-backend-4-gpu-b200", ], + "gb200": [ + "unit-test-backend-4-gpu-gb200", + ], } stats = { @@ -180,6 +183,7 @@ class SGLangCIAnalyzer: "unit-test-deepep-8-gpu", "unit-test-backend-8-gpu-deepseek-v32", "unit-test-backend-4-gpu-b200", + "unit-test-backend-4-gpu-gb200", "vllm-dependency-test", "nightly-test-eval-text-models", "nightly-test-perf-text-models", diff --git a/scripts/ci_monitor/ci_analyzer_balance.py b/scripts/ci_monitor/ci_analyzer_balance.py index c3a92de09..818339fe1 100755 --- a/scripts/ci_monitor/ci_analyzer_balance.py +++ b/scripts/ci_monitor/ci_analyzer_balance.py @@ -181,6 +181,7 @@ class SGLangTestBalanceAnalyzer: "unit-test-backend-8-gpu-h200", "unit-test-backend-8-gpu-h20", "unit-test-backend-4-gpu-b200", + "unit-test-backend-4-gpu-gb200", "unit-test-deepep-4-gpu", "unit-test-deepep-8-gpu", "unit-test-backend-8-gpu-deepseek-v32", diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 7e65a2e2d..22238c057 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -185,6 +185,9 @@ suites = { TestFile("test_gpt_oss_4gpu.py", 600), TestFile("test_llama31_fp4.py", 300), ], + "per-commit-4-gpu-gb200": [ + TestFile("test_deepseek_v3_fp4_4gpu.py", 3600), + ], "per-commit-4-gpu-deepep": [ TestFile("ep/test_deepep_small.py", 531), TestFile("ep/test_mooncake_ep_small.py", 450),