fix: increasing H200 test timeout (#15600)

.github/workflows/nightly-test-nvidia.yml

@@ -110,14 +110,15 @@ jobs:
 
       - name: Run common 8-GPU model tests
         if: always()
-        timeout-minutes: 200
+        timeout-minutes: 300
         env:
           TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
           PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
           GPU_CONFIG: "8-gpu-h200"
+          IS_H200: "1"
         run: |
           cd test
-          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=18000 --continue-on-error
 
       - name: Run test
         timeout-minutes: 30

python/sglang/srt/environ.py

@@ -154,6 +154,7 @@ class Envs:
     SGLANG_TEST_STUCK_DP_CONTROLLER = EnvFloat(0)
     SGLANG_TEST_STUCK_TOKENIZER = EnvFloat(0)
     IS_BLACKWELL = EnvBool(False)
+    IS_H200 = EnvBool(False)
     SGLANG_SET_CPU_AFFINITY = EnvBool(False)
     SGLANG_PROFILE_WITH_STACK = EnvBool(True)
     SGLANG_PROFILE_RECORD_SHAPES = EnvBool(True)

python/sglang/test/run_combined_tests.py

@@ -1,3 +1,4 @@
+import time
 from typing import List, Optional
 
 from sglang.test.accuracy_test_runner import (
@@ -115,6 +116,10 @@ def run_combined_tests(
                 all_passed = False
                 model_result["errors"].append(perf_result.error)
 
+            # Wait for GPU memory and port cleanup
+            print("\nWaiting 20 seconds for resource cleanup...")
+            time.sleep(20)
+
         # Run accuracy test
         if run_accuracy:
             acc_result: AccuracyTestResult = run_accuracy_test(
@@ -127,6 +132,10 @@ def run_combined_tests(
                 all_passed = False
                 model_result["errors"].append(acc_result.error)
 
+            # Wait for GPU memory and port cleanup
+            print("\nWaiting 20 seconds for resource cleanup...")
+            time.sleep(20)
+
         all_results.append(model_result)
 
     # Write performance report if we ran perf tests

python/sglang/test/test_utils.py

@@ -168,6 +168,11 @@ def is_blackwell_system():
     return envs.IS_BLACKWELL.get()
 
 
+def is_h200_system():
+    """Return whether it is running on an H200 system."""
+    return envs.IS_H200.get()
+
+
 def _use_cached_default_models(model_repo: str):
     cache_dir = os.getenv("DEFAULT_MODEL_CACHE_DIR")
     if cache_dir and model_repo:
@@ -193,6 +198,9 @@ if is_in_amd_ci():
 if is_blackwell_system():
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3000
 
+if is_h200_system():
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600
+
 
 def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None):
     assert url is not None

test/registered/8-gpu-models/test_deepseek_v32.py

@@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
 
-register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True)
+register_cuda_ci(est_time=18000, suite="nightly-8-gpu-common", nightly=True)
 
 DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
 
@@ -142,13 +142,13 @@ class TestDeepseekV32Unified(unittest.TestCase):
             accuracy_params=AccuracyTestParams(
                 dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
             ),
-            performance_params=PerformanceTestParams(
-                batch_sizes=[1, 8, 16, 64],
-                profile_dir="performance_profiles_deepseek_v32_nsa",
-            ),
+            performance_params=None,
         )
 
-    @unittest.skipIf(not is_blackwell_system(), "Requires B200")
+    @unittest.skipIf(
+        not is_blackwell_system(),
+        "Hardware agnostic - just using B200 for efficiency reasons",
+    )
     def test_deepseek_v32_b200(self):
         """Test DeepSeek V3.2 with GPQA evaluation using thinking mode (B200 only).