From f9dd90ac351e35cb1fb62629b6d956a91f1b8471 Mon Sep 17 00:00:00 2001
From: Douglas Yang <dyang@college.harvard.edu>
Date: Tue, 23 Dec 2025 01:00:37 -0800
Subject: [PATCH] fix: increasing H200 test timeout (#15600)

---
 .github/workflows/nightly-test-nvidia.yml         |  5 +++--
 python/sglang/srt/environ.py                      |  1 +
 python/sglang/test/run_combined_tests.py          |  9 +++++++++
 python/sglang/test/test_utils.py                  |  8 ++++++++
 test/registered/8-gpu-models/test_deepseek_v32.py | 12 ++++++------
 5 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml
index 568d179f0..a05117244 100644
--- a/.github/workflows/nightly-test-nvidia.yml
+++ b/.github/workflows/nightly-test-nvidia.yml
@@ -110,14 +110,15 @@ jobs:
 
       - name: Run common 8-GPU model tests
         if: always()
-        timeout-minutes: 200
+        timeout-minutes: 300
         env:
           TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
           PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
           GPU_CONFIG: "8-gpu-h200"
+          IS_H200: "1"
         run: |
           cd test
-          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=18000 --continue-on-error
 
       - name: Run test
         timeout-minutes: 30
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 3544685f8..76c463777 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -154,6 +154,7 @@ class Envs:
     SGLANG_TEST_STUCK_DP_CONTROLLER = EnvFloat(0)
     SGLANG_TEST_STUCK_TOKENIZER = EnvFloat(0)
     IS_BLACKWELL = EnvBool(False)
+    IS_H200 = EnvBool(False)
     SGLANG_SET_CPU_AFFINITY = EnvBool(False)
     SGLANG_PROFILE_WITH_STACK = EnvBool(True)
     SGLANG_PROFILE_RECORD_SHAPES = EnvBool(True)
diff --git a/python/sglang/test/run_combined_tests.py b/python/sglang/test/run_combined_tests.py
index b80a77d44..07dc59976 100644
--- a/python/sglang/test/run_combined_tests.py
+++ b/python/sglang/test/run_combined_tests.py
@@ -1,3 +1,4 @@
+import time
 from typing import List, Optional
 
 from sglang.test.accuracy_test_runner import (
@@ -115,6 +116,10 @@ def run_combined_tests(
                 all_passed = False
                 model_result["errors"].append(perf_result.error)
 
+            # Wait for GPU memory and port cleanup
+            print("\nWaiting 20 seconds for resource cleanup...")
+            time.sleep(20)
+
         # Run accuracy test
         if run_accuracy:
             acc_result: AccuracyTestResult = run_accuracy_test(
@@ -127,6 +132,10 @@ def run_combined_tests(
                 all_passed = False
                 model_result["errors"].append(acc_result.error)
 
+            # Wait for GPU memory and port cleanup
+            print("\nWaiting 20 seconds for resource cleanup...")
+            time.sleep(20)
+
         all_results.append(model_result)
 
     # Write performance report if we ran perf tests
diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py
index 2500d6193..7be72ad98 100644
--- a/python/sglang/test/test_utils.py
+++ b/python/sglang/test/test_utils.py
@@ -168,6 +168,11 @@ def is_blackwell_system():
     return envs.IS_BLACKWELL.get()
 
 
+def is_h200_system():
+    """Return whether it is running on an H200 system."""
+    return envs.IS_H200.get()
+
+
 def _use_cached_default_models(model_repo: str):
     cache_dir = os.getenv("DEFAULT_MODEL_CACHE_DIR")
     if cache_dir and model_repo:
@@ -193,6 +198,9 @@ if is_in_amd_ci():
 if is_blackwell_system():
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3000
 
+if is_h200_system():
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600
+
 
 def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None):
     assert url is not None
diff --git a/test/registered/8-gpu-models/test_deepseek_v32.py b/test/registered/8-gpu-models/test_deepseek_v32.py
index 7fbef77bb..2c5e64fc5 100644
--- a/test/registered/8-gpu-models/test_deepseek_v32.py
+++ b/test/registered/8-gpu-models/test_deepseek_v32.py
@@ -6,7 +6,7 @@ from sglang.test.performance_test_runner import PerformanceTestParams
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
 
-register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True)
+register_cuda_ci(est_time=18000, suite="nightly-8-gpu-common", nightly=True)
 
 DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
 
@@ -142,13 +142,13 @@ class TestDeepseekV32Unified(unittest.TestCase):
             accuracy_params=AccuracyTestParams(
                 dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
             ),
-            performance_params=PerformanceTestParams(
-                batch_sizes=[1, 8, 16, 64],
-                profile_dir="performance_profiles_deepseek_v32_nsa",
-            ),
+            performance_params=None,
         )
 
-    @unittest.skipIf(not is_blackwell_system(), "Requires B200")
+    @unittest.skipIf(
+        not is_blackwell_system(),
+        "Hardware agnostic - just using B200 for efficiency reasons",
+    )
     def test_deepseek_v32_b200(self):
         """Test DeepSeek V3.2 with GPQA evaluation using thinking mode (B200 only).