Add nightly performance test for GPT-OSS 4GPU models (#12805)

scripts/ci/ci_install_dependency.sh

@@ -120,7 +120,7 @@ fi
 # Show current packages
 $PIP_CMD list
 
-$PIP_CMD install mooncake-transfer-engine==0.3.7.post2 "${NVRTC_SPEC}" py-spy scipy huggingface_hub[hf_xet] $PIP_INSTALL_SUFFIX
+$PIP_CMD install mooncake-transfer-engine==0.3.7.post2 "${NVRTC_SPEC}" py-spy scipy huggingface_hub[hf_xet] pytest $PIP_INSTALL_SUFFIX
 
 if [ "$IS_BLACKWELL" != "1" ]; then
     # For lmms_evals evaluating MMMU

test/srt/run_suite.py

@@ -219,6 +219,7 @@ suites = {
     ],
     "nightly-4-gpu-b200": [
         TestFile("test_fp4_moe.py", 300),
+        TestFile("test_nightly_gpt_oss_4gpu_perf.py", 600),
     ],
     "nightly-4-gpu": [],
     "nightly-8-gpu": [],

test/srt/test_nightly_gpt_oss_4gpu_perf.py

@@ -0,0 +1,134 @@
+import os
+import subprocess
+import time
+import unittest
+
+from sglang.bench_one_batch_server import BenchmarkResult, generate_markdown_report
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    is_in_ci,
+    popen_launch_server,
+    write_github_step_summary,
+)
+
+PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
+
+
+class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.models = [
+            ("lmsys/gpt-oss-120b-bf16", ["--tp", "4", "--cuda-graph-max-bs", "200"]),
+            (
+                "openai/gpt-oss-120b",
+                [
+                    "--tp",
+                    "4",
+                    "--cuda-graph-max-bs",
+                    "200",
+                    "--mem-fraction-static",
+                    "0.93",
+                ],
+            ),
+        ]
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.batch_sizes = [1, 1, 8, 16, 64]
+        cls.input_lens = (4096,)
+        cls.output_lens = (512,)
+        os.makedirs(PROFILE_DIR, exist_ok=True)
+        cls.full_report = f"## {cls.__name__}\n" + BenchmarkResult.help_str()
+
+    def test_bench_one_batch(self):
+        all_benchmark_results = []
+        all_model_succeed = True
+        for model_path, other_args in self.models:
+            benchmark_results = []
+            with self.subTest(model=model_path):
+                process = popen_launch_server(
+                    model=model_path,
+                    base_url=self.base_url,
+                    other_args=other_args,
+                    timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                )
+                try:
+
+                    profile_filename = (
+                        f"{model_path.replace('/', '_')}_{int(time.time())}"
+                    )
+                    profile_path_prefix = os.path.join(PROFILE_DIR, profile_filename)
+                    json_output_file = f"results_{model_path.replace('/', '_')}_{int(time.time())}.json"
+
+                    command = [
+                        "python3",
+                        "-m",
+                        "sglang.bench_one_batch_server",
+                        "--model",
+                        model_path,
+                        "--base-url",
+                        self.base_url,
+                        "--batch-size",
+                        *[str(x) for x in self.batch_sizes],
+                        "--input-len",
+                        *[str(x) for x in self.input_lens],
+                        "--output-len",
+                        *[str(x) for x in self.output_lens],
+                        "--show-report",
+                        "--profile",
+                        "--profile-by-stage",
+                        "--profile-filename-prefix",
+                        profile_path_prefix,
+                        f"--output-path={json_output_file}",
+                        "--no-append-to-github-summary",
+                    ]
+
+                    print(f"Running command: {' '.join(command)}")
+                    result = subprocess.run(command, capture_output=True, text=True)
+
+                    if result.returncode != 0:
+                        print(
+                            f"Error running benchmark for {model_path} with batch size:"
+                        )
+                        print(result.stderr)
+                        all_model_succeed = False
+                        continue
+
+                    # Load and deserialize JSON results
+                    if os.path.exists(json_output_file):
+                        import json
+
+                        with open(json_output_file, "r") as f:
+                            json_data = json.load(f)
+
+                        # Convert JSON data to BenchmarkResult objects
+                        for data in json_data:
+                            benchmark_result = BenchmarkResult(**data)
+                            all_benchmark_results.append(benchmark_result)
+                            benchmark_results.append(benchmark_result)
+
+                        print(
+                            f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
+                        )
+
+                        # Clean up JSON file
+                        os.remove(json_output_file)
+                    else:
+                        all_model_succeed = False
+                        print(f"Warning: JSON output file {json_output_file} not found")
+
+                finally:
+                    kill_process_tree(process.pid)
+
+                report_part = generate_markdown_report(PROFILE_DIR, benchmark_results)
+                self.full_report += report_part + "\n"
+
+        if is_in_ci():
+            write_github_step_summary(self.full_report)
+
+        if not all_model_succeed:
+            raise AssertionError("Some models failed the perf tests.")
+
+
+if __name__ == "__main__":
+    unittest.main()