Implement nightly test workflow naming conventions (#13170)

test/srt/nightly/nightly_utils.py

@@ -29,6 +29,7 @@ class NightlyBenchmarkRunner:
         profile_dir: str,
         test_name: str,
         base_url: str,
+        gpu_config: str = None,
     ):
         """Initialize the benchmark runner.
 
@@ -36,11 +37,19 @@ class NightlyBenchmarkRunner:
             profile_dir: Directory to store performance profiles
             test_name: Name of the test (used for reporting)
             base_url: Base URL for the server
+            gpu_config: Optional GPU configuration string (e.g., "2-gpu-h100", "8-gpu-b200")
         """
         self.profile_dir = profile_dir
         self.test_name = test_name
         self.base_url = base_url
-        self.full_report = f"## {test_name}\n" + BenchmarkResult.help_str()
+        self.gpu_config = gpu_config or os.environ.get("GPU_CONFIG", "")
+
+        # Include GPU config in report header if available
+        header = f"## {test_name}"
+        if self.gpu_config:
+            header += f" ({self.gpu_config})"
+        header += "\n"
+        self.full_report = header + BenchmarkResult.help_str()
 
     def setup_profile_directory(self) -> None:
         """Create the profile directory if it doesn't exist."""
@@ -241,6 +250,11 @@ class NightlyBenchmarkRunner:
             )
 
             # Build and run benchmark command
+            # Prepare extra args with run_name if variant is specified
+            bench_args = list(extra_bench_args) if extra_bench_args else []
+            if variant:
+                bench_args.extend(["--run-name", variant])
+
             command = self.build_benchmark_command(
                 model_path,
                 batch_sizes,
@@ -248,7 +262,7 @@ class NightlyBenchmarkRunner:
                 output_lens,
                 profile_path_prefix,
                 json_output_file,
-                extra_args=extra_bench_args,
+                extra_args=bench_args,
             )
 
             result, cmd_success = self.run_benchmark_command(command, model_description)

test/srt/test_deepseek_v32_nsabackend.py

@@ -1,3 +1,4 @@
+import os
 import unittest
 from types import SimpleNamespace
 
@@ -14,6 +15,9 @@ from sglang.test.test_utils import (
 
 DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
 
+# Global list to collect results
+TEST_RESULTS = []
+
 
 class TestDeepseekV32NasBackend_flashmla(CustomTestCase):
     @classmethod
@@ -61,10 +65,16 @@ class TestDeepseekV32NasBackend_flashmla(CustomTestCase):
         print(f"{metrics=}")
 
         if is_in_ci():
-            write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n'
+            TEST_RESULTS.append(
+                {
+                    "variant": "flashmla",
+                    "prefill_backend": "flashmla_sparse",
+                    "decode_backend": "flashmla_kv",
+                    "kv_cache": "fp16",
+                    "accuracy": metrics["accuracy"],
+                }
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+        self.assertGreater(metrics["accuracy"], 0.935)
 
 
 class TestDeepseekV32NasBackend_fa3(CustomTestCase):
@@ -113,10 +123,16 @@ class TestDeepseekV32NasBackend_fa3(CustomTestCase):
         print(f"{metrics=}")
 
         if is_in_ci():
-            write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n'
+            TEST_RESULTS.append(
+                {
+                    "variant": "fa3",
+                    "prefill_backend": "fa3",
+                    "decode_backend": "fa3",
+                    "kv_cache": "fp16",
+                    "accuracy": metrics["accuracy"],
+                }
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+        self.assertGreater(metrics["accuracy"], 0.935)
 
 
 class TestDeepseekV32NasBackend_fp8kvcache(CustomTestCase):
@@ -163,10 +179,42 @@ class TestDeepseekV32NasBackend_fp8kvcache(CustomTestCase):
         print(f"{metrics=}")
 
         if is_in_ci():
-            write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n'
+            TEST_RESULTS.append(
+                {
+                    "variant": "fp8kvcache",
+                    "prefill_backend": "default",
+                    "decode_backend": "default",
+                    "kv_cache": "fp8_e4m3",
+                    "accuracy": metrics["accuracy"],
+                }
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+
+            # Write the summary table after all tests complete
+            _write_summary_table()
+        self.assertGreater(metrics["accuracy"], 0.935)
+
+
+def _write_summary_table():
+    """Write a markdown table with all test results."""
+    if not TEST_RESULTS:
+        return
+
+    gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200")
+
+    # Build table header
+    summary = f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy [{gpu_config}]\n\n"
+    summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy |\n"
+    summary += "|---------|-----------------|----------------|----------|----------|\n"
+
+    # Add each result as a row
+    for result in TEST_RESULTS:
+        summary += (
+            f"| {result['variant']} | {result['prefill_backend']} | "
+            f"{result['decode_backend']} | {result['kv_cache']} | "
+            f"{result['accuracy']:.3f} |\n"
+        )
+
+    write_github_step_summary(summary)
 
 
 if __name__ == "__main__":