Upload nightly test metrics to GH artifacts (#17696)

python/sglang/bench_one_batch_server.py

@@ -31,6 +31,7 @@ def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs):
             results,
             pydantic_result_filename=bench_args.pydantic_result_filename,
             model_path=server_args.model_path,
+            server_args=bench_args.server_args_for_metrics,
         )
 
     return results, server_info

python/sglang/test/bench_one_batch_server_internal.py

@@ -109,6 +109,7 @@ class BenchArgs:
     append_to_github_summary: bool = True
     seed: int = 42
     cache_hit_rate: float = 0.0
+    server_args_for_metrics: Optional[List[str]] = None
 
     @staticmethod
     def add_cli_args(parser: argparse.ArgumentParser):
@@ -192,6 +193,13 @@ class BenchArgs:
             help="Cache hit rate for benchmarking (0.0-1.0). "
             "0.0 means no cache hits (flush all), 0.4 means 40%% of input tokens are cached.",
         )
+        parser.add_argument(
+            "--server-args-for-metrics",
+            type=str,
+            nargs="*",
+            default=None,
+            help="Server launch arguments to record in metrics output (for tracking configurations).",
+        )
 
     @classmethod
     def from_cli_args(cls, args: argparse.Namespace):

python/sglang/test/nightly_bench_utils.py

@@ -31,6 +31,7 @@ class BenchmarkResult(BaseModel):
     acc_length: Optional[float] = None
     profile_link_extend: Optional[str] = None
     profile_link_decode: Optional[str] = None
+    server_args: Optional[List[str]] = None
 
     @staticmethod
     def help_str() -> str:
@@ -122,7 +123,10 @@ def generate_markdown_report(
 
 
 def save_results_as_pydantic_models(
-    results: List, pydantic_result_filename: str, model_path: str
+    results: List,
+    pydantic_result_filename: str,
+    model_path: str,
+    server_args: Optional[List[str]] = None,
 ):
     """Save benchmark results as JSON using Pydantic models."""
     json_results = []
@@ -167,6 +171,7 @@ def save_results_as_pydantic_models(
             acc_length=res.acc_length,
             profile_link_extend=profile_link_extend,
             profile_link_decode=profile_link_decode,
+            server_args=server_args,
         )
         json_results.append(benchmark_result.model_dump())
 

python/sglang/test/nightly_utils.py

@@ -93,6 +93,7 @@ class NightlyBenchmarkRunner:
         profile_path_prefix: str,
         json_output_file: str,
         extra_args: Optional[List[str]] = None,
+        server_args: Optional[List[str]] = None,
     ) -> List[str]:
         """Build the benchmark command with all required arguments.
 
@@ -104,6 +105,7 @@ class NightlyBenchmarkRunner:
             profile_path_prefix: Prefix for profile output files
             json_output_file: Path to JSON output file
             extra_args: Optional extra arguments to append to command
+            server_args: Optional server launch arguments to record in metrics
 
         Returns:
             List of command arguments ready for subprocess.run()
@@ -135,6 +137,11 @@ class NightlyBenchmarkRunner:
         if extra_args:
             command.extend(extra_args)
 
+        # Record server launch arguments in metrics for tracking configurations
+        if server_args:
+            command.append("--server-args-for-metrics")
+            command.extend(server_args)
+
         return command
 
     def run_benchmark_command(
@@ -192,17 +199,14 @@ class NightlyBenchmarkRunner:
                 f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
             )
 
-            # Clean up JSON file
-            os.remove(json_output_file)
+            # Note: JSON files are preserved for metrics collection by CI scripts
+            # They will be collected by scripts/ci/save_metrics.py
 
             return benchmark_results, True
 
         except Exception as e:
             desc = model_description or "model"
             print(f"Error loading benchmark results for {desc}: {e}")
-            # Try to clean up the file anyway
-            if os.path.exists(json_output_file):
-                os.remove(json_output_file)
             return benchmark_results, False
 
     def run_benchmark_for_model(
@@ -268,6 +272,7 @@ class NightlyBenchmarkRunner:
                 profile_path_prefix,
                 json_output_file,
                 extra_args=bench_args,
+                server_args=other_args,
             )
 
             result, cmd_success = self.run_benchmark_command(command, model_description)

python/sglang/test/performance_test_runner.py

@@ -77,6 +77,7 @@ def run_performance_test(
             input_lens=input_lens,
             output_lens=output_lens,
             other_args=model.extra_args,
+            variant=model.variant or "",
             extra_bench_args=extra_bench_args,
         )