From 344eeaee90e3a926340bb92b589c93527a1b2447 Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Sun, 25 Jan 2026 14:35:14 -0800 Subject: [PATCH] Upload nightly test metrics to GH artifacts (#17696) --- .github/workflows/nightly-test-nvidia.yml | 83 +++++++ python/sglang/bench_one_batch_server.py | 1 + .../test/bench_one_batch_server_internal.py | 8 + python/sglang/test/nightly_bench_utils.py | 7 +- python/sglang/test/nightly_utils.py | 15 +- python/sglang/test/performance_test_runner.py | 1 + scripts/ci/merge_metrics.py | 134 ++++++++++++ scripts/ci/save_metrics.py | 207 ++++++++++++++++++ 8 files changed, 450 insertions(+), 6 deletions(-) create mode 100755 scripts/ci/merge_metrics.py create mode 100755 scripts/ci/save_metrics.py diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index d4d84a18d..83dae8e25 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -134,6 +134,26 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error + - name: Collect performance metrics + if: always() + run: | + python3 scripts/ci/save_metrics.py \ + --gpu-config 8-gpu-h200 \ + --partition ${{ matrix.partition }} \ + --run-id ${{ github.run_id }} \ + --output test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json \ + --search-dir test/performance_profiles_8_gpu \ + --search-dir test + + - name: Upload partition metrics + if: always() + uses: actions/upload-artifact@v4 + with: + name: metrics-8gpu-h200-partition-${{ matrix.partition }} + path: test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json + retention-days: 5 + if-no-files-found: ignore + # General tests - 8 GPU H20 nightly-test-general-8-gpu-h20: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') @@ -187,6 +207,26 @@ jobs: cd test IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=3 + - name: Collect performance metrics + if: always() + run: | + python3 scripts/ci/save_metrics.py \ + --gpu-config 8-gpu-b200 \ + --partition ${{ matrix.partition }} \ + --run-id ${{ github.run_id }} \ + --output test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json \ + --search-dir test/performance_profiles_8_gpu \ + --search-dir test + + - name: Upload partition metrics + if: always() + uses: actions/upload-artifact@v4 + with: + name: metrics-8gpu-b200-partition-${{ matrix.partition }} + path: test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json + retention-days: 5 + if-no-files-found: ignore + # Text model accuracy tests nightly-test-text-accuracy-2-gpu-runner: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner') @@ -403,6 +443,48 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400 + # Consolidate performance metrics from all 8-GPU jobs + consolidate-metrics: + if: github.repository == 'sgl-project/sglang' && always() + needs: + - nightly-test-general-8-gpu-h200 + - nightly-test-general-8-gpu-b200 + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Download all partition metrics + uses: actions/download-artifact@v4 + with: + pattern: metrics-* + path: metrics/ + merge-multiple: true + + - name: List downloaded metrics + run: | + echo "Downloaded metrics files:" + find metrics/ -name "*.json" -type f 2>/dev/null || echo "No metrics files found" + + - name: Merge metrics + run: | + python3 scripts/ci/merge_metrics.py \ + --input-dir metrics/ \ + --output consolidated-metrics-${{ github.run_id }}.json \ + --run-id ${{ github.run_id }} \ + --commit-sha ${{ github.sha }} \ + --branch ${{ github.ref_name }} + + - name: Upload consolidated metrics + uses: actions/upload-artifact@v4 + with: + name: consolidated-metrics-${{ github.run_id }} + path: consolidated-metrics-${{ github.run_id }}.json + retention-days: 90 + if-no-files-found: warn + # Final check job check-all-jobs: if: github.repository == 'sgl-project/sglang' && always() @@ -420,6 +502,7 @@ jobs: - nightly-test-multimodal-server-2-gpu - nightly-test-perf-4-gpu-b200 - nightly-test-specialized-8-gpu-b200 + - consolidate-metrics runs-on: ubuntu-latest steps: - name: Check if any job failed diff --git a/python/sglang/bench_one_batch_server.py b/python/sglang/bench_one_batch_server.py index c1d35637e..a2d8325f8 100644 --- a/python/sglang/bench_one_batch_server.py +++ b/python/sglang/bench_one_batch_server.py @@ -31,6 +31,7 @@ def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs): results, pydantic_result_filename=bench_args.pydantic_result_filename, model_path=server_args.model_path, + server_args=bench_args.server_args_for_metrics, ) return results, server_info diff --git a/python/sglang/test/bench_one_batch_server_internal.py b/python/sglang/test/bench_one_batch_server_internal.py index 12549132c..c7dc7b4f1 100644 --- a/python/sglang/test/bench_one_batch_server_internal.py +++ b/python/sglang/test/bench_one_batch_server_internal.py @@ -109,6 +109,7 @@ class BenchArgs: append_to_github_summary: bool = True seed: int = 42 cache_hit_rate: float = 0.0 + server_args_for_metrics: Optional[List[str]] = None @staticmethod def add_cli_args(parser: argparse.ArgumentParser): @@ -192,6 +193,13 @@ class BenchArgs: help="Cache hit rate for benchmarking (0.0-1.0). " "0.0 means no cache hits (flush all), 0.4 means 40%% of input tokens are cached.", ) + parser.add_argument( + "--server-args-for-metrics", + type=str, + nargs="*", + default=None, + help="Server launch arguments to record in metrics output (for tracking configurations).", + ) @classmethod def from_cli_args(cls, args: argparse.Namespace): diff --git a/python/sglang/test/nightly_bench_utils.py b/python/sglang/test/nightly_bench_utils.py index 0e9074cc1..88648aa44 100644 --- a/python/sglang/test/nightly_bench_utils.py +++ b/python/sglang/test/nightly_bench_utils.py @@ -31,6 +31,7 @@ class BenchmarkResult(BaseModel): acc_length: Optional[float] = None profile_link_extend: Optional[str] = None profile_link_decode: Optional[str] = None + server_args: Optional[List[str]] = None @staticmethod def help_str() -> str: @@ -122,7 +123,10 @@ def generate_markdown_report( def save_results_as_pydantic_models( - results: List, pydantic_result_filename: str, model_path: str + results: List, + pydantic_result_filename: str, + model_path: str, + server_args: Optional[List[str]] = None, ): """Save benchmark results as JSON using Pydantic models.""" json_results = [] @@ -167,6 +171,7 @@ def save_results_as_pydantic_models( acc_length=res.acc_length, profile_link_extend=profile_link_extend, profile_link_decode=profile_link_decode, + server_args=server_args, ) json_results.append(benchmark_result.model_dump()) diff --git a/python/sglang/test/nightly_utils.py b/python/sglang/test/nightly_utils.py index a5e854311..111eaeb42 100644 --- a/python/sglang/test/nightly_utils.py +++ b/python/sglang/test/nightly_utils.py @@ -93,6 +93,7 @@ class NightlyBenchmarkRunner: profile_path_prefix: str, json_output_file: str, extra_args: Optional[List[str]] = None, + server_args: Optional[List[str]] = None, ) -> List[str]: """Build the benchmark command with all required arguments. @@ -104,6 +105,7 @@ class NightlyBenchmarkRunner: profile_path_prefix: Prefix for profile output files json_output_file: Path to JSON output file extra_args: Optional extra arguments to append to command + server_args: Optional server launch arguments to record in metrics Returns: List of command arguments ready for subprocess.run() @@ -135,6 +137,11 @@ class NightlyBenchmarkRunner: if extra_args: command.extend(extra_args) + # Record server launch arguments in metrics for tracking configurations + if server_args: + command.append("--server-args-for-metrics") + command.extend(server_args) + return command def run_benchmark_command( @@ -192,17 +199,14 @@ class NightlyBenchmarkRunner: f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}" ) - # Clean up JSON file - os.remove(json_output_file) + # Note: JSON files are preserved for metrics collection by CI scripts + # They will be collected by scripts/ci/save_metrics.py return benchmark_results, True except Exception as e: desc = model_description or "model" print(f"Error loading benchmark results for {desc}: {e}") - # Try to clean up the file anyway - if os.path.exists(json_output_file): - os.remove(json_output_file) return benchmark_results, False def run_benchmark_for_model( @@ -268,6 +272,7 @@ class NightlyBenchmarkRunner: profile_path_prefix, json_output_file, extra_args=bench_args, + server_args=other_args, ) result, cmd_success = self.run_benchmark_command(command, model_description) diff --git a/python/sglang/test/performance_test_runner.py b/python/sglang/test/performance_test_runner.py index 8871316eb..4a3700bbc 100644 --- a/python/sglang/test/performance_test_runner.py +++ b/python/sglang/test/performance_test_runner.py @@ -77,6 +77,7 @@ def run_performance_test( input_lens=input_lens, output_lens=output_lens, other_args=model.extra_args, + variant=model.variant or "", extra_bench_args=extra_bench_args, ) diff --git a/scripts/ci/merge_metrics.py b/scripts/ci/merge_metrics.py new file mode 100755 index 000000000..0eb7cad8b --- /dev/null +++ b/scripts/ci/merge_metrics.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Merge per-partition metrics into a consolidated metrics file. + +This script reads all per-partition metric JSON files and consolidates them +into a single JSON file with run-level metadata. + +Usage: + python3 scripts/ci/merge_metrics.py \ + --input-dir metrics/ \ + --output consolidated-metrics-12345678.json \ + --run-id 12345678 \ + --commit-sha abc123def456 +""" + +import argparse +import glob +import json +import os +import sys +from datetime import datetime, timezone + + +def find_partition_files(input_dir: str) -> list[str]: + """Find all partition metric files in the input directory.""" + pattern = os.path.join(input_dir, "**/metrics-*.json") + return glob.glob(pattern, recursive=True) + + +def load_partition_metrics(filepath: str) -> dict | None: + """Load a partition metrics file.""" + try: + with open(filepath, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + print(f"Warning: Failed to load {filepath}: {e}") + return None + + +def merge_metrics( + input_dir: str, + output_file: str, + run_id: str, + commit_sha: str, + branch: str | None = None, +) -> bool: + """Merge all partition metrics into a consolidated file.""" + run_date = datetime.now(timezone.utc).isoformat() + + # Find all partition files + partition_files = find_partition_files(input_dir) + print(f"Found {len(partition_files)} partition file(s)") + + all_results = [] + if not partition_files: + print("No partition metrics files found") + else: + # Load all partition files + for filepath in sorted(partition_files): + print(f" Reading: {filepath}") + metrics = load_partition_metrics(filepath) + if metrics and "results" in metrics: + all_results.extend(metrics["results"]) + print(f"Total results collected: {len(all_results)}") + + # Create consolidated structure + consolidated = { + "run_id": run_id, + "run_date": run_date, + "commit_sha": commit_sha, + "branch": branch, + "results": all_results, + } + + # Ensure output directory exists and write output + try: + os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True) + with open(output_file, "w", encoding="utf-8") as f: + json.dump(consolidated, f, indent=2) + + if not partition_files: + print(f"Created empty consolidated file: {output_file}") + else: + print(f"Saved consolidated metrics to: {output_file}") + return True + except OSError as e: + print(f"Error writing consolidated file: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Merge per-partition metrics into consolidated file" + ) + parser.add_argument( + "--input-dir", + required=True, + help="Directory containing partition metric files", + ) + parser.add_argument( + "--output", + required=True, + help="Output file path for consolidated metrics JSON", + ) + parser.add_argument( + "--run-id", + required=True, + help="GitHub Actions run ID", + ) + parser.add_argument( + "--commit-sha", + required=True, + help="Git commit SHA", + ) + parser.add_argument( + "--branch", + default=None, + help="Git branch name (optional)", + ) + + args = parser.parse_args() + + success = merge_metrics( + input_dir=args.input_dir, + output_file=args.output, + run_id=args.run_id, + commit_sha=args.commit_sha, + branch=args.branch, + ) + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/save_metrics.py b/scripts/ci/save_metrics.py new file mode 100755 index 000000000..ac136e1b3 --- /dev/null +++ b/scripts/ci/save_metrics.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Collect and save performance metrics from nightly benchmark results. + +This script reads benchmark result JSON files from performance profile directories +and saves them with metadata for artifact collection in CI. + +Usage: + python3 scripts/ci/save_metrics.py \ + --gpu-config 8-gpu-h200 \ + --partition 0 \ + --run-id 12345678 \ + --output test/metrics-8gpu-h200-partition-0.json +""" + +import argparse +import glob +import json +import os +import sys +from datetime import datetime, timezone + + +def find_result_files(search_dirs: list[str]) -> list[str]: + """Find all results_*.json files in the given directories.""" + result_files = set() + for search_dir in search_dirs: + if os.path.exists(search_dir): + pattern = os.path.join(search_dir, "**/results_*.json") + result_files.update(glob.glob(pattern, recursive=True)) + return list(result_files) + + +def parse_result_file(filepath: str) -> list[dict]: + """Parse a benchmark result JSON file.""" + try: + with open(filepath, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, list): + return data + return [data] + except (json.JSONDecodeError, OSError) as e: + print(f"Warning: Failed to parse {filepath}: {e}") + return [] + + +def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict: + """Transform a benchmark result to the metrics schema.""" + # Handle None values safely for numeric conversions + latency = result.get("latency") + last_ttft = result.get("last_ttft") + + return { + "batch_size": result.get("batch_size"), + "input_len": result.get("input_len"), + "output_len": result.get("output_len"), + "latency_ms": latency * 1000 if latency is not None else None, + "input_throughput": result.get("input_throughput"), + "output_throughput": result.get("output_throughput"), + "overall_throughput": result.get("overall_throughput"), + "ttft_ms": last_ttft * 1000 if last_ttft is not None else None, + "acc_length": result.get("acc_length"), + } + + +def group_results_by_model( + results: list[dict], gpu_config: str, partition: int +) -> list[dict]: + """Group benchmark results by model, variant, and server_args.""" + groups = {} + + for result in results: + model_path = result.get("model_path", "unknown") + run_name = result.get("run_name", "default") + variant = run_name if run_name != "default" else None + server_args = result.get("server_args") + # Convert server_args list to tuple for use as dict key (lists are not hashable) + server_args_key = tuple(server_args) if server_args else None + + key = (model_path, variant, server_args_key) + if key not in groups: + groups[key] = { + "gpu_config": gpu_config, + "partition": partition, + "model": model_path, + "variant": variant, + "server_args": server_args, + "benchmarks": [], + } + + groups[key]["benchmarks"].append( + transform_benchmark_result(result, gpu_config, partition) + ) + + return list(groups.values()) + + +def save_metrics( + gpu_config: str, + partition: int, + run_id: str, + output_file: str, + search_dirs: list[str], +) -> bool: + """Collect metrics and save to output file.""" + timestamp = datetime.now(timezone.utc).isoformat() + + # Find all result files + result_files = find_result_files(search_dirs) + print(f"Found {len(result_files)} result file(s)") + + grouped = [] + if not result_files: + print("No benchmark result files found") + else: + # Parse all result files + all_results = [] + for filepath in sorted(result_files): + print(f" Reading: {filepath}") + results = parse_result_file(filepath) + all_results.extend(results) + print(f"Total benchmark results: {len(all_results)}") + + # Group by model/variant + grouped = group_results_by_model(all_results, gpu_config, partition) + + # Create metrics structure + metrics = { + "run_id": run_id, + "timestamp": timestamp, + "gpu_config": gpu_config, + "partition": partition, + "results": grouped, + } + + # Ensure output directory exists and write output + try: + os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True) + with open(output_file, "w", encoding="utf-8") as f: + json.dump(metrics, f, indent=2) + + if not result_files: + print(f"Created empty metrics file: {output_file}") + else: + print(f"Saved metrics to: {output_file}") + return True + except OSError as e: + print(f"Error writing metrics file: {e}") + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Collect performance metrics from benchmark results" + ) + parser.add_argument( + "--gpu-config", + required=True, + help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)", + ) + parser.add_argument( + "--partition", + type=int, + required=True, + help="Partition number (0, 1, 2, etc.)", + ) + parser.add_argument( + "--run-id", + required=True, + help="GitHub Actions run ID", + ) + parser.add_argument( + "--output", + required=True, + help="Output file path for metrics JSON", + ) + parser.add_argument( + "--search-dir", + action="append", + default=[], + dest="search_dirs", + help="Directory to search for result files (can be specified multiple times)", + ) + + args = parser.parse_args() + + # Default search directories if none specified + search_dirs = args.search_dirs or [ + "test/performance_profiles_8_gpu", + "test/performance_profiles_text_models", + "test/performance_profiles_vlms", + "test", + ".", + ] + + success = save_metrics( + gpu_config=args.gpu_config, + partition=args.partition, + run_id=args.run_id, + output_file=args.output, + search_dirs=search_dirs, + ) + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main()