chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Upload nightly test metrics to GH artifacts (#17696)

Browse Source
This commit is contained in:
Kangyan-Zhou
2026-01-25 14:35:14 -08:00
committed by GitHub
parent 97a36a72b7
commit 344eeaee90
8 changed files with 450 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-nvidia.yml
+83
View File
@@ -134,6 +134,26 @@ jobs:
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error
- name: Collect performance metrics
if: always()
run: |
python3 scripts/ci/save_metrics.py \
--gpu-config 8-gpu-h200 \
--partition ${{ matrix.partition }} \
--run-id ${{ github.run_id }} \
--output test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json \
--search-dir test/performance_profiles_8_gpu \
--search-dir test
- name: Upload partition metrics
if: always()
uses: actions/upload-artifact@v4
with:
name: metrics-8gpu-h200-partition-${{ matrix.partition }}
path: test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json
retention-days: 5
if-no-files-found: ignore
# General tests - 8 GPU H20
nightly-test-general-8-gpu-h20:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -187,6 +207,26 @@ jobs:
cd test
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=3
- name: Collect performance metrics
if: always()
run: |
python3 scripts/ci/save_metrics.py \
--gpu-config 8-gpu-b200 \
--partition ${{ matrix.partition }} \
--run-id ${{ github.run_id }} \
--output test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json \
--search-dir test/performance_profiles_8_gpu \
--search-dir test
- name: Upload partition metrics
if: always()
uses: actions/upload-artifact@v4
with:
name: metrics-8gpu-b200-partition-${{ matrix.partition }}
path: test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json
retention-days: 5
if-no-files-found: ignore
# Text model accuracy tests
nightly-test-text-accuracy-2-gpu-runner:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
@@ -403,6 +443,48 @@ jobs:
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
# Consolidate performance metrics from all 8-GPU jobs
consolidate-metrics:
if: github.repository == 'sgl-project/sglang' && always()
needs:
- nightly-test-general-8-gpu-h200
- nightly-test-general-8-gpu-b200
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Download all partition metrics
uses: actions/download-artifact@v4
with:
pattern: metrics-*
path: metrics/
merge-multiple: true
- name: List downloaded metrics
run: |
echo "Downloaded metrics files:"
find metrics/ -name "*.json" -type f 2>/dev/null || echo "No metrics files found"
- name: Merge metrics
run: |
python3 scripts/ci/merge_metrics.py \
--input-dir metrics/ \
--output consolidated-metrics-${{ github.run_id }}.json \
--run-id ${{ github.run_id }} \
--commit-sha ${{ github.sha }} \
--branch ${{ github.ref_name }}
- name: Upload consolidated metrics
uses: actions/upload-artifact@v4
with:
name: consolidated-metrics-${{ github.run_id }}
path: consolidated-metrics-${{ github.run_id }}.json
retention-days: 90
if-no-files-found: warn
# Final check job
check-all-jobs:
if: github.repository == 'sgl-project/sglang' && always()
@@ -420,6 +502,7 @@ jobs:
- nightly-test-multimodal-server-2-gpu
- nightly-test-perf-4-gpu-b200
- nightly-test-specialized-8-gpu-b200
- consolidate-metrics
runs-on: ubuntu-latest
steps:
- name: Check if any job failed
python/sglang/bench_one_batch_server.py
+1
View File
@@ -31,6 +31,7 @@ def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs):
results,
pydantic_result_filename=bench_args.pydantic_result_filename,
model_path=server_args.model_path,
server_args=bench_args.server_args_for_metrics,
)
return results, server_info
python/sglang/test/bench_one_batch_server_internal.py
+8
View File
@@ -109,6 +109,7 @@ class BenchArgs:
append_to_github_summary: bool = True
seed: int = 42
cache_hit_rate: float = 0.0
server_args_for_metrics: Optional[List[str]] = None
@staticmethod
def add_cli_args(parser: argparse.ArgumentParser):
@@ -192,6 +193,13 @@ class BenchArgs:
help="Cache hit rate for benchmarking (0.0-1.0). "
"0.0 means no cache hits (flush all), 0.4 means 40%% of input tokens are cached.",
)
parser.add_argument(
"--server-args-for-metrics",
type=str,
nargs="*",
default=None,
help="Server launch arguments to record in metrics output (for tracking configurations).",
)
@classmethod
def from_cli_args(cls, args: argparse.Namespace):
python/sglang/test/nightly_bench_utils.py
+6 -1
View File
@@ -31,6 +31,7 @@ class BenchmarkResult(BaseModel):
acc_length: Optional[float] = None
profile_link_extend: Optional[str] = None
profile_link_decode: Optional[str] = None
server_args: Optional[List[str]] = None
@staticmethod
def help_str() -> str:
@@ -122,7 +123,10 @@ def generate_markdown_report(
def save_results_as_pydantic_models(
results: List, pydantic_result_filename: str, model_path: str
results: List,
pydantic_result_filename: str,
model_path: str,
server_args: Optional[List[str]] = None,
):
"""Save benchmark results as JSON using Pydantic models."""
json_results = []
@@ -167,6 +171,7 @@ def save_results_as_pydantic_models(
acc_length=res.acc_length,
profile_link_extend=profile_link_extend,
profile_link_decode=profile_link_decode,
server_args=server_args,
)
json_results.append(benchmark_result.model_dump())
python/sglang/test/nightly_utils.py
+10 -5
View File
@@ -93,6 +93,7 @@ class NightlyBenchmarkRunner:
profile_path_prefix: str,
json_output_file: str,
extra_args: Optional[List[str]] = None,
server_args: Optional[List[str]] = None,
) -> List[str]:
"""Build the benchmark command with all required arguments.
@@ -104,6 +105,7 @@ class NightlyBenchmarkRunner:
profile_path_prefix: Prefix for profile output files
json_output_file: Path to JSON output file
extra_args: Optional extra arguments to append to command
server_args: Optional server launch arguments to record in metrics
Returns:
List of command arguments ready for subprocess.run()
@@ -135,6 +137,11 @@ class NightlyBenchmarkRunner:
if extra_args:
command.extend(extra_args)
# Record server launch arguments in metrics for tracking configurations
if server_args:
command.append("--server-args-for-metrics")
command.extend(server_args)
return command
def run_benchmark_command(
@@ -192,17 +199,14 @@ class NightlyBenchmarkRunner:
f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
)
# Clean up JSON file
os.remove(json_output_file)
# Note: JSON files are preserved for metrics collection by CI scripts
# They will be collected by scripts/ci/save_metrics.py
return benchmark_results, True
except Exception as e:
desc = model_description or "model"
print(f"Error loading benchmark results for {desc}: {e}")
# Try to clean up the file anyway
if os.path.exists(json_output_file):
os.remove(json_output_file)
return benchmark_results, False
def run_benchmark_for_model(
@@ -268,6 +272,7 @@ class NightlyBenchmarkRunner:
profile_path_prefix,
json_output_file,
extra_args=bench_args,
server_args=other_args,
)
result, cmd_success = self.run_benchmark_command(command, model_description)
python/sglang/test/performance_test_runner.py
+1
View File
@@ -77,6 +77,7 @@ def run_performance_test(
input_lens=input_lens,
output_lens=output_lens,
other_args=model.extra_args,
variant=model.variant or "",
extra_bench_args=extra_bench_args,
)
scripts/ci/merge_metrics.py
Executable
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
"""Merge per-partition metrics into a consolidated metrics file.
This script reads all per-partition metric JSON files and consolidates them
into a single JSON file with run-level metadata.
Usage:
python3 scripts/ci/merge_metrics.py \
--input-dir metrics/ \
--output consolidated-metrics-12345678.json \
--run-id 12345678 \
--commit-sha abc123def456
"""
import argparse
import glob
import json
import os
import sys
from datetime import datetime, timezone
def find_partition_files(input_dir: str) -> list[str]:
"""Find all partition metric files in the input directory."""
pattern = os.path.join(input_dir, "**/metrics-*.json")
return glob.glob(pattern, recursive=True)
def load_partition_metrics(filepath: str) -> dict | None:
"""Load a partition metrics file."""
try:
with open(filepath, "r", encoding="utf-8") as f:
return json.load(f)
except (json.JSONDecodeError, OSError) as e:
print(f"Warning: Failed to load {filepath}: {e}")
return None
def merge_metrics(
input_dir: str,
output_file: str,
run_id: str,
commit_sha: str,
branch: str | None = None,
) -> bool:
"""Merge all partition metrics into a consolidated file."""
run_date = datetime.now(timezone.utc).isoformat()
# Find all partition files
partition_files = find_partition_files(input_dir)
print(f"Found {len(partition_files)} partition file(s)")
all_results = []
if not partition_files:
print("No partition metrics files found")
else:
# Load all partition files
for filepath in sorted(partition_files):
print(f" Reading: {filepath}")
metrics = load_partition_metrics(filepath)
if metrics and "results" in metrics:
all_results.extend(metrics["results"])
print(f"Total results collected: {len(all_results)}")
# Create consolidated structure
consolidated = {
"run_id": run_id,
"run_date": run_date,
"commit_sha": commit_sha,
"branch": branch,
"results": all_results,
}
# Ensure output directory exists and write output
try:
os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
with open(output_file, "w", encoding="utf-8") as f:
json.dump(consolidated, f, indent=2)
if not partition_files:
print(f"Created empty consolidated file: {output_file}")
else:
print(f"Saved consolidated metrics to: {output_file}")
return True
except OSError as e:
print(f"Error writing consolidated file: {e}")
return False
def main():
parser = argparse.ArgumentParser(
description="Merge per-partition metrics into consolidated file"
)
parser.add_argument(
"--input-dir",
required=True,
help="Directory containing partition metric files",
)
parser.add_argument(
"--output",
required=True,
help="Output file path for consolidated metrics JSON",
)
parser.add_argument(
"--run-id",
required=True,
help="GitHub Actions run ID",
)
parser.add_argument(
"--commit-sha",
required=True,
help="Git commit SHA",
)
parser.add_argument(
"--branch",
default=None,
help="Git branch name (optional)",
)
args = parser.parse_args()
success = merge_metrics(
input_dir=args.input_dir,
output_file=args.output,
run_id=args.run_id,
commit_sha=args.commit_sha,
branch=args.branch,
)
sys.exit(0 if success else 1)
if __name__ == "__main__":
main()
scripts/ci/save_metrics.py
Executable
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""Collect and save performance metrics from nightly benchmark results.
This script reads benchmark result JSON files from performance profile directories
and saves them with metadata for artifact collection in CI.
Usage:
python3 scripts/ci/save_metrics.py \
--gpu-config 8-gpu-h200 \
--partition 0 \
--run-id 12345678 \
--output test/metrics-8gpu-h200-partition-0.json
"""
import argparse
import glob
import json
import os
import sys
from datetime import datetime, timezone
def find_result_files(search_dirs: list[str]) -> list[str]:
"""Find all results_*.json files in the given directories."""
result_files = set()
for search_dir in search_dirs:
if os.path.exists(search_dir):
pattern = os.path.join(search_dir, "**/results_*.json")
result_files.update(glob.glob(pattern, recursive=True))
return list(result_files)
def parse_result_file(filepath: str) -> list[dict]:
"""Parse a benchmark result JSON file."""
try:
with open(filepath, "r", encoding="utf-8") as f:
data = json.load(f)
if isinstance(data, list):
return data
return [data]
except (json.JSONDecodeError, OSError) as e:
print(f"Warning: Failed to parse {filepath}: {e}")
return []
def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
"""Transform a benchmark result to the metrics schema."""
# Handle None values safely for numeric conversions
latency = result.get("latency")
last_ttft = result.get("last_ttft")
return {
"batch_size": result.get("batch_size"),
"input_len": result.get("input_len"),
"output_len": result.get("output_len"),
"latency_ms": latency * 1000 if latency is not None else None,
"input_throughput": result.get("input_throughput"),
"output_throughput": result.get("output_throughput"),
"overall_throughput": result.get("overall_throughput"),
"ttft_ms": last_ttft * 1000 if last_ttft is not None else None,
"acc_length": result.get("acc_length"),
}
def group_results_by_model(
results: list[dict], gpu_config: str, partition: int
) -> list[dict]:
"""Group benchmark results by model, variant, and server_args."""
groups = {}
for result in results:
model_path = result.get("model_path", "unknown")
run_name = result.get("run_name", "default")
variant = run_name if run_name != "default" else None
server_args = result.get("server_args")
# Convert server_args list to tuple for use as dict key (lists are not hashable)
server_args_key = tuple(server_args) if server_args else None
key = (model_path, variant, server_args_key)
if key not in groups:
groups[key] = {
"gpu_config": gpu_config,
"partition": partition,
"model": model_path,
"variant": variant,
"server_args": server_args,
"benchmarks": [],
}
groups[key]["benchmarks"].append(
transform_benchmark_result(result, gpu_config, partition)
)
return list(groups.values())
def save_metrics(
gpu_config: str,
partition: int,
run_id: str,
output_file: str,
search_dirs: list[str],
) -> bool:
"""Collect metrics and save to output file."""
timestamp = datetime.now(timezone.utc).isoformat()
# Find all result files
result_files = find_result_files(search_dirs)
print(f"Found {len(result_files)} result file(s)")
grouped = []
if not result_files:
print("No benchmark result files found")
else:
# Parse all result files
all_results = []
for filepath in sorted(result_files):
print(f" Reading: {filepath}")
results = parse_result_file(filepath)
all_results.extend(results)
print(f"Total benchmark results: {len(all_results)}")
# Group by model/variant
grouped = group_results_by_model(all_results, gpu_config, partition)
# Create metrics structure
metrics = {
"run_id": run_id,
"timestamp": timestamp,
"gpu_config": gpu_config,
"partition": partition,
"results": grouped,
}
# Ensure output directory exists and write output
try:
os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
with open(output_file, "w", encoding="utf-8") as f:
json.dump(metrics, f, indent=2)
if not result_files:
print(f"Created empty metrics file: {output_file}")
else:
print(f"Saved metrics to: {output_file}")
return True
except OSError as e:
print(f"Error writing metrics file: {e}")
return False
def main():
parser = argparse.ArgumentParser(
description="Collect performance metrics from benchmark results"
)
parser.add_argument(
"--gpu-config",
required=True,
help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)",
)
parser.add_argument(
"--partition",
type=int,
required=True,
help="Partition number (0, 1, 2, etc.)",
)
parser.add_argument(
"--run-id",
required=True,
help="GitHub Actions run ID",
)
parser.add_argument(
"--output",
required=True,
help="Output file path for metrics JSON",
)
parser.add_argument(
"--search-dir",
action="append",
default=[],
dest="search_dirs",
help="Directory to search for result files (can be specified multiple times)",
)
args = parser.parse_args()
# Default search directories if none specified
search_dirs = args.search_dirs or [
"test/performance_profiles_8_gpu",
"test/performance_profiles_text_models",
"test/performance_profiles_vlms",
"test",
".",
]
success = save_metrics(
gpu_config=args.gpu_config,
partition=args.partition,
run_id=args.run_id,
output_file=args.output,
search_dirs=search_dirs,
)
sys.exit(0 if success else 1)
if __name__ == "__main__":
main()