Upload nightly test metrics to GH artifacts (#17696)

2026-01-25 14:35:14 -08:00
parent 97a36a72b7
commit 344eeaee90
8 changed files with 450 additions and 6 deletions
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Merge per-partition metrics into a consolidated metrics file.
+
+This script reads all per-partition metric JSON files and consolidates them
+into a single JSON file with run-level metadata.
+
+Usage:
+    python3 scripts/ci/merge_metrics.py \
+        --input-dir metrics/ \
+        --output consolidated-metrics-12345678.json \
+        --run-id 12345678 \
+        --commit-sha abc123def456
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_partition_files(input_dir: str) -> list[str]:
+    """Find all partition metric files in the input directory."""
+    pattern = os.path.join(input_dir, "**/metrics-*.json")
+    return glob.glob(pattern, recursive=True)
+
+
+def load_partition_metrics(filepath: str) -> dict | None:
+    """Load a partition metrics file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to load {filepath}: {e}")
+        return None
+
+
+def merge_metrics(
+    input_dir: str,
+    output_file: str,
+    run_id: str,
+    commit_sha: str,
+    branch: str | None = None,
+) -> bool:
+    """Merge all partition metrics into a consolidated file."""
+    run_date = datetime.now(timezone.utc).isoformat()
+
+    # Find all partition files
+    partition_files = find_partition_files(input_dir)
+    print(f"Found {len(partition_files)} partition file(s)")
+
+    all_results = []
+    if not partition_files:
+        print("No partition metrics files found")
+    else:
+        # Load all partition files
+        for filepath in sorted(partition_files):
+            print(f"  Reading: {filepath}")
+            metrics = load_partition_metrics(filepath)
+            if metrics and "results" in metrics:
+                all_results.extend(metrics["results"])
+        print(f"Total results collected: {len(all_results)}")
+
+    # Create consolidated structure
+    consolidated = {
+        "run_id": run_id,
+        "run_date": run_date,
+        "commit_sha": commit_sha,
+        "branch": branch,
+        "results": all_results,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(consolidated, f, indent=2)
+
+        if not partition_files:
+            print(f"Created empty consolidated file: {output_file}")
+        else:
+            print(f"Saved consolidated metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing consolidated file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Merge per-partition metrics into consolidated file"
+    )
+    parser.add_argument(
+        "--input-dir",
+        required=True,
+        help="Directory containing partition metric files",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for consolidated metrics JSON",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--commit-sha",
+        required=True,
+        help="Git commit SHA",
+    )
+    parser.add_argument(
+        "--branch",
+        default=None,
+        help="Git branch name (optional)",
+    )
+
+    args = parser.parse_args()
+
+    success = merge_metrics(
+        input_dir=args.input_dir,
+        output_file=args.output,
+        run_id=args.run_id,
+        commit_sha=args.commit_sha,
+        branch=args.branch,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Collect and save performance metrics from nightly benchmark results.
+
+This script reads benchmark result JSON files from performance profile directories
+and saves them with metadata for artifact collection in CI.
+
+Usage:
+    python3 scripts/ci/save_metrics.py \
+        --gpu-config 8-gpu-h200 \
+        --partition 0 \
+        --run-id 12345678 \
+        --output test/metrics-8gpu-h200-partition-0.json
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_result_files(search_dirs: list[str]) -> list[str]:
+    """Find all results_*.json files in the given directories."""
+    result_files = set()
+    for search_dir in search_dirs:
+        if os.path.exists(search_dir):
+            pattern = os.path.join(search_dir, "**/results_*.json")
+            result_files.update(glob.glob(pattern, recursive=True))
+    return list(result_files)
+
+
+def parse_result_file(filepath: str) -> list[dict]:
+    """Parse a benchmark result JSON file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, list):
+            return data
+        return [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {filepath}: {e}")
+        return []
+
+
+def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
+    """Transform a benchmark result to the metrics schema."""
+    # Handle None values safely for numeric conversions
+    latency = result.get("latency")
+    last_ttft = result.get("last_ttft")
+
+    return {
+        "batch_size": result.get("batch_size"),
+        "input_len": result.get("input_len"),
+        "output_len": result.get("output_len"),
+        "latency_ms": latency * 1000 if latency is not None else None,
+        "input_throughput": result.get("input_throughput"),
+        "output_throughput": result.get("output_throughput"),
+        "overall_throughput": result.get("overall_throughput"),
+        "ttft_ms": last_ttft * 1000 if last_ttft is not None else None,
+        "acc_length": result.get("acc_length"),
+    }
+
+
+def group_results_by_model(
+    results: list[dict], gpu_config: str, partition: int
+) -> list[dict]:
+    """Group benchmark results by model, variant, and server_args."""
+    groups = {}
+
+    for result in results:
+        model_path = result.get("model_path", "unknown")
+        run_name = result.get("run_name", "default")
+        variant = run_name if run_name != "default" else None
+        server_args = result.get("server_args")
+        # Convert server_args list to tuple for use as dict key (lists are not hashable)
+        server_args_key = tuple(server_args) if server_args else None
+
+        key = (model_path, variant, server_args_key)
+        if key not in groups:
+            groups[key] = {
+                "gpu_config": gpu_config,
+                "partition": partition,
+                "model": model_path,
+                "variant": variant,
+                "server_args": server_args,
+                "benchmarks": [],
+            }
+
+        groups[key]["benchmarks"].append(
+            transform_benchmark_result(result, gpu_config, partition)
+        )
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    partition: int,
+    run_id: str,
+    output_file: str,
+    search_dirs: list[str],
+) -> bool:
+    """Collect metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Find all result files
+    result_files = find_result_files(search_dirs)
+    print(f"Found {len(result_files)} result file(s)")
+
+    grouped = []
+    if not result_files:
+        print("No benchmark result files found")
+    else:
+        # Parse all result files
+        all_results = []
+        for filepath in sorted(result_files):
+            print(f"  Reading: {filepath}")
+            results = parse_result_file(filepath)
+            all_results.extend(results)
+        print(f"Total benchmark results: {len(all_results)}")
+
+        # Group by model/variant
+        grouped = group_results_by_model(all_results, gpu_config, partition)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "partition": partition,
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not result_files:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect performance metrics from benchmark results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)",
+    )
+    parser.add_argument(
+        "--partition",
+        type=int,
+        required=True,
+        help="Partition number (0, 1, 2, etc.)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--search-dir",
+        action="append",
+        default=[],
+        dest="search_dirs",
+        help="Directory to search for result files (can be specified multiple times)",
+    )
+
+    args = parser.parse_args()
+
+    # Default search directories if none specified
+    search_dirs = args.search_dirs or [
+        "test/performance_profiles_8_gpu",
+        "test/performance_profiles_text_models",
+        "test/performance_profiles_vlms",
+        "test",
+        ".",
+    ]
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        partition=args.partition,
+        run_id=args.run_id,
+        output_file=args.output,
+        search_dirs=search_dirs,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()