Add runner utilization report workflow (#17234)
This commit is contained in:
Alison Shao
43
.github/workflows/runner-utilization.yml
vendored
Normal file
43
.github/workflows/runner-utilization.yml
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
name: Runner Utilization Report
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 8 * * *' # Daily at 8 AM UTC
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/runner-utilization.yml'
|
||||
- 'scripts/ci/runner_utilization_report.py'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
hours:
|
||||
description: 'Time window in hours'
|
||||
required: false
|
||||
default: '24'
|
||||
type: string
|
||||
filter:
|
||||
description: 'Filter runner labels (e.g., 5090, h200)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
report:
|
||||
name: Generate Report
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Generate Utilization Report
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python scripts/ci/runner_utilization_report.py \
|
||||
--repo ${{ github.repository }} \
|
||||
--hours ${{ inputs.hours || '6' }} \
|
||||
${{ inputs.filter && format('--filter {0}', inputs.filter) || '' }}
|
||||
309
scripts/ci/runner_utilization_report.py
Executable file
309
scripts/ci/runner_utilization_report.py
Executable file
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Runner Utilization Report
|
||||
|
||||
Analyzes GitHub Actions job data to calculate runner utilization metrics.
|
||||
Reports idle time, active time, and utilization percentage per runner label.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
# Labels to skip when grouping runners (GitHub default labels)
|
||||
DEFAULT_LABELS_TO_IGNORE = {"self-hosted", "Linux", "X64", "ARM64"}
|
||||
GITHUB_HOSTED_LABELS = {"ubuntu-latest", "ubuntu-22.04", "ubuntu-24.04"}
|
||||
|
||||
|
||||
def run_gh_command(args: list[str]) -> dict:
|
||||
"""Run gh CLI command and return JSON result."""
|
||||
result = subprocess.run(
|
||||
["gh", "api"] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f"gh api failed: {result.stderr}")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def get_workflow_runs(repo: str, hours: int = 24) -> list[dict]:
|
||||
"""Get workflow runs from the last N hours."""
|
||||
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
||||
|
||||
runs = []
|
||||
page = 1
|
||||
while True:
|
||||
data = run_gh_command(
|
||||
[
|
||||
f"repos/{repo}/actions/runs?per_page=100&page={page}",
|
||||
]
|
||||
)
|
||||
page_runs = data.get("workflow_runs", [])
|
||||
|
||||
# Filter by time
|
||||
for run in page_runs:
|
||||
created_at = parse_time(run.get("created_at"))
|
||||
if created_at and created_at >= since:
|
||||
runs.append(run)
|
||||
elif created_at and created_at < since:
|
||||
# Runs are ordered by created_at desc, so we can stop
|
||||
return runs
|
||||
|
||||
if len(page_runs) < 100:
|
||||
break
|
||||
page += 1
|
||||
if page > 20: # Safety limit
|
||||
break
|
||||
return runs
|
||||
|
||||
|
||||
def get_jobs_for_run(repo: str, run_id: int) -> list[dict]:
|
||||
"""Get all jobs for a workflow run."""
|
||||
jobs = []
|
||||
page = 1
|
||||
while True:
|
||||
data = run_gh_command(
|
||||
[
|
||||
f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}",
|
||||
]
|
||||
)
|
||||
jobs.extend(data.get("jobs", []))
|
||||
if len(data.get("jobs", [])) < 100:
|
||||
break
|
||||
page += 1
|
||||
if page > 5: # Safety limit
|
||||
break
|
||||
return jobs
|
||||
|
||||
|
||||
def get_runners(repo: str, online_only: bool = True) -> list[dict]:
|
||||
"""Get all self-hosted runners with pagination. Returns empty if no permission."""
|
||||
try:
|
||||
all_runners = []
|
||||
page = 1
|
||||
while True:
|
||||
data = run_gh_command(
|
||||
[f"repos/{repo}/actions/runners?per_page=100&page={page}"]
|
||||
)
|
||||
runners = data.get("runners", [])
|
||||
all_runners.extend(runners)
|
||||
if len(runners) < 100:
|
||||
break
|
||||
page += 1
|
||||
if page > 10: # Safety limit
|
||||
break
|
||||
if online_only:
|
||||
all_runners = [r for r in all_runners if r.get("status") == "online"]
|
||||
return all_runners
|
||||
except Exception as e:
|
||||
print(f"Warning: Cannot access runners API (need admin): {e}")
|
||||
return []
|
||||
|
||||
|
||||
def parse_time(time_str: str) -> datetime:
|
||||
"""Parse ISO timestamp to datetime."""
|
||||
if not time_str:
|
||||
return None
|
||||
return datetime.fromisoformat(time_str.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
# Known runner counts per label (fallback when API unavailable)
|
||||
KNOWN_RUNNER_COUNTS = {
|
||||
"1-gpu-5090": 16,
|
||||
"h200": 8,
|
||||
"h20": 4,
|
||||
"b200": 4,
|
||||
"amd": 8,
|
||||
"github-hosted": 20, # GitHub hosted runners (variable)
|
||||
"other": 10,
|
||||
}
|
||||
|
||||
|
||||
def calculate_utilization(repo: str, hours: int = 24, runner_filter: str = None):
|
||||
"""Calculate runner utilization metrics."""
|
||||
|
||||
print(f"Fetching workflow runs from last {hours} hours...")
|
||||
runs = get_workflow_runs(repo, hours)
|
||||
print(f"Found {len(runs)} workflow runs")
|
||||
|
||||
# Try to get online runners from API
|
||||
print("Fetching online runners...")
|
||||
runners = get_runners(repo, online_only=True)
|
||||
|
||||
# Build label -> set of online runner names from API
|
||||
api_label_runners = defaultdict(set)
|
||||
if runners:
|
||||
for runner in runners:
|
||||
for label in runner.get("labels", []):
|
||||
label_name = label.get("name", "")
|
||||
if label_name not in DEFAULT_LABELS_TO_IGNORE:
|
||||
api_label_runners[label_name].add(runner["name"])
|
||||
print(f"Got {len(runners)} online runners from API")
|
||||
else:
|
||||
print("No runner API access, will use observed runners from job data")
|
||||
|
||||
# Track runners seen in jobs (for labels not in API or when API unavailable)
|
||||
job_label_runners = defaultdict(set)
|
||||
label_jobs = defaultdict(list) # label -> list of job_info
|
||||
|
||||
total_runs = len(runs)
|
||||
for i, run in enumerate(runs):
|
||||
if (i + 1) % 50 == 0:
|
||||
print(f"Processing run {i+1}/{total_runs}...")
|
||||
|
||||
try:
|
||||
jobs = get_jobs_for_run(repo, run["id"])
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for job in jobs:
|
||||
runner_name = job.get("runner_name")
|
||||
if not runner_name:
|
||||
continue
|
||||
|
||||
created_at = parse_time(job.get("created_at"))
|
||||
started_at = parse_time(job.get("started_at"))
|
||||
completed_at = parse_time(job.get("completed_at"))
|
||||
|
||||
if not started_at or not completed_at:
|
||||
continue
|
||||
|
||||
duration = (completed_at - started_at).total_seconds()
|
||||
queue_time = (started_at - created_at).total_seconds() if created_at else 0
|
||||
job_info = {
|
||||
"start": started_at,
|
||||
"end": completed_at,
|
||||
"duration": duration,
|
||||
"queue_time": queue_time,
|
||||
"job_name": job["name"],
|
||||
"runner_name": runner_name,
|
||||
}
|
||||
|
||||
# Use job labels directly (available in job data)
|
||||
job_labels = job.get("labels", [])
|
||||
for label in job_labels:
|
||||
# Skip generic labels
|
||||
if label in DEFAULT_LABELS_TO_IGNORE | GITHUB_HOSTED_LABELS:
|
||||
continue
|
||||
job_label_runners[label].add(runner_name)
|
||||
label_jobs[label].append(job_info)
|
||||
|
||||
# Merge API runners and job-observed runners
|
||||
# Prefer API count (online runners) when available
|
||||
all_labels = set(api_label_runners.keys()) | set(job_label_runners.keys())
|
||||
|
||||
# Filter labels if specified
|
||||
if runner_filter:
|
||||
all_labels = {lbl for lbl in all_labels if runner_filter in lbl}
|
||||
|
||||
print(f"Tracking {len(all_labels)} runner labels: {sorted(all_labels)}")
|
||||
|
||||
# Calculate metrics per label
|
||||
window_seconds = hours * 3600
|
||||
|
||||
results = []
|
||||
|
||||
for label in sorted(all_labels):
|
||||
# Use API runner count if available, otherwise use job-observed count
|
||||
if label in api_label_runners and api_label_runners[label]:
|
||||
num_runners = len(api_label_runners[label])
|
||||
elif label in job_label_runners:
|
||||
num_runners = len(job_label_runners[label])
|
||||
else:
|
||||
num_runners = KNOWN_RUNNER_COUNTS.get(label, 1)
|
||||
|
||||
total_capacity_seconds = window_seconds * num_runners
|
||||
|
||||
jobs = label_jobs.get(label, [])
|
||||
total_active_seconds = sum(j["duration"] for j in jobs)
|
||||
|
||||
utilization = (
|
||||
(total_active_seconds / total_capacity_seconds * 100)
|
||||
if total_capacity_seconds > 0
|
||||
else 0
|
||||
)
|
||||
idle_seconds = total_capacity_seconds - total_active_seconds
|
||||
|
||||
# Calculate queue time metrics
|
||||
queue_times = [j["queue_time"] for j in jobs if j["queue_time"] > 0]
|
||||
avg_queue_time = sum(queue_times) / len(queue_times) if queue_times else 0
|
||||
max_queue_time = max(queue_times) if queue_times else 0
|
||||
|
||||
results.append(
|
||||
{
|
||||
"label": label,
|
||||
"num_runners": num_runners,
|
||||
"num_jobs": len(jobs),
|
||||
"total_active_hours": total_active_seconds / 3600,
|
||||
"total_idle_hours": idle_seconds / 3600,
|
||||
"total_capacity_hours": total_capacity_seconds / 3600,
|
||||
"utilization_pct": utilization,
|
||||
"avg_queue_min": avg_queue_time / 60,
|
||||
"max_queue_min": max_queue_time / 60,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def format_report(results: list[dict], hours: int) -> str:
|
||||
"""Format results as markdown report."""
|
||||
lines = [
|
||||
"# Runner Utilization Report",
|
||||
"",
|
||||
f"**Time window:** Last {hours} hours",
|
||||
f"**Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
|
||||
"",
|
||||
"## Summary by Runner Label",
|
||||
"",
|
||||
"| Label | Runners | Jobs | Active (hrs) | Utilization | Avg Queue | Max Queue |",
|
||||
"|-------|---------|------|--------------|-------------|-----------|-----------|",
|
||||
]
|
||||
|
||||
for r in results:
|
||||
utilization_bar = "█" * int(r["utilization_pct"] / 10) + "░" * (
|
||||
10 - int(r["utilization_pct"] / 10)
|
||||
)
|
||||
lines.append(
|
||||
f"| {r['label']} | {r['num_runners']} | {r['num_jobs']} | "
|
||||
f"{r['total_active_hours']:.1f} | "
|
||||
f"{r['utilization_pct']:.1f}% {utilization_bar} | "
|
||||
f"{r['avg_queue_min']:.1f}m | {r['max_queue_min']:.1f}m |"
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate runner utilization report")
|
||||
parser.add_argument("--repo", default="sgl-project/sglang", help="GitHub repo")
|
||||
parser.add_argument("--hours", type=int, default=24, help="Time window in hours")
|
||||
parser.add_argument(
|
||||
"--filter", type=str, help="Filter runner labels (e.g., '5090', 'h200')"
|
||||
)
|
||||
parser.add_argument("--output", type=str, help="Output file (default: stdout)")
|
||||
args = parser.parse_args()
|
||||
|
||||
results = calculate_utilization(args.repo, args.hours, args.filter)
|
||||
report = format_report(results, args.hours)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
f.write(report)
|
||||
print(f"Report written to {args.output}")
|
||||
else:
|
||||
print(report)
|
||||
|
||||
# Also write to GITHUB_STEP_SUMMARY if available
|
||||
summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if summary_file:
|
||||
with open(summary_file, "a") as f:
|
||||
f.write(report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user