diff --git a/.github/workflows/runner-utilization.yml b/.github/workflows/runner-utilization.yml new file mode 100644 index 000000000..ef772d77d --- /dev/null +++ b/.github/workflows/runner-utilization.yml @@ -0,0 +1,43 @@ +name: Runner Utilization Report + +on: + schedule: + - cron: '0 8 * * *' # Daily at 8 AM UTC + pull_request: + paths: + - '.github/workflows/runner-utilization.yml' + - 'scripts/ci/runner_utilization_report.py' + workflow_dispatch: + inputs: + hours: + description: 'Time window in hours' + required: false + default: '24' + type: string + filter: + description: 'Filter runner labels (e.g., 5090, h200)' + required: false + type: string + +jobs: + report: + name: Generate Report + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Generate Utilization Report + timeout-minutes: 30 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python scripts/ci/runner_utilization_report.py \ + --repo ${{ github.repository }} \ + --hours ${{ inputs.hours || '6' }} \ + ${{ inputs.filter && format('--filter {0}', inputs.filter) || '' }} diff --git a/scripts/ci/runner_utilization_report.py b/scripts/ci/runner_utilization_report.py new file mode 100755 index 000000000..f260e7b9f --- /dev/null +++ b/scripts/ci/runner_utilization_report.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +Runner Utilization Report + +Analyzes GitHub Actions job data to calculate runner utilization metrics. +Reports idle time, active time, and utilization percentage per runner label. +""" + +import argparse +import json +import os +import subprocess +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +# Labels to skip when grouping runners (GitHub default labels) +DEFAULT_LABELS_TO_IGNORE = {"self-hosted", "Linux", "X64", "ARM64"} +GITHUB_HOSTED_LABELS = {"ubuntu-latest", "ubuntu-22.04", "ubuntu-24.04"} + + +def run_gh_command(args: list[str]) -> dict: + """Run gh CLI command and return JSON result.""" + result = subprocess.run( + ["gh", "api"] + args, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise Exception(f"gh api failed: {result.stderr}") + return json.loads(result.stdout) + + +def get_workflow_runs(repo: str, hours: int = 24) -> list[dict]: + """Get workflow runs from the last N hours.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + + runs = [] + page = 1 + while True: + data = run_gh_command( + [ + f"repos/{repo}/actions/runs?per_page=100&page={page}", + ] + ) + page_runs = data.get("workflow_runs", []) + + # Filter by time + for run in page_runs: + created_at = parse_time(run.get("created_at")) + if created_at and created_at >= since: + runs.append(run) + elif created_at and created_at < since: + # Runs are ordered by created_at desc, so we can stop + return runs + + if len(page_runs) < 100: + break + page += 1 + if page > 20: # Safety limit + break + return runs + + +def get_jobs_for_run(repo: str, run_id: int) -> list[dict]: + """Get all jobs for a workflow run.""" + jobs = [] + page = 1 + while True: + data = run_gh_command( + [ + f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}", + ] + ) + jobs.extend(data.get("jobs", [])) + if len(data.get("jobs", [])) < 100: + break + page += 1 + if page > 5: # Safety limit + break + return jobs + + +def get_runners(repo: str, online_only: bool = True) -> list[dict]: + """Get all self-hosted runners with pagination. Returns empty if no permission.""" + try: + all_runners = [] + page = 1 + while True: + data = run_gh_command( + [f"repos/{repo}/actions/runners?per_page=100&page={page}"] + ) + runners = data.get("runners", []) + all_runners.extend(runners) + if len(runners) < 100: + break + page += 1 + if page > 10: # Safety limit + break + if online_only: + all_runners = [r for r in all_runners if r.get("status") == "online"] + return all_runners + except Exception as e: + print(f"Warning: Cannot access runners API (need admin): {e}") + return [] + + +def parse_time(time_str: str) -> datetime: + """Parse ISO timestamp to datetime.""" + if not time_str: + return None + return datetime.fromisoformat(time_str.replace("Z", "+00:00")) + + +# Known runner counts per label (fallback when API unavailable) +KNOWN_RUNNER_COUNTS = { + "1-gpu-5090": 16, + "h200": 8, + "h20": 4, + "b200": 4, + "amd": 8, + "github-hosted": 20, # GitHub hosted runners (variable) + "other": 10, +} + + +def calculate_utilization(repo: str, hours: int = 24, runner_filter: str = None): + """Calculate runner utilization metrics.""" + + print(f"Fetching workflow runs from last {hours} hours...") + runs = get_workflow_runs(repo, hours) + print(f"Found {len(runs)} workflow runs") + + # Try to get online runners from API + print("Fetching online runners...") + runners = get_runners(repo, online_only=True) + + # Build label -> set of online runner names from API + api_label_runners = defaultdict(set) + if runners: + for runner in runners: + for label in runner.get("labels", []): + label_name = label.get("name", "") + if label_name not in DEFAULT_LABELS_TO_IGNORE: + api_label_runners[label_name].add(runner["name"]) + print(f"Got {len(runners)} online runners from API") + else: + print("No runner API access, will use observed runners from job data") + + # Track runners seen in jobs (for labels not in API or when API unavailable) + job_label_runners = defaultdict(set) + label_jobs = defaultdict(list) # label -> list of job_info + + total_runs = len(runs) + for i, run in enumerate(runs): + if (i + 1) % 50 == 0: + print(f"Processing run {i+1}/{total_runs}...") + + try: + jobs = get_jobs_for_run(repo, run["id"]) + except Exception: + continue + + for job in jobs: + runner_name = job.get("runner_name") + if not runner_name: + continue + + created_at = parse_time(job.get("created_at")) + started_at = parse_time(job.get("started_at")) + completed_at = parse_time(job.get("completed_at")) + + if not started_at or not completed_at: + continue + + duration = (completed_at - started_at).total_seconds() + queue_time = (started_at - created_at).total_seconds() if created_at else 0 + job_info = { + "start": started_at, + "end": completed_at, + "duration": duration, + "queue_time": queue_time, + "job_name": job["name"], + "runner_name": runner_name, + } + + # Use job labels directly (available in job data) + job_labels = job.get("labels", []) + for label in job_labels: + # Skip generic labels + if label in DEFAULT_LABELS_TO_IGNORE | GITHUB_HOSTED_LABELS: + continue + job_label_runners[label].add(runner_name) + label_jobs[label].append(job_info) + + # Merge API runners and job-observed runners + # Prefer API count (online runners) when available + all_labels = set(api_label_runners.keys()) | set(job_label_runners.keys()) + + # Filter labels if specified + if runner_filter: + all_labels = {lbl for lbl in all_labels if runner_filter in lbl} + + print(f"Tracking {len(all_labels)} runner labels: {sorted(all_labels)}") + + # Calculate metrics per label + window_seconds = hours * 3600 + + results = [] + + for label in sorted(all_labels): + # Use API runner count if available, otherwise use job-observed count + if label in api_label_runners and api_label_runners[label]: + num_runners = len(api_label_runners[label]) + elif label in job_label_runners: + num_runners = len(job_label_runners[label]) + else: + num_runners = KNOWN_RUNNER_COUNTS.get(label, 1) + + total_capacity_seconds = window_seconds * num_runners + + jobs = label_jobs.get(label, []) + total_active_seconds = sum(j["duration"] for j in jobs) + + utilization = ( + (total_active_seconds / total_capacity_seconds * 100) + if total_capacity_seconds > 0 + else 0 + ) + idle_seconds = total_capacity_seconds - total_active_seconds + + # Calculate queue time metrics + queue_times = [j["queue_time"] for j in jobs if j["queue_time"] > 0] + avg_queue_time = sum(queue_times) / len(queue_times) if queue_times else 0 + max_queue_time = max(queue_times) if queue_times else 0 + + results.append( + { + "label": label, + "num_runners": num_runners, + "num_jobs": len(jobs), + "total_active_hours": total_active_seconds / 3600, + "total_idle_hours": idle_seconds / 3600, + "total_capacity_hours": total_capacity_seconds / 3600, + "utilization_pct": utilization, + "avg_queue_min": avg_queue_time / 60, + "max_queue_min": max_queue_time / 60, + } + ) + + return results + + +def format_report(results: list[dict], hours: int) -> str: + """Format results as markdown report.""" + lines = [ + "# Runner Utilization Report", + "", + f"**Time window:** Last {hours} hours", + f"**Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}", + "", + "## Summary by Runner Label", + "", + "| Label | Runners | Jobs | Active (hrs) | Utilization | Avg Queue | Max Queue |", + "|-------|---------|------|--------------|-------------|-----------|-----------|", + ] + + for r in results: + utilization_bar = "█" * int(r["utilization_pct"] / 10) + "░" * ( + 10 - int(r["utilization_pct"] / 10) + ) + lines.append( + f"| {r['label']} | {r['num_runners']} | {r['num_jobs']} | " + f"{r['total_active_hours']:.1f} | " + f"{r['utilization_pct']:.1f}% {utilization_bar} | " + f"{r['avg_queue_min']:.1f}m | {r['max_queue_min']:.1f}m |" + ) + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description="Generate runner utilization report") + parser.add_argument("--repo", default="sgl-project/sglang", help="GitHub repo") + parser.add_argument("--hours", type=int, default=24, help="Time window in hours") + parser.add_argument( + "--filter", type=str, help="Filter runner labels (e.g., '5090', 'h200')" + ) + parser.add_argument("--output", type=str, help="Output file (default: stdout)") + args = parser.parse_args() + + results = calculate_utilization(args.repo, args.hours, args.filter) + report = format_report(results, args.hours) + + if args.output: + with open(args.output, "w") as f: + f.write(report) + print(f"Report written to {args.output}") + else: + print(report) + + # Also write to GITHUB_STEP_SUMMARY if available + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file: + with open(summary_file, "a") as f: + f.write(report) + + +if __name__ == "__main__": + main()