diff --git a/.github/workflows/amd-ci-job-monitor.yml b/.github/workflows/amd-ci-job-monitor.yml new file mode 100644 index 000000000..1d89deb9a --- /dev/null +++ b/.github/workflows/amd-ci-job-monitor.yml @@ -0,0 +1,149 @@ +name: AMD CI Job Monitor + +on: + schedule: + - cron: '0 0 * * *' # Daily at midnight UTC + pull_request: + paths: + - '.github/workflows/amd-ci-job-monitor.yml' + - 'scripts/ci/query_job_status.py' + workflow_dispatch: + inputs: + hours: + description: 'Time window in hours' + required: false + default: '24' + type: string + job_filter: + description: 'Job name filter (leave empty for all AMD jobs)' + required: false + type: string + +jobs: + # Single job filter mode + custom-report: + name: Custom Job Report + if: ${{ inputs.job_filter }} + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install tabulate + + - name: Generate Custom Job Report + timeout-minutes: 30 + run: | + python scripts/ci/query_job_status.py \ + --repo ${{ github.repository }} \ + --job "${{ inputs.job_filter }}" \ + --workflow "pr-test-amd.yml" \ + --hours ${{ inputs.hours || '24' }} \ + --summary + + # Parse workflow files to get job names dynamically + parse-workflows: + name: Parse Workflow Jobs + if: ${{ !inputs.job_filter }} + runs-on: ubuntu-latest + outputs: + pr_jobs: ${{ steps.parse.outputs.pr_jobs }} + nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Parse workflow files + id: parse + run: | + # Parse pr-test-amd.yml and extract job names (exclude utility jobs) + # Excluded: call-gate, check-changes, pr-test-amd-finish, cancel, check-all-jobs + pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \ + grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs)$' | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT + echo "PR jobs: $pr_jobs" + + # Parse nightly-test-amd.yml and extract job names (exclude utility jobs) + # Excluded: check-all-jobs + nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-amd.yml | \ + grep -v -E '^(check-all-jobs)$' | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT + echo "Nightly jobs: $nightly_jobs" + + # PR CI reports using dynamic matrix + pr-ci-reports: + name: PR - ${{ matrix.job_name }} + needs: parse-workflows + if: ${{ !inputs.job_filter }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_jobs) }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install tabulate + + - name: Generate Report + timeout-minutes: 15 + run: | + python scripts/ci/query_job_status.py \ + --repo ${{ github.repository }} \ + --job "${{ matrix.job_name }}" \ + --workflow "pr-test-amd.yml" \ + --hours ${{ inputs.hours || '24' }} \ + --summary + + # Nightly AMD test reports using dynamic matrix + nightly-reports: + name: Nightly - ${{ matrix.job_name }} + needs: parse-workflows + if: ${{ !inputs.job_filter }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_jobs) }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install tabulate + + - name: Generate Nightly Report + timeout-minutes: 15 + run: | + python scripts/ci/query_job_status.py \ + --repo ${{ github.repository }} \ + --job "${{ matrix.job_name }}" \ + --workflow "nightly-test-amd.yml" \ + --hours ${{ inputs.hours || '24' }} \ + --summary diff --git a/scripts/ci/query_job_status.py b/scripts/ci/query_job_status.py new file mode 100755 index 000000000..da17a2814 --- /dev/null +++ b/scripts/ci/query_job_status.py @@ -0,0 +1,925 @@ +#!/usr/bin/env python3 +""" +Query GitHub Actions job status for specific jobs. + +Usage: + python scripts/ci/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" + python scripts/ci/query_job_status.py --job "stage-c-test-large-8-gpu-amd-mi35x" --hours 48 + python scripts/ci/query_job_status.py --job "AMD" --workflow pr-test-amd.yml + +Requirements: + pip install tabulate +""" + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime, timedelta, timezone +from typing import Optional + +try: + from tabulate import tabulate +except ImportError: + print("Please install tabulate: pip install tabulate") + exit(1) + + +def check_gh_cli_available() -> bool: + """Check if gh CLI is installed and authenticated.""" + try: + result = subprocess.run( + ["gh", "--version"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return False + + # Check if authenticated + auth_result = subprocess.run( + ["gh", "auth", "status"], + capture_output=True, + text=True, + ) + if auth_result.returncode != 0: + print( + "Error: gh CLI is not authenticated. Please run 'gh auth login' first.", + file=sys.stderr, + ) + print(f"Details: {auth_result.stderr}", file=sys.stderr) + return False + + return True + except FileNotFoundError: + print( + "Error: gh CLI is not installed. Please install it from https://cli.github.com/", + file=sys.stderr, + ) + return False + + +def run_gh_command(args: list[str]) -> dict: + """Run gh CLI command and return JSON result.""" + try: + result = subprocess.run( + ["gh", "api"] + args, + capture_output=True, + text=True, + ) + except FileNotFoundError: + raise Exception("gh CLI not found. Please install from https://cli.github.com/") + + if result.returncode != 0: + raise Exception(f"gh api failed: {result.stderr}") + return json.loads(result.stdout) + + +def parse_time(time_str: str) -> Optional[datetime]: + """Parse ISO timestamp to datetime.""" + if not time_str: + return None + return datetime.fromisoformat(time_str.replace("Z", "+00:00")) + + +def format_time(time_str: str) -> str: + """Format ISO timestamp to readable format in UTC.""" + if not time_str: + return "-" + dt = parse_time(time_str) + if dt: + # Ensure UTC + dt_utc = dt.astimezone(timezone.utc) + return dt_utc.strftime("%m-%d %H:%M") + return "-" + + +def get_workflow_runs(repo: str, workflow: str, hours: int = 24) -> list[dict]: + """Get workflow runs from the last N hours.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + + runs = [] + page = 1 + while True: + url = f"repos/{repo}/actions/runs?per_page=100&page={page}" + if workflow: + url = f"repos/{repo}/actions/workflows/{workflow}/runs?per_page=100&page={page}" + + data = run_gh_command([url]) + page_runs = data.get("workflow_runs", []) + + for run in page_runs: + created_at = parse_time(run.get("created_at")) + if created_at and created_at >= since: + runs.append(run) + elif created_at and created_at < since: + return runs + + if len(page_runs) < 100: + break + page += 1 + if page > 20: + break + return runs + + +def get_jobs_for_run(repo: str, run_id: int) -> list[dict]: + """Get all jobs for a workflow run.""" + jobs = [] + page = 1 + while True: + data = run_gh_command( + [f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}"] + ) + jobs.extend(data.get("jobs", [])) + if len(data.get("jobs", [])) < 100: + break + page += 1 + if page > 5: + break + return jobs + + +def get_pr_number_from_run(run: dict) -> Optional[int]: + """Extract PR number from run data.""" + # Try to get from pull_requests array + prs = run.get("pull_requests", []) + if prs: + return prs[0].get("number") + return None + + +def query_jobs( + repo: str, + job_filter: str, + workflow: str = None, + hours: int = 24, + status_filter: str = None, +) -> list[dict]: + """Query jobs matching the filter.""" + + print(f"Fetching workflow runs from last {hours} hours...", file=sys.stderr) + runs = get_workflow_runs(repo, workflow, hours) + print(f"Found {len(runs)} workflow runs", file=sys.stderr) + + results = [] + total_runs = len(runs) + + for i, run in enumerate(runs): + if (i + 1) % 20 == 0: + print(f"Processing run {i+1}/{total_runs}...", file=sys.stderr) + + try: + jobs = get_jobs_for_run(repo, run["id"]) + except Exception as e: + print( + f"Warning: Failed to get jobs for run {run['id']}: {e}", file=sys.stderr + ) + continue + + pr_number = get_pr_number_from_run(run) + branch = run.get("head_branch", "") + run_status = run.get("status", "unknown") + run_conclusion = run.get("conclusion") or "-" + + for job in jobs: + job_name = job.get("name", "") + + # Filter by job name + # Use prefix matching to avoid e.g. "stage-c-test-large-8-gpu-amd" + # also matching "stage-c-test-large-8-gpu-amd-mi35x" + job_name_lower = job_name.lower() + filter_lower = job_filter.lower() + if not job_name_lower.startswith(filter_lower): + continue + # If there are characters after the filter, ensure it's not a + # continuation of the base job name (e.g., "-mi35x") + if len(job_name_lower) > len(filter_lower): + next_char = job_name_lower[len(filter_lower)] + if next_char not in (" ", "("): + continue + + # Filter by status if specified + if status_filter and job.get("status") != status_filter: + continue + + job_status = job.get("status", "unknown") + runner_name = job.get("runner_name") or "-" + + # Detect stuck/ghost jobs: + # - Job is in_progress but no runner assigned + # - Job is in_progress but workflow run is cancelled/completed + is_stuck = False + if job_status == "in_progress": + if runner_name == "-": + is_stuck = True + elif run_status == "completed" and run_conclusion in ( + "cancelled", + "failure", + ): + is_stuck = True + + results.append( + { + "job_name": job_name, + "status": job_status, + "conclusion": job.get("conclusion") or "-", + "created_at": job.get("created_at", ""), + "started_at": job.get("started_at", ""), + "completed_at": job.get("completed_at", ""), + "runner_name": runner_name, + "run_id": run["id"], + "run_status": run_status, + "run_conclusion": run_conclusion, + "pr_number": pr_number, + "branch": branch, + "html_url": job.get("html_url", ""), + "is_stuck": is_stuck, + } + ) + + return results + + +def calculate_duration(started_at: str, completed_at: str) -> str: + """Calculate duration between start and completion.""" + if not started_at or not completed_at: + return "-" + start = parse_time(started_at) + end = parse_time(completed_at) + if start and end: + duration = (end - start).total_seconds() + if duration < 0: + return "-" # Invalid data, skip + minutes = int(duration // 60) + seconds = int(duration % 60) + if minutes >= 60: + hours = minutes // 60 + minutes = minutes % 60 + return f"{hours}h{minutes}m" + return f"{minutes}m{seconds}s" + return "-" + + +def calculate_queue_time( + created_at: str, + started_at: str, + status: str = None, + report_time: datetime = None, +) -> str: + """ + Calculate queue time between creation and start. + + For queued/waiting jobs that haven't truly started yet, calculate + queue time as (report_time - created_at) and mark as "still queuing". + """ + if not created_at: + return "-" + + created = parse_time(created_at) + if not created: + return "-" + + # For queued/waiting jobs, calculate time since creation + if status in ("queued", "waiting"): + if report_time: + queue_seconds = (report_time - created).total_seconds() + else: + queue_seconds = (datetime.now(timezone.utc) - created).total_seconds() + + if queue_seconds < 0: + return "-" + + minutes = int(queue_seconds // 60) + seconds = int(queue_seconds % 60) + if minutes >= 60: + hours = minutes // 60 + minutes = minutes % 60 + return f"{hours}h{minutes}m (queuing)" + return f"{minutes}m{seconds}s (queuing)" + + # For completed/in_progress jobs, calculate actual queue time + if not started_at: + return "-" + + started = parse_time(started_at) + if not started: + return "-" + + queue_seconds = (started - created).total_seconds() + if queue_seconds < 0: + return "-" # Invalid data + + minutes = int(queue_seconds // 60) + seconds = int(queue_seconds % 60) + if minutes >= 60: + hours = minutes // 60 + minutes = minutes % 60 + return f"{hours}h{minutes}m" + return f"{minutes}m{seconds}s" + + +def process_results( + results: list[dict], repo: str, report_time: datetime = None +) -> dict: + """ + Process raw results into structured data for presentation. + Returns a dictionary containing: + - status_summary: dict of job_name -> status counts + - sorted_results: list of results sorted by created_at descending + - active_jobs: list of in_progress/queued/waiting jobs (excluding stuck) + - stuck_jobs: list of stuck/ghost jobs + - failed_jobs: list of failed jobs + - processed_jobs: list of jobs with calculated fields (queue_time, duration, etc.) + """ + if report_time is None: + report_time = datetime.now(timezone.utc) + + if not results: + return { + "status_summary": {}, + "sorted_results": [], + "active_jobs": [], + "stuck_jobs": [], + "failed_jobs": [], + "processed_jobs": [], + } + + # Group by job name for summary + status_summary = {} + for r in results: + job_name = r["job_name"] + status = r["status"] + conclusion = r.get("conclusion", "-") + is_stuck = r.get("is_stuck", False) + if job_name not in status_summary: + status_summary[job_name] = { + "in_progress": 0, + "queued": 0, + "waiting": 0, + "stuck": 0, + "success": 0, + "failure": 0, + "cancelled": 0, + } + if is_stuck: + status_summary[job_name]["stuck"] += 1 + elif status == "completed": + # For completed jobs, count by conclusion + if conclusion == "success": + status_summary[job_name]["success"] += 1 + elif conclusion == "failure": + status_summary[job_name]["failure"] += 1 + elif conclusion in ("cancelled", "timed_out", "action_required"): + status_summary[job_name]["cancelled"] += 1 + elif status in status_summary[job_name]: + status_summary[job_name][status] += 1 + + # Sort by created_at descending + sorted_results = sorted(results, key=lambda x: x["created_at"], reverse=True) + + # Filter into categories (mutually exclusive) + active_jobs = [ + r + for r in results + if r.get("status") in ("in_progress", "queued", "waiting") + and not r.get("is_stuck", False) + ] + stuck_jobs = [r for r in results if r.get("is_stuck", False)] + # Only include jobs with conclusion "failure" + # Exclude stuck jobs to avoid double-counting + failed_jobs = [ + r + for r in results + if r.get("conclusion", "-") == "failure" and not r.get("is_stuck", False) + ] + + # Process jobs with calculated fields + processed_jobs = [] + for r in sorted_results: + processed = r.copy() + processed["created_formatted"] = format_time(r["created_at"]) + processed["started_formatted"] = format_time(r["started_at"]) + processed["queue_time"] = calculate_queue_time( + r["created_at"], r["started_at"], r["status"], report_time + ) + processed["duration"] = calculate_duration(r["started_at"], r["completed_at"]) + # Use the job's html_url for direct link to the specific job + processed["url"] = ( + r.get("html_url") or f"https://github.com/{repo}/actions/runs/{r['run_id']}" + ) + + if r["pr_number"]: + processed["pr_info"] = f"PR#{r['pr_number']}" + else: + processed["pr_info"] = r["branch"] if r["branch"] else "-" + + # Status display with stuck marker + if r.get("is_stuck", False): + processed["status_display"] = f"STUCK ({r['status']})" + else: + processed["status_display"] = r["status"] + + processed_jobs.append(processed) + + return { + "status_summary": status_summary, + "sorted_results": sorted_results, + "active_jobs": active_jobs, + "stuck_jobs": stuck_jobs, + "failed_jobs": failed_jobs, + "processed_jobs": processed_jobs, + } + + +def print_table( + results: list[dict], repo: str, generated_time: str, report_time: datetime = None +): + """Print results as a formatted table using tabulate.""" + print("") + print(f"Report generated: {generated_time} UTC") + print("Note: All times are in UTC") + print("") + + if not results: + print("No jobs found matching the filter.") + return + + # Process data + data = process_results(results, repo, report_time) + status_summary = data["status_summary"] + processed_jobs = data["processed_jobs"] + active_jobs = data["active_jobs"] + stuck_jobs = data["stuck_jobs"] + + # Print summary table + print("\n" + "=" * 100) + print("SUMMARY BY JOB NAME") + print("=" * 100) + + summary_data = [] + for job_name, counts in sorted(status_summary.items()): + summary_data.append( + [ + job_name, + counts["in_progress"], + counts["queued"], + counts["waiting"], + counts["stuck"], + counts["success"], + counts["failure"], + counts["cancelled"], + ] + ) + + print( + tabulate( + summary_data, + headers=[ + "Job Name", + "Running", + "Queued", + "Waiting", + "Stuck", + "Success", + "Failure", + "Cancelled", + ], + tablefmt="grid", + ) + ) + + # Print detailed table + print("\n" + "=" * 100) + print("DETAILED JOB LIST") + print("=" * 100) + + detail_data = [] + for p in processed_jobs: + detail_data.append( + [ + p["job_name"], + p["status_display"], + p["conclusion"], + p["created_formatted"], + p["started_formatted"], + p["queue_time"], + p["duration"], + p["runner_name"] or "-", + p["pr_info"], + p["run_id"], + ] + ) + + print( + tabulate( + detail_data, + headers=[ + "Job Name", + "Status", + "Conclusion", + "Created", + "Started", + "Queue", + "Duration", + "Runner", + "PR/Branch", + "Run ID", + ], + tablefmt="grid", + ) + ) + + # Print links for active jobs (use processed_jobs for correct queue_time) + if active_jobs: + print("\n" + "=" * 100) + print("ACTIVE JOB LINKS") + print("=" * 100) + + link_data = [] + for r in active_jobs: + # Find the corresponding processed job to get pre-calculated fields + p = next( + ( + p + for p in processed_jobs + if p["run_id"] == r["run_id"] and p["job_name"] == r["job_name"] + ), + None, + ) + if p: + link_data.append( + [ + p["job_name"], + p["status"], + p["queue_time"], + p["pr_info"], + p["runner_name"] or "-", + p["url"], + ] + ) + + print( + tabulate( + link_data, + headers=["Job Name", "Status", "Queue", "PR/Branch", "Runner", "URL"], + tablefmt="simple", + ) + ) + + # Print stuck jobs (use processed_jobs for correct data) + if stuck_jobs: + print("\n" + "=" * 100) + print("STUCK/GHOST JOBS (in_progress but no runner or workflow cancelled)") + print("=" * 100) + + stuck_data = [] + for r in stuck_jobs: + # Find the corresponding processed job + p = next( + ( + p + for p in processed_jobs + if p["run_id"] == r["run_id"] and p["job_name"] == r["job_name"] + ), + None, + ) + if p: + run_info = f"{r.get('run_status', '-')}/{r.get('run_conclusion', '-')}" + stuck_data.append( + [ + p["job_name"], + p["status"], + run_info, + p["pr_info"], + p["runner_name"] or "-", + p["url"], + ] + ) + + print( + tabulate( + stuck_data, + headers=[ + "Job Name", + "Job Status", + "Run Status/Conclusion", + "PR/Branch", + "Runner", + "URL", + ], + tablefmt="simple", + ) + ) + + +def format_markdown( + results: list[dict], + repo: str, + job_filter: str, + hours: int, + generated_time: str, + report_time: datetime = None, +) -> str: + """Format results as markdown for GitHub Actions summary.""" + lines = [] + + # Header + lines.append(f"# Job Status Report: `{job_filter}`") + lines.append("") + lines.append(f"**Time window:** Last {hours} hours") + lines.append(f"**Generated:** {generated_time} UTC") + lines.append(f"**Total jobs found:** {len(results)}") + lines.append("") + lines.append("> **Note:** All times are displayed in UTC") + lines.append("") + + if not results: + lines.append("> No jobs found matching the filter.") + return "\n".join(lines) + + # Process data using shared function + data = process_results(results, repo, report_time) + status_summary = data["status_summary"] + processed_jobs = data["processed_jobs"] + active_jobs = data["active_jobs"] + stuck_jobs = data["stuck_jobs"] + failed_jobs = data["failed_jobs"] + + # Summary table + lines.append("## Summary by Job Name") + lines.append("") + lines.append( + "> **Status meanings:** Running = executing, Queued = waiting for runner, Waiting = waiting for dependent jobs, Stuck = ghost job, Cancelled = cancelled/timed_out" + ) + lines.append("") + lines.append( + "| Job Name | Running | Queued | Waiting | Stuck | Success | Failure | Cancelled |" + ) + lines.append( + "|----------|---------|--------|---------|-------|---------|---------|-----------|" + ) + + for job_name, counts in sorted(status_summary.items()): + running = f"**{counts['in_progress']}**" if counts["in_progress"] > 0 else "0" + queued = f"**{counts['queued']}**" if counts["queued"] > 0 else "0" + waiting = f"**{counts['waiting']}**" if counts["waiting"] > 0 else "0" + stuck = f"**{counts['stuck']}**" if counts["stuck"] > 0 else "0" + success = str(counts["success"]) + failure = f"**{counts['failure']}**" if counts["failure"] > 0 else "0" + cancelled = str(counts["cancelled"]) + lines.append( + f"| `{job_name}` | {running} | {queued} | {waiting} | {stuck} | {success} | {failure} | {cancelled} |" + ) + + lines.append("") + + # Active jobs section + if active_jobs: + lines.append("## Active Jobs") + lines.append("") + lines.append( + "| Status | Job Name | Created | Started | Queue | PR/Branch | Runner | Link |" + ) + lines.append( + "|--------|----------|---------|---------|-------|-----------|--------|------|" + ) + + for r in sorted( + active_jobs, key=lambda x: (x["status"], x["created_at"]), reverse=True + ): + # Find the processed version for this job + p = next( + ( + p + for p in processed_jobs + if p["run_id"] == r["run_id"] and p["job_name"] == r["job_name"] + ), + None, + ) + if p: + lines.append( + f"| {p['status']} | `{p['job_name']}` | {p['created_formatted']} | {p['started_formatted']} | {p['queue_time']} | {p['pr_info']} | `{p['runner_name'] or '-'}` | [View]({p['url']}) |" + ) + + lines.append("") + + # Stuck/Ghost jobs section + if stuck_jobs: + lines.append("## Stuck/Ghost Jobs") + lines.append("") + lines.append( + "> Jobs showing `in_progress` but have no runner assigned or workflow run is cancelled" + ) + lines.append("") + lines.append( + "| Job Status | Run Status | Job Name | PR/Branch | Runner | Link |" + ) + lines.append( + "|------------|------------|----------|-----------|--------|------|" + ) + + for r in sorted(stuck_jobs, key=lambda x: x["created_at"], reverse=True): + p = next( + ( + p + for p in processed_jobs + if p["run_id"] == r["run_id"] and p["job_name"] == r["job_name"] + ), + None, + ) + if p: + run_info = f"{r.get('run_status', '-')}/{r.get('run_conclusion', '-')}" + lines.append( + f"| {p['status']} | {run_info} | `{p['job_name']}` | {p['pr_info']} | `{p['runner_name'] or '-'}` | [View]({p['url']}) |" + ) + + lines.append("") + + # Failed jobs section (before All Jobs) + if failed_jobs: + lines.append(f"## Failed Jobs ({len(failed_jobs)} total)") + lines.append("") + lines.append( + "| Conclusion | Job Name | Created | Started | Queue | Duration | Runner | PR/Branch | Link |" + ) + lines.append( + "|------------|----------|---------|---------|-------|----------|--------|-----------|------|" + ) + + for r in sorted(failed_jobs, key=lambda x: x["created_at"], reverse=True): + p = next( + ( + p + for p in processed_jobs + if p["run_id"] == r["run_id"] and p["job_name"] == r["job_name"] + ), + None, + ) + if p: + lines.append( + f"| {p['conclusion']} | `{p['job_name']}` | {p['created_formatted']} | {p['started_formatted']} | {p['queue_time']} | {p['duration']} | `{p['runner_name'] or '-'}` | {p['pr_info']} | [View]({p['url']}) |" + ) + + lines.append("") + + # Detailed table (all jobs) - collapsible + lines.append("
") + lines.append( + f"All Jobs ({len(results)} total) - Click to expand" + ) + lines.append("") + lines.append( + "| Job Name | Status | Conclusion | Created | Started | Queue | Duration | Runner | PR/Branch | Link |" + ) + lines.append( + "|----------|--------|------------|---------|---------|-------|----------|--------|-----------|------|" + ) + + for p in processed_jobs: + # Mark stuck jobs in markdown with bold + if p.get("is_stuck", False): + status_display = f"**STUCK** ({p['status']})" + else: + status_display = p["status"] + + lines.append( + f"| `{p['job_name']}` | {status_display} | {p['conclusion']} | {p['created_formatted']} | {p['started_formatted']} | {p['queue_time']} | {p['duration']} | `{p['runner_name'] or '-'}` | {p['pr_info']} | [View]({p['url']}) |" + ) + + lines.append("") + lines.append("
") + lines.append("") + + return "\n".join(lines) + + +def main(): + # Check gh CLI availability before proceeding + if not check_gh_cli_available(): + sys.exit(1) + + # Capture the time when the command is run (both datetime and formatted string) + report_time = datetime.now(timezone.utc) + report_generated_time = report_time.strftime("%Y-%m-%d %H:%M:%S") + + parser = argparse.ArgumentParser(description="Query GitHub Actions job status") + parser.add_argument( + "--repo", + default="sgl-project/sglang", + help="GitHub repo (default: sgl-project/sglang)", + ) + parser.add_argument( + "--job", + required=True, + help="Job name filter (e.g., 'stage-c-test-large-8-gpu-amd-mi35x')", + ) + parser.add_argument( + "--workflow", + default="pr-test-amd.yml", + help="Workflow file name (default: pr-test-amd.yml)", + ) + parser.add_argument( + "--hours", + type=int, + default=24, + help="Time window in hours (default: 24)", + ) + parser.add_argument( + "--status", + choices=["in_progress", "queued", "completed", "waiting"], + help="Filter by job status", + ) + parser.add_argument( + "--output", + choices=["table", "csv", "json", "markdown"], + default="table", + help="Output format (default: table)", + ) + parser.add_argument( + "--summary", + action="store_true", + help="Write markdown output to GITHUB_STEP_SUMMARY", + ) + parser.add_argument( + "--output-file", + type=str, + help="Write output to file", + ) + args = parser.parse_args() + + results = query_jobs( + args.repo, + args.job, + args.workflow, + args.hours, + args.status, + ) + + output_content = None + + if args.output == "table": + print_table(results, args.repo, report_generated_time, report_time) + elif args.output == "csv": + lines = [ + "job_name,status,is_stuck,conclusion,created_at,started_at,queue_time,duration,runner,run_status,run_conclusion,pr_number,branch,url" + ] + for r in sorted(results, key=lambda x: x["created_at"], reverse=True): + queue_time = calculate_queue_time( + r["created_at"], r["started_at"], r["status"], report_time + ) + duration = calculate_duration(r["started_at"], r["completed_at"]) + is_stuck = "true" if r.get("is_stuck", False) else "false" + lines.append( + f'"{r["job_name"]}",{r["status"]},{is_stuck},{r["conclusion"]},{r["created_at"]},{r["started_at"]},{queue_time},{duration},{r["runner_name"]},{r.get("run_status", "-")},{r.get("run_conclusion", "-")},{r["pr_number"] or ""},{r["branch"]},{r["html_url"]}' + ) + output_content = "\n".join(lines) + print(output_content) + elif args.output == "json": + # Add calculated fields to JSON output for consistency + json_results = [] + for r in sorted(results, key=lambda x: x["created_at"], reverse=True): + r_copy = r.copy() + r_copy["queue_time"] = calculate_queue_time( + r["created_at"], r["started_at"], r["status"], report_time + ) + r_copy["duration"] = calculate_duration(r["started_at"], r["completed_at"]) + r_copy["created_at_formatted"] = format_time(r["created_at"]) + r_copy["started_at_formatted"] = format_time(r["started_at"]) + json_results.append(r_copy) + output_content = json.dumps(json_results, indent=2) + print(output_content) + elif args.output == "markdown": + output_content = format_markdown( + results, args.repo, args.job, args.hours, report_generated_time, report_time + ) + print(output_content) + + # Write to file if specified + if args.output_file and output_content: + with open(args.output_file, "w") as f: + f.write(output_content) + print(f"\nOutput written to {args.output_file}", file=sys.stderr) + + # Write to GITHUB_STEP_SUMMARY if requested + if args.summary: + md_content = format_markdown( + results, args.repo, args.job, args.hours, report_generated_time, report_time + ) + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file: + with open(summary_file, "a") as f: + f.write(md_content) + f.write("\n") + print(f"Summary written to GITHUB_STEP_SUMMARY", file=sys.stderr) + else: + print( + "Warning: GITHUB_STEP_SUMMARY not set, printing markdown instead:", + file=sys.stderr, + ) + print(md_content) + + +if __name__ == "__main__": + main()