diff --git a/.github/workflows/ci-coverage-overview.yml b/.github/workflows/ci-coverage-overview.yml
new file mode 100644
index 000000000..3fc5512f6
--- /dev/null
+++ b/.github/workflows/ci-coverage-overview.yml
@@ -0,0 +1,92 @@
+name: CI Coverage Overview
+
+on:
+ schedule:
+ - cron: '0 6 * * *' # Daily at 6 AM UTC
+ pull_request:
+ paths:
+ - '.github/workflows/ci-coverage-overview.yml'
+ - 'scripts/ci/ci_coverage_report.py'
+ - 'test/registered/**'
+ workflow_dispatch:
+ inputs:
+ output_format:
+ description: 'Output format'
+ required: false
+ default: 'markdown'
+ type: choice
+ options:
+ - markdown
+ - json
+
+jobs:
+ summary:
+ name: Summary
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: Generate Summary Report
+ run: |
+ python scripts/ci/ci_coverage_report.py --section summary
+
+ by-folder:
+ name: Tests by Folder
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: Generate Tests by Folder Report
+ run: |
+ python scripts/ci/ci_coverage_report.py --section by-folder
+
+ by-suite:
+ name: Tests by Suite
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: Generate Tests by Suite Report
+ run: |
+ python scripts/ci/ci_coverage_report.py --section by-suite
+
+ json-export:
+ name: JSON Export
+ runs-on: ubuntu-latest
+ if: inputs.output_format == 'json'
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.10'
+
+ - name: Generate JSON Report
+ run: |
+ python scripts/ci/ci_coverage_report.py --output-format json > ci_coverage.json
+
+ - name: Upload JSON artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: ci-coverage-report
+ path: ci_coverage.json
diff --git a/scripts/ci/ci_coverage_report.py b/scripts/ci/ci_coverage_report.py
new file mode 100755
index 000000000..2ba07d560
--- /dev/null
+++ b/scripts/ci/ci_coverage_report.py
@@ -0,0 +1,474 @@
+#!/usr/bin/env python3
+"""
+CI Coverage Report Generator
+
+Collects all CI test registrations from test/registered/ and generates
+a coverage report organized by folder, backend, and suite.
+
+Usage:
+ python scripts/ci/ci_coverage_report.py [--output-format markdown|json]
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+# Add the ci_register module path directly to avoid heavy sglang imports
+sys.path.insert(
+ 0, str(Path(__file__).parent.parent.parent / "python" / "sglang" / "test" / "ci")
+)
+
+from ci_register import CIRegistry, HWBackend, ut_parse_one_file
+
+
+def collect_all_tests(registered_dir: str) -> list[CIRegistry]:
+ """Collect all CI registrations from registered directory."""
+ files = glob.glob(f"{registered_dir}/**/*.py", recursive=True)
+ all_tests = []
+
+ for file in sorted(files):
+ try:
+ registries = ut_parse_one_file(file)
+ all_tests.extend(registries)
+ except Exception as e:
+ print(f"Warning: Failed to parse {file}: {e}", file=sys.stderr)
+
+ return all_tests
+
+
+def get_folder_name(filename: str) -> str:
+ """Extract folder name from test filename."""
+ # e.g., "registered/models/test_foo.py" -> "models"
+ parts = Path(filename).parts
+ if "registered" in parts:
+ idx = parts.index("registered")
+ if idx + 1 < len(parts) - 1: # Has subfolder
+ return parts[idx + 1]
+ return "root"
+
+
+def get_test_basename(filename: str) -> str:
+ """Extract just the test file name from the path."""
+ return Path(filename).name
+
+
+def organize_test_data(tests: list[CIRegistry]) -> dict:
+ """Organize tests into various groupings."""
+ by_backend = defaultdict(list)
+ by_folder = defaultdict(list)
+ disabled_tests = []
+
+ for t in tests:
+ by_backend[t.backend.name].append(t)
+ by_folder[get_folder_name(t.filename)].append(t)
+ if t.disabled:
+ disabled_tests.append(t)
+
+ # Count unique test files (a file may be registered for multiple backends)
+ unique_files = set(t.filename for t in tests)
+ unique_enabled_files = set(t.filename for t in tests if not t.disabled)
+ unique_disabled_files = set(t.filename for t in tests if t.disabled)
+
+ return {
+ "total": len(tests),
+ "total_unique_files": len(unique_files),
+ "enabled": len(tests) - len(disabled_tests),
+ "enabled_unique_files": len(unique_enabled_files),
+ "disabled_count": len(disabled_tests),
+ "disabled_unique_files": len(unique_disabled_files),
+ "by_backend": by_backend,
+ "by_folder": by_folder,
+ "disabled_tests": disabled_tests,
+ }
+
+
+def generate_summary_section(data: dict) -> str:
+ """Generate the summary/overview section."""
+ lines = []
+ lines.append("# CI Coverage Overview\n")
+ lines.append(
+ f"**Unique Test Files:** {data['total_unique_files']} ({data['enabled_unique_files']} enabled, {data['disabled_unique_files']} disabled)\n"
+ )
+ lines.append(
+ f"**Total Registrations:** {data['total']} ({data['enabled']} enabled, {data['disabled_count']} disabled)\n"
+ )
+ lines.append(
+ "*Note: A test file may be registered for multiple backends (e.g., CUDA + AMD), so total registrations > unique files.*\n"
+ )
+
+ by_backend = data["by_backend"]
+ by_folder = data["by_folder"]
+ disabled_tests = data["disabled_tests"]
+
+ # Backend summary (collapsible)
+ lines.append("")
+ lines.append("Backend Summary
\n")
+ lines.append("| Backend | Total | Enabled | Disabled | Per-Commit | Nightly |")
+ lines.append("|---------|-------|---------|----------|------------|---------|")
+
+ for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+ backend_tests = by_backend.get(backend, [])
+ if not backend_tests:
+ continue
+ b_total = len(backend_tests)
+ b_disabled = sum(1 for t in backend_tests if t.disabled)
+ b_enabled = b_total - b_disabled
+ b_per_commit = sum(1 for t in backend_tests if not t.nightly and not t.disabled)
+ b_nightly = sum(1 for t in backend_tests if t.nightly and not t.disabled)
+ lines.append(
+ f"| {backend} | {b_total} | {b_enabled} | {b_disabled} | {b_per_commit} | {b_nightly} |"
+ )
+
+ lines.append("\n \n")
+
+ # Folder summary (collapsible)
+ lines.append("")
+ lines.append("Folder Summary
\n")
+ lines.append("| Folder | CUDA | AMD | NPU | CPU | Total |")
+ lines.append("|--------|------|-----|-----|-----|-------|")
+
+ for folder in sorted(by_folder.keys()):
+ folder_tests = by_folder[folder]
+ cuda = sum(1 for t in folder_tests if t.backend == HWBackend.CUDA)
+ amd = sum(1 for t in folder_tests if t.backend == HWBackend.AMD)
+ npu = sum(1 for t in folder_tests if t.backend == HWBackend.NPU)
+ cpu = sum(1 for t in folder_tests if t.backend == HWBackend.CPU)
+ lines.append(
+ f"| {folder} | {cuda} | {amd} | {npu} | {cpu} | {len(folder_tests)} |"
+ )
+
+ lines.append("\n \n")
+
+ # Disabled tests section (collapsible)
+ if disabled_tests:
+ lines.append("")
+ lines.append("Disabled Tests
\n")
+ lines.append("| File | Backend | Suite | Reason |")
+ lines.append("|------|---------|-------|--------|")
+ for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+ test_name = get_test_basename(t.filename)
+ reason = t.disabled[:50] + "..." if len(t.disabled) > 50 else t.disabled
+ lines.append(f"| `{test_name}` | {t.backend.name} | {t.suite} | {reason} |")
+ lines.append("\n \n")
+
+ return "\n".join(lines)
+
+
+def generate_by_folder_section(data: dict) -> str:
+ """Generate the 'All Tests by Folder' section."""
+ lines = []
+ by_folder = data["by_folder"]
+
+ lines.append("# All Tests by Folder\n")
+
+ for folder in sorted(by_folder.keys()):
+ folder_tests = by_folder[folder]
+ lines.append("")
+ lines.append(
+ f"{folder}/ ({len(folder_tests)} tests)
\n"
+ )
+
+ # Group by backend within folder
+ folder_by_backend = defaultdict(list)
+ for t in folder_tests:
+ folder_by_backend[t.backend.name].append(t)
+
+ for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+ backend_tests = folder_by_backend.get(backend, [])
+ if not backend_tests:
+ continue
+
+ lines.append(f"### {backend} ({len(backend_tests)} tests)\n")
+ lines.append("| Test File | Suite | Est. Time | Status |")
+ lines.append("|-----------|-------|-----------|--------|")
+
+ for t in sorted(backend_tests, key=lambda x: x.filename):
+ test_name = get_test_basename(t.filename)
+ status = (
+ "Disabled"
+ if t.disabled
+ else ("Nightly" if t.nightly else "Per-Commit")
+ )
+ lines.append(
+ f"| `{test_name}` | {t.suite} | {t.est_time:.0f}s | {status} |"
+ )
+
+ lines.append("")
+
+ lines.append(" \n")
+
+ return "\n".join(lines)
+
+
+def generate_by_suite_section(data: dict) -> str:
+ """Generate the 'All Tests by Test Suite' section."""
+ lines = []
+ by_backend = data["by_backend"]
+
+ lines.append("# All Tests by Test Suite\n")
+
+ for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+ backend_tests = by_backend.get(backend, [])
+ if not backend_tests:
+ continue
+
+ b_total = len(backend_tests)
+ b_disabled = sum(1 for t in backend_tests if t.disabled)
+ b_enabled = b_total - b_disabled
+
+ lines.append("")
+ lines.append(
+ f"{backend} Backend ({b_enabled} enabled, {b_disabled} disabled)
\n"
+ )
+
+ # Group by suite within backend
+ backend_suites = defaultdict(list)
+ for t in backend_tests:
+ backend_suites[t.suite].append(t)
+
+ for suite in sorted(backend_suites.keys()):
+ suite_tests = backend_suites[suite]
+ s_enabled = sum(1 for t in suite_tests if not t.disabled)
+ s_disabled = sum(1 for t in suite_tests if t.disabled)
+ s_est_time = sum(t.est_time for t in suite_tests if not t.disabled)
+ is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+ suite_type = "Nightly" if is_nightly else "Per-Commit"
+ lines.append("")
+ lines.append(
+ f"{suite} ({s_enabled} enabled, {s_disabled} disabled) - {suite_type}
\n"
+ )
+ lines.append(f"*Estimated total time: {s_est_time:.0f}s*\n")
+
+ lines.append("| Test File | Folder | Est. Time | Status |")
+ lines.append("|-----------|--------|-----------|--------|")
+
+ for t in sorted(suite_tests, key=lambda x: x.filename):
+ test_name = get_test_basename(t.filename)
+ folder = get_folder_name(t.filename)
+ if t.disabled:
+ status = (
+ f"Disabled: {t.disabled[:30]}..."
+ if len(t.disabled) > 30
+ else f"Disabled: {t.disabled}"
+ )
+ else:
+ status = "Nightly" if t.nightly else "Per-Commit"
+ lines.append(
+ f"| `{test_name}` | {folder} | {t.est_time:.0f}s | {status} |"
+ )
+
+ lines.append("\n \n")
+
+ lines.append(" \n")
+
+ return "\n".join(lines)
+
+
+def generate_markdown_report(tests: list[CIRegistry], section: str = "all") -> str:
+ """Generate markdown report for GitHub step summary."""
+ data = organize_test_data(tests)
+
+ if section == "summary":
+ return generate_summary_section(data)
+ elif section == "by-folder":
+ return generate_by_folder_section(data)
+ elif section == "by-suite":
+ return generate_by_suite_section(data)
+ else: # "all"
+ parts = [
+ generate_summary_section(data),
+ "---",
+ generate_by_folder_section(data),
+ "---",
+ generate_by_suite_section(data),
+ ]
+ return "\n".join(parts)
+
+
+def generate_json_report(tests: list[CIRegistry]) -> str:
+ """Generate JSON report with detailed test listings."""
+ by_backend = defaultdict(list)
+ by_folder = defaultdict(list)
+
+ for t in tests:
+ by_backend[t.backend.name].append(t)
+ by_folder[get_folder_name(t.filename)].append(t)
+
+ disabled_tests = [t for t in tests if t.disabled]
+
+ # Build structured data
+ data = {
+ "summary": {
+ "total": len(tests),
+ "enabled": len(tests) - len(disabled_tests),
+ "disabled": len(disabled_tests),
+ },
+ "tests_by_folder": {},
+ "tests_by_suite": {},
+ "backend_summary": {},
+ "folder_summary": {},
+ "disabled_tests": [],
+ }
+
+ # Section 1: Tests by Folder
+ for folder in sorted(by_folder.keys()):
+ folder_tests = by_folder[folder]
+ folder_by_backend = defaultdict(list)
+ for t in folder_tests:
+ folder_by_backend[t.backend.name].append(t)
+
+ data["tests_by_folder"][folder] = {
+ "total": len(folder_tests),
+ "backends": {},
+ }
+
+ for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+ backend_tests = folder_by_backend.get(backend, [])
+ if backend_tests:
+ data["tests_by_folder"][folder]["backends"][backend] = [
+ {
+ "filename": get_test_basename(t.filename),
+ "suite": t.suite,
+ "est_time": t.est_time,
+ "status": (
+ "disabled"
+ if t.disabled
+ else ("nightly" if t.nightly else "per-commit")
+ ),
+ }
+ for t in sorted(backend_tests, key=lambda x: x.filename)
+ ]
+
+ # Section 2: Tests by Suite (Backend -> Suite)
+ for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+ backend_tests = by_backend.get(backend, [])
+ if not backend_tests:
+ continue
+
+ backend_suites = defaultdict(list)
+ for t in backend_tests:
+ backend_suites[t.suite].append(t)
+
+ data["tests_by_suite"][backend] = {
+ "total": len(backend_tests),
+ "enabled": sum(1 for t in backend_tests if not t.disabled),
+ "disabled": sum(1 for t in backend_tests if t.disabled),
+ "suites": {},
+ }
+
+ for suite in sorted(backend_suites.keys()):
+ suite_tests = backend_suites[suite]
+ is_nightly = any(t.nightly for t in suite_tests if not t.disabled)
+
+ data["tests_by_suite"][backend]["suites"][suite] = {
+ "total": len(suite_tests),
+ "enabled": sum(1 for t in suite_tests if not t.disabled),
+ "disabled": sum(1 for t in suite_tests if t.disabled),
+ "est_time": sum(t.est_time for t in suite_tests if not t.disabled),
+ "type": "nightly" if is_nightly else "per-commit",
+ "tests": [
+ {
+ "filename": get_test_basename(t.filename),
+ "folder": get_folder_name(t.filename),
+ "est_time": t.est_time,
+ "status": (
+ "disabled"
+ if t.disabled
+ else ("nightly" if t.nightly else "per-commit")
+ ),
+ "disabled_reason": t.disabled if t.disabled else None,
+ }
+ for t in sorted(suite_tests, key=lambda x: x.filename)
+ ],
+ }
+
+ # Backend summary
+ for backend in ["CUDA", "AMD", "NPU", "CPU"]:
+ backend_tests = by_backend.get(backend, [])
+ if backend_tests:
+ data["backend_summary"][backend] = {
+ "total": len(backend_tests),
+ "enabled": sum(1 for t in backend_tests if not t.disabled),
+ "disabled": sum(1 for t in backend_tests if t.disabled),
+ "per_commit": sum(
+ 1 for t in backend_tests if not t.nightly and not t.disabled
+ ),
+ "nightly": sum(
+ 1 for t in backend_tests if t.nightly and not t.disabled
+ ),
+ }
+
+ # Folder summary
+ for folder in sorted(by_folder.keys()):
+ folder_tests = by_folder[folder]
+ data["folder_summary"][folder] = {
+ "CUDA": sum(1 for t in folder_tests if t.backend == HWBackend.CUDA),
+ "AMD": sum(1 for t in folder_tests if t.backend == HWBackend.AMD),
+ "NPU": sum(1 for t in folder_tests if t.backend == HWBackend.NPU),
+ "CPU": sum(1 for t in folder_tests if t.backend == HWBackend.CPU),
+ "total": len(folder_tests),
+ }
+
+ # Disabled tests
+ for t in sorted(disabled_tests, key=lambda x: (x.backend.name, x.filename)):
+ data["disabled_tests"].append(
+ {
+ "filename": get_test_basename(t.filename),
+ "backend": t.backend.name,
+ "suite": t.suite,
+ "reason": t.disabled,
+ }
+ )
+
+ return json.dumps(data, indent=2)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Generate CI coverage report")
+ parser.add_argument(
+ "--output-format",
+ choices=["markdown", "json"],
+ default="markdown",
+ help="Output format (default: markdown)",
+ )
+ parser.add_argument(
+ "--section",
+ choices=["all", "summary", "by-folder", "by-suite"],
+ default="all",
+ help="Which section to output (default: all). Only applies to markdown format.",
+ )
+ parser.add_argument(
+ "--registered-dir",
+ default="test/registered",
+ help="Path to registered test directory",
+ )
+ args = parser.parse_args()
+
+ # Change to repo root if needed
+ script_dir = Path(__file__).parent
+ repo_root = script_dir.parent.parent
+ os.chdir(repo_root)
+
+ tests = collect_all_tests(args.registered_dir)
+
+ if args.output_format == "markdown":
+ report = generate_markdown_report(tests, section=args.section)
+ else:
+ report = generate_json_report(tests)
+
+ print(report)
+
+ # Write to GITHUB_STEP_SUMMARY if available
+ summary_file = os.environ.get("GITHUB_STEP_SUMMARY")
+ if summary_file and args.output_format == "markdown":
+ with open(summary_file, "a") as f:
+ f.write(report)
+
+
+if __name__ == "__main__":
+ main()