From f2ccc44240a72a683efa636423c198fb483b71c7 Mon Sep 17 00:00:00 2001 From: Douglas Yang Date: Thu, 25 Dec 2025 13:13:27 -0800 Subject: [PATCH] fix: improving format and design (#15791) --- scripts/ci_monitor/ci_failures_analysis.py | 61 ++++++---- .../ci_monitor/post_ci_failures_to_slack.py | 108 ++++++++++++------ 2 files changed, 116 insertions(+), 53 deletions(-) diff --git a/scripts/ci_monitor/ci_failures_analysis.py b/scripts/ci_monitor/ci_failures_analysis.py index 5e52b4b83..de8300a7b 100644 --- a/scripts/ci_monitor/ci_failures_analysis.py +++ b/scripts/ci_monitor/ci_failures_analysis.py @@ -699,10 +699,8 @@ class SGLangFailuresAnalyzer: # Build final results job_streak_data = {} for job_name in job_current_streak.keys(): - # Get last 5 runs (most recent first) - recent_runs = job_recent_runs.get(job_name, [])[-5:][ - ::-1 - ] # Last 5, reversed + # Get last 5 runs (oldest to latest, chronological order) + recent_runs = job_recent_runs.get(job_name, [])[-5:] job_streak_data[job_name] = { "current_streak": job_current_streak[job_name], @@ -832,8 +830,10 @@ class SGLangFailuresAnalyzer: if broken: print(f"## {title} ({len(broken)} jobs with active streaks)") print("=" * 130) + # Add sub-header to clarify this shows ongoing failure streaks + print("\nšŸ”„ Consecutive failures (>=2) & currently failing") print( - f"\n{'Job Name':<40} {'Current':<8} {'Max':<6} {'Runs':<6} {'First':<13} {'Last':<13} {'Recent History':<30}" + f"\n{'Job Name':<40} {'Current':<8} {'Max':<6} {'Runs':<6} {'First':<13} {'Last':<13} {'Recent Runs (oldest ā†’ latest)':<30}" ) print("-" * 130) for job_name, d in broken[:15]: @@ -856,7 +856,7 @@ class SGLangFailuresAnalyzer: # Recent history (last 5 runs as emoji) recent_runs = d.get("recent_runs", []) history_str = ( - " ".join([r["status"] for r in recent_runs]) + "ā€¦ " + " ".join([r["status"] for r in recent_runs]) if recent_runs else "N/A" ) @@ -864,11 +864,11 @@ class SGLangFailuresAnalyzer: # Color red if color_failures is True (for critical sections) if color_failures: print( - f"\033[91m{display_name:<40}\033[0m {d['current_streak']:<8} {d['max_streak']:<6} {d['total_runs']:<6} {first_str:<13} {last_str:<13} {history_str:<30}" + f"\033[91m{display_name:<40}\033[0m {d['current_streak']:<8} {d['max_streak']:<6} {d['total_runs']:<6} {first_str:<13} {last_str:<13} {history_str:<32}" ) else: print( - f"{display_name:<40} {d['current_streak']:<8} {d['max_streak']:<6} {d['total_runs']:<6} {first_str:<13} {last_str:<13} {history_str:<30}" + f"{display_name:<40} {d['current_streak']:<8} {d['max_streak']:<6} {d['total_runs']:<6} {first_str:<13} {last_str:<13} {history_str:<32}" ) else: print(f"## {title}") @@ -877,11 +877,17 @@ class SGLangFailuresAnalyzer: # Show recently failed jobs in a collapsed section (terminal doesn't support collapse, so just show as separate section) if recently_failed: + # Extract just the workflow name without the run count for cleaner display + short_title = title.split("(")[0].strip() + if short_title and short_title[0].isdigit(): + short_title = short_title.split(".", 1)[-1].strip() + # Get the max total_runs from recently_failed jobs to show the analysis window + max_total_runs = max(d["total_runs"] for _, d in recently_failed) print( - f"\n Recently failed jobs (no active streak): {len(recently_failed)} jobs" + f"\n šŸ“‹ [{short_title}] No current failure streak, but had failures in the past {max_total_runs} runs - {len(recently_failed)} jobs" ) print( - f" {'Job Name':<38} {'Failures':<12} {'Fail Rate':<12} {'Total Runs':<12} {'Recent History (last 5)':<30}" + f" {'Job Name':<38} {'Failures':<12} {'Fail Rate':<12} {'Total Runs':<12} {'Recent Runs (oldest ā†’ latest)':<30}" ) print(" " + "-" * 120) for job_name, d in recently_failed[:10]: @@ -890,12 +896,12 @@ class SGLangFailuresAnalyzer: ) recent_runs = d.get("recent_runs", []) history_str = ( - " ".join([r["status"] for r in recent_runs]) + "ā€¦ " + " ".join([r["status"] for r in recent_runs]) if recent_runs else "N/A" ) print( - f" {display_name:<38} {d['total_failures']:<12} {d['failure_rate']:.1f}%{'':<7} {d['total_runs']:<12} {history_str:<30}" + f" {display_name:<38} {d['total_failures']:<12} {d['failure_rate']:.1f}%{'':<7} {d['total_runs']:<12} {history_str:<32}" ) # ========== SCHEDULED/MAIN BRANCH RUNS (9 sections) ========== @@ -1185,7 +1191,9 @@ class SGLangFailuresAnalyzer: summary_lines.append( f"**Analysis Timestamp:** {report_data['summary']['analysis_timestamp']}" ) - summary_lines.append("_Note: Recent runs are shown left to right_") + summary_lines.append( + "_Note: Recent runs are shown oldest ā†’ latest (left to right)_" + ) summary_lines.append("") # Summary stats - COLLAPSIBLE @@ -1267,11 +1275,16 @@ class SGLangFailuresAnalyzer: summary_lines.append("") if broken: + # Add sub-header to clarify this shows ongoing failure streaks summary_lines.append( - "| Job Name | Current | Max | Runs | First | Last | Recent History |" + "šŸ”„ **Consecutive failures (ā‰„2) & currently failing**" + ) + summary_lines.append("") + summary_lines.append( + "| Job Name | Current | Max | Runs | First | Last | Recent Runs (oldest ā†’ latest) |" ) summary_lines.append( - "|----------|---------|-----|------|-------|------|----------------|" + "|----------|---------|-----|------|-------|------|-------------------------------|" ) for job_name, d in broken[:15]: display_name = ( @@ -1295,7 +1308,7 @@ class SGLangFailuresAnalyzer: # Recent history (last 5 runs as clickable emoji) recent_runs = d.get("recent_runs", []) if recent_runs: - history_links = " ".join( + history_links = "ā€¦ " + " ".join( [ f"[{r['status']}]({r['job_url']})" for r in recent_runs @@ -1324,16 +1337,24 @@ class SGLangFailuresAnalyzer: # Show recently failed jobs in a collapsible section if recently_failed: + # Extract just the workflow name without the run count for cleaner display + # e.g., "1. PR Test NVIDIA - Scheduled (latest 12 runs)" -> "PR Test NVIDIA - Scheduled" + short_title = title.split("(")[0].strip() + # Remove the leading number and period if present + if short_title and short_title[0].isdigit(): + short_title = short_title.split(".", 1)[-1].strip() + # Get the max total_runs from recently_failed jobs to show the analysis window + max_total_runs = max(d["total_runs"] for _, d in recently_failed) summary_lines.append("
") summary_lines.append( - f"Recently failed jobs (no active streak) - {len(recently_failed)} jobs" + f"šŸ“‹ [{short_title}] No current failure streak, but had failures in the past {max_total_runs} runs - {len(recently_failed)} jobs" ) summary_lines.append("") summary_lines.append( - "| Job Name | Failures | Fail Rate | Total Runs | Recent History (last 5) |" + "| Job Name | Failures | Fail Rate | Total Runs | Recent Runs (oldest ā†’ latest) |" ) summary_lines.append( - "|----------|----------|-----------|------------|-------------------------|" + "|----------|----------|-----------|------------|-------------------------------|" ) for job_name, d in recently_failed[:15]: display_name = ( @@ -1341,7 +1362,7 @@ class SGLangFailuresAnalyzer: ) recent_runs = d.get("recent_runs", []) if recent_runs: - history_links = " ".join( + history_links = "ā€¦ " + " ".join( [ f"[{r['status']}]({r['job_url']})" for r in recent_runs diff --git a/scripts/ci_monitor/post_ci_failures_to_slack.py b/scripts/ci_monitor/post_ci_failures_to_slack.py index 6f6629010..34f1bc294 100755 --- a/scripts/ci_monitor/post_ci_failures_to_slack.py +++ b/scripts/ci_monitor/post_ci_failures_to_slack.py @@ -55,21 +55,29 @@ def post_ci_failures_to_slack(report_file: str) -> bool: critical_failures = [] - # Map workflow data keys to display names - workflow_name_map = { - # PR Tests - Scheduled (5 workflows) - "pr_test_nvidia_scheduled_data": "PR Test (Nvidia, scheduled)", - "pr_test_amd_scheduled_data": "PR Test (AMD, scheduled)", - "pr_test_xeon_scheduled_data": "PR Test (Xeon, scheduled)", - "pr_test_xpu_scheduled_data": "PR Test (XPU, scheduled)", - "pr_test_npu_scheduled_data": "PR Test (NPU, scheduled)", - # Nightly Tests - Scheduled (4 workflows) - "nightly_nvidia_scheduled_data": "Nightly Test (Nvidia, scheduled)", - "nightly_amd_scheduled_data": "Nightly Test (AMD, scheduled)", - "nightly_intel_scheduled_data": "Nightly Test (Intel, scheduled)", - "nightly_npu_scheduled_data": "Nightly Test (NPU, scheduled)", + # Map workflow data keys to display names and hardware category + # Format: (display_name, hardware, test_type_order) + # test_type_order: 0 = PR Test, 1 = Nightly (so PR Test comes first) + workflow_info_map = { + # Nvidia + "pr_test_nvidia_scheduled_data": ("PR Test", "Nvidia", 0), + "nightly_nvidia_scheduled_data": ("Nightly", "Nvidia", 1), + # AMD + "pr_test_amd_scheduled_data": ("PR Test", "AMD", 0), + "nightly_amd_scheduled_data": ("Nightly", "AMD", 1), + # Intel/Xeon + "pr_test_xeon_scheduled_data": ("PR Test", "Intel", 0), + "nightly_intel_scheduled_data": ("Nightly", "Intel", 1), + # XPU + "pr_test_xpu_scheduled_data": ("PR Test", "XPU", 0), + # NPU + "pr_test_npu_scheduled_data": ("PR Test", "NPU", 0), + "nightly_npu_scheduled_data": ("Nightly", "NPU", 1), } + # Hardware priority order (Nvidia first) + hardware_order = ["Nvidia", "AMD", "Intel", "XPU", "NPU"] + # Iterate through each workflow section for workflow_key, workflow_data in report_data.items(): # Skip non-workflow keys (summary, limits, etc.) @@ -79,13 +87,12 @@ def post_ci_failures_to_slack(report_file: str) -> bool: ): continue - # Get workflow display name - workflow_name = workflow_name_map.get(workflow_key, workflow_key) - - # Only process scheduled workflows - if "scheduled" not in workflow_key.lower(): + # Only process scheduled workflows that are in our map + if workflow_key not in workflow_info_map: continue + test_type, hardware, test_order = workflow_info_map[workflow_key] + # Check each job in this workflow for job_name, job_data in workflow_data.items(): if not isinstance(job_data, dict): @@ -100,7 +107,9 @@ def post_ci_failures_to_slack(report_file: str) -> bool: critical_failures.append( { - "workflow_name": workflow_name, + "hardware": hardware, + "test_type": test_type, + "test_order": test_order, "job_name": job_name, "consecutive_failures": current_streak, "first_failed_at": ( @@ -124,14 +133,18 @@ def post_ci_failures_to_slack(report_file: str) -> bool: } ) - # Group by workflow - workflow_jobs = {} + # Group by hardware, then by test type + # Structure: {hardware: {test_type: [job_names]}} + hardware_jobs = {} for job in critical_failures: - workflow = job.get("workflow_name", "Unknown") + hardware = job.get("hardware", "Unknown") + test_type = job.get("test_type", "Unknown") job_name = job.get("job_name", "unknown") - if workflow not in workflow_jobs: - workflow_jobs[workflow] = [] - workflow_jobs[workflow].append(job_name) + if hardware not in hardware_jobs: + hardware_jobs[hardware] = {} + if test_type not in hardware_jobs[hardware]: + hardware_jobs[hardware][test_type] = [] + hardware_jobs[hardware][test_type].append(job_name) # Create summary message workflow_url = "" @@ -140,7 +153,7 @@ def post_ci_failures_to_slack(report_file: str) -> bool: f"https://github.com/sgl-project/sglang/actions/runs/{run_id}" ) - if not workflow_jobs: + if not hardware_jobs: summary = "āœ… No critical failures detected in scheduled runs" if workflow_url: summary += f"\n<{workflow_url}|View CI Monitor Run>" @@ -149,9 +162,20 @@ def post_ci_failures_to_slack(report_file: str) -> bool: # Ping relevant people when there are failures mentions = "<@U09RR5TNC94> <@U09ABMCKQPM>" summary_lines = [f"{mentions} šŸšØ *CI Critical Failures (Scheduled Runs)*"] - for workflow, jobs in sorted(workflow_jobs.items()): - job_list = ", ".join(jobs) - summary_lines.append(f"ā€¢ *{workflow}*: {job_list}") + + # Iterate in hardware priority order, with PR Test before Nightly + test_type_order = ["PR Test", "Nightly"] + for hardware in hardware_order: + if hardware not in hardware_jobs: + continue + summary_lines.append(f"\n*{hardware}:*") + for test_type in test_type_order: + if test_type not in hardware_jobs[hardware]: + continue + jobs = hardware_jobs[hardware][test_type] + job_list = ", ".join(jobs) + summary_lines.append(f" ā€¢ {test_type}: {job_list}") + if workflow_url: summary_lines.append(f"\n<{workflow_url}|View Full CI Monitor Report>") summary = "\n".join(summary_lines) @@ -174,11 +198,24 @@ def post_ci_failures_to_slack(report_file: str) -> bool: thread_ts = response["ts"] # If there are failures, post detailed breakdown in thread - if workflow_jobs: + if hardware_jobs: details_lines = ["*Detailed Failure Breakdown*\n"] - for job in critical_failures: - workflow = job.get("workflow_name", "Unknown") + # Sort critical_failures by hardware order, then test_order + hardware_order_map = {hw: i for i, hw in enumerate(hardware_order)} + sorted_failures = sorted( + critical_failures, + key=lambda x: ( + hardware_order_map.get(x.get("hardware", ""), 99), + x.get("test_order", 99), + x.get("job_name", ""), + ), + ) + + current_hardware = None + for job in sorted_failures: + hardware = job.get("hardware", "Unknown") + test_type = job.get("test_type", "Unknown") job_name = job.get("job_name", "unknown") consecutive = job.get("consecutive_failures", 0) first_url = job.get("first_failed_url", "") @@ -186,8 +223,13 @@ def post_ci_failures_to_slack(report_file: str) -> bool: last_url = job.get("last_failed_url", "") last_at = job.get("last_failed_at", "unknown") + # Add hardware section header + if hardware != current_hardware: + details_lines.append(f"\n*ā”ā”ā” {hardware} ā”ā”ā”*") + current_hardware = hardware + details_lines.append( - f"ā€¢ *{workflow}* ā†’ `{job_name}`\n" + f"ā€¢ *{test_type}* ā†’ `{job_name}`\n" f" Consecutive failures: {consecutive}\n" f" First failed: <{first_url}|{first_at}>\n" f" Last failed: <{last_url}|{last_at}>\n"