Enhance error resilience in dump comparator (#19685)

2026-03-02 19:08:35 +08:00
parent e5ef845cad
commit 7579ab3f33
18 changed files with 495 additions and 303 deletions
--- a/python/sglang/srt/debug_utils/comparator/init.py
+++ b/python/sglang/srt/debug_utils/comparator/init.py
@@ -4,6 +4,6 @@ from sglang.srt.debug_utils.comparator.aligner.entrypoint.traced_types import (
 from sglang.srt.debug_utils.comparator.aligner.entrypoint.types import (  # noqa: F401
    AlignerPlan,
 )
-from sglang.srt.debug_utils.comparator.output_types import TensorComparisonRecord
+from sglang.srt.debug_utils.comparator.output_types import ComparisonTensorRecord

-TensorComparisonRecord.model_rebuild()
+ComparisonTensorRecord.model_rebuild()
--- a/python/sglang/srt/debug_utils/comparator/bundle_comparator.py
+++ b/python/sglang/srt/debug_utils/comparator/bundle_comparator.py
@@ -31,10 +31,10 @@ from sglang.srt.debug_utils.comparator.meta_overrider import MetaOverrider
 from sglang.srt.debug_utils.comparator.output_types import (
    BundleFileInfo,
    BundleSideInfo,
+    ComparisonNonTensorRecord,
+    ComparisonSkipRecord,
+    ComparisonTensorRecord,
    ErrorLog,
-    NonTensorComparisonRecord,
-    SkipComparisonRecord,
-    TensorComparisonRecord,
    _split_logs,
 )
 from sglang.srt.debug_utils.comparator.tensor_comparator.comparator import (
@@ -51,7 +51,7 @@ def _collect_bundle_side_info(
    metas: list[dict[str, Any]],
 ) -> BundleSideInfo:
    from sglang.srt.debug_utils.comparator.display import (
-        PARALLEL_INFO_KEYS,
+        _PARALLEL_INFO_KEYS,
        extract_parallel_info,
    )

@@ -61,7 +61,7 @@ def _collect_bundle_side_info(
        tensor: torch.Tensor = item.value

        parallel_info: dict[str, str] = {}
-        for key in PARALLEL_INFO_KEYS:
+        for key in _PARALLEL_INFO_KEYS:
            extract_parallel_info(row_data=parallel_info, info=meta.get(key, {}))

        files.append(
@@ -91,7 +91,7 @@ def compare_bundle_pair(
    viz_output_dir: Optional[Path] = None,
    compute_per_token: bool = False,
    meta_overrider: Optional[MetaOverrider] = None,
-) -> Union[TensorComparisonRecord, SkipComparisonRecord, NonTensorComparisonRecord]:
+) -> Union[ComparisonTensorRecord, ComparisonSkipRecord, ComparisonNonTensorRecord]:
    with log_sink.context() as collected_logs:
        result = _compare_bundle_pair_inner(
            name=name,
@@ -124,7 +124,7 @@ def _compare_bundle_pair_inner(
    viz_output_dir: Optional[Path] = None,
    compute_per_token: bool = False,
    meta_overrider: Optional[MetaOverrider] = None,
-) -> Union[TensorComparisonRecord, SkipComparisonRecord, NonTensorComparisonRecord]:
+) -> Union[ComparisonTensorRecord, ComparisonSkipRecord, ComparisonNonTensorRecord]:
    # 1. Load all successfully loaded values
    all_pair: Pair[list[ValueWithMeta]] = Pair(
        x=_load_all_values(filenames=filenames_pair.x, base_path=dir_pair.x),
@@ -133,7 +133,7 @@ def _compare_bundle_pair_inner(

    if not all_pair.x or not all_pair.y:
        reason = "baseline_load_failed" if not all_pair.x else "target_load_failed"
-        return SkipComparisonRecord(name=name, reason=reason)
+        return ComparisonSkipRecord(name=name, reason=reason)

    # 1b. Dims override: patch meta["dims"] before DP filter reads it
    # (--override-dims may add ``# dp:=moe_dp``, so it must run first)
@@ -203,10 +203,10 @@ def _compare_bundle_pair_tensor_type(
    ),
    viz_output_dir: Optional[Path] = None,
    compute_per_token: bool = False,
-) -> Union[TensorComparisonRecord, SkipComparisonRecord]:
+) -> Union[ComparisonTensorRecord, ComparisonSkipRecord]:
    if not valid_pair.x or not valid_pair.y:
        reason = "baseline_load_failed" if not valid_pair.x else "target_load_failed"
-        return SkipComparisonRecord(name=name, reason=reason)
+        return ComparisonSkipRecord(name=name, reason=reason)

    # Plan (meta only, no tensor)
    metas_pair: Pair[list[dict[str, Any]]] = valid_pair.map(
@@ -245,7 +245,7 @@ def _compare_bundle_pair_tensor_type(
        assert aligner_result.failed_side_xy is not None
        side_name: str = _FAILED_SIDE_MAP[aligner_result.failed_side_xy]
        reason: str = f"{side_name}_load_failed"
-        return SkipComparisonRecord(name=name, reason=reason)
+        return ComparisonSkipRecord(name=name, reason=reason)

    # Resolve seq_dim for per-token computation
    seq_dim: Optional[int] = (
@@ -263,7 +263,7 @@ def _compare_bundle_pair_tensor_type(
        diff_threshold=diff_threshold,
        seq_dim=seq_dim,
    )
-    record = TensorComparisonRecord(
+    record = ComparisonTensorRecord(
        **info.model_dump(),
        traced_plan=aligner_result.traced_plan,
        replicated_checks=replicated_checks,
@@ -331,7 +331,7 @@ def _compare_bundle_pair_non_tensor_type(
    *,
    name: str,
    value_pair: Pair[list[ValueWithMeta]],
-) -> NonTensorComparisonRecord:
+) -> ComparisonNonTensorRecord:
    baseline_value: Any = value_pair.x[0].value
    target_value: Any = value_pair.y[0].value

@@ -340,7 +340,7 @@ def _compare_bundle_pair_non_tensor_type(
    except Exception:
        values_equal = False

-    return NonTensorComparisonRecord(
+    return ComparisonNonTensorRecord(
        name=name,
        baseline_value=repr(baseline_value),
        target_value=repr(target_value),
--- a/python/sglang/srt/debug_utils/comparator/display.py
+++ b/python/sglang/srt/debug_utils/comparator/display.py
@@ -3,13 +3,9 @@ from __future__ import annotations
 from collections import defaultdict
 from io import StringIO
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import Any, Optional

 import polars as pl
-import rich.table
-
-if TYPE_CHECKING:
-    from rich.table import Table

 from sglang.srt.debug_utils.comparator.output_types import (
    InputIdsRecord,
@@ -43,45 +39,19 @@ def emit_display_records(

 def _render_polars_as_text(df: pl.DataFrame, *, title: Optional[str] = None) -> str:
    from rich.console import Console
+    from rich.table import Table

-    table = _build_rich_table(df, title=title)
+    table = Table(title=title)
+    for col in df.columns:
+        table.add_column(col)
+    for row in df.iter_rows():
+        table.add_row(*[str(v) for v in row])

    buf = StringIO()
    Console(file=buf, force_terminal=False, width=200).print(table)
    return buf.getvalue().rstrip("\n")


-def _render_polars_as_rich_table(
-    df: pl.DataFrame, *, title: Optional[str] = None
-) -> "Table":
-    return _build_rich_table(df, title=title)
-
-
-def _build_rich_table(df: pl.DataFrame, *, title: Optional[str] = None) -> "Table":
-    from rich.table import Table
-
-    table = Table(title=title)
-    for col in df.columns:
-        table.add_column(col)
-    for row in df.iter_rows():
-        table.add_row(*[str(v) for v in row])
-
-    return table
-
-
-def _render_polars_as_rich_table(
-    df: pl.DataFrame, *, title: Optional[str] = None
-) -> "rich.table.Table":
-    from rich.table import Table
-
-    table = Table(title=title)
-    for col in df.columns:
-        table.add_column(col)
-    for row in df.iter_rows():
-        table.add_row(*[str(v) for v in row])
-    return table
-
-
 def _collect_rank_info(
    df: pl.DataFrame, dump_dir: Path
 ) -> Optional[list[dict[str, Any]]]:
@@ -99,7 +69,7 @@ def _collect_rank_info(

        row_data: dict[str, Any] = {"rank": row["rank"]}
        for key in PARALLEL_INFO_KEYS:
-            extract_parallel_info(row_data=row_data, info=meta.get(key, {}))
+            _extract_parallel_info(row_data=row_data, info=meta.get(key, {}))
        table_rows.append(row_data)

    return table_rows or None
@@ -149,7 +119,7 @@ def _collect_input_ids_and_positions(
    return table_rows or None


-def extract_parallel_info(row_data: dict[str, Any], info: dict[str, Any]) -> None:
+def _extract_parallel_info(row_data: dict[str, Any], info: dict[str, Any]) -> None:
    if not info or info.get("error"):
        return

--- a/python/sglang/srt/debug_utils/comparator/entrypoint.py
+++ b/python/sglang/srt/debug_utils/comparator/entrypoint.py
@@ -2,6 +2,7 @@ from __future__ import annotations

 import argparse
 import sys
+import traceback as _traceback_module
 from pathlib import Path
 from typing import Any, Iterator, Optional, Union

@@ -25,12 +26,13 @@ from sglang.srt.debug_utils.comparator.bundle_matcher import (
 from sglang.srt.debug_utils.comparator.display import emit_display_records
 from sglang.srt.debug_utils.comparator.meta_overrider import MetaOverrider
 from sglang.srt.debug_utils.comparator.output_types import (
+    ComparisonErrorRecord,
+    ComparisonNonTensorRecord,
+    ComparisonSkipRecord,
+    ComparisonTensorRecord,
    ConfigRecord,
-    NonTensorComparisonRecord,
    RecordLocation,
-    SkipComparisonRecord,
    SummaryRecord,
-    TensorComparisonRecord,
 )
 from sglang.srt.debug_utils.comparator.per_token_visualizer import (
    generate_per_token_heatmap,
@@ -83,12 +85,6 @@ def run(args: argparse.Namespace) -> int:
        verbosity=args.verbosity,
    )

-    report_path: Optional[Path] = _resolve_report_path(
-        target_path=dir_pair.y,
-        report_path_arg=args.report_path,
-    )
-    report_sink.configure(output_format=args.output_format, report_path=report_path)
-
    try:
        report_sink.add(ConfigRecord(config=vars(args)))

@@ -142,9 +138,11 @@ def run(args: argparse.Namespace) -> int:
            compute_per_token=visualize_per_token is not None,
            meta_overrider=meta_overrider,
        )
-        summary, skipped_names, failed_names = _consume_comparison_records(
-            comparison_records=comparison_records,
-            visualize_per_token=visualize_per_token,
+        summary, skipped_names, failed_names, errored_names = (
+            _consume_comparison_records(
+                comparison_records=comparison_records,
+                visualize_per_token=visualize_per_token,
+            )
        )
        return compute_exit_code(
            summary,
@@ -152,6 +150,7 @@ def run(args: argparse.Namespace) -> int:
            skipped_names=skipped_names,
            allow_failed_pattern=args.allow_failed_pattern,
            failed_names=failed_names,
+            errored_names=errored_names,
        )
    finally:
        report_sink.close()
@@ -219,7 +218,12 @@ def _compare_bundle_pairs(
    compute_per_token: bool = False,
    meta_overrider: Optional[MetaOverrider] = None,
 ) -> Iterator[
-    Union[TensorComparisonRecord, SkipComparisonRecord, NonTensorComparisonRecord]
+    Union[
+        ComparisonTensorRecord,
+        ComparisonSkipRecord,
+        ComparisonNonTensorRecord,
+        ComparisonErrorRecord,
+    ]
 ]:
    for bundle_info_pair in bundle_info_pairs:
        if not bundle_info_pair.y:
@@ -229,20 +233,32 @@ def _compare_bundle_pairs(
        filenames_pair: Pair[list[str]] = bundle_info_pair.map(
            lambda infos: [info.filename for info in infos]
        )
+
        record: Union[
-            TensorComparisonRecord, SkipComparisonRecord, NonTensorComparisonRecord
-        ] = compare_bundle_pair(
-            name=name,
-            filenames_pair=filenames_pair,
-            dir_pair=dir_pair,
-            token_aligner_mode=token_aligner_mode,
-            token_aligner_plan=token_aligner_plan,
-            diff_threshold=diff_threshold,
-            thd_seq_lens_by_step_pair=thd_seq_lens_by_step_pair,
-            viz_output_dir=viz_output_dir,
-            compute_per_token=compute_per_token,
-            meta_overrider=meta_overrider,
-        )
+            ComparisonTensorRecord,
+            ComparisonSkipRecord,
+            ComparisonNonTensorRecord,
+            ComparisonErrorRecord,
+        ]
+        try:
+            record = compare_bundle_pair(
+                name=name,
+                filenames_pair=filenames_pair,
+                dir_pair=dir_pair,
+                token_aligner_mode=token_aligner_mode,
+                token_aligner_plan=token_aligner_plan,
+                diff_threshold=diff_threshold,
+                thd_seq_lens_by_step_pair=thd_seq_lens_by_step_pair,
+                viz_output_dir=viz_output_dir,
+                compute_per_token=compute_per_token,
+                meta_overrider=meta_overrider,
+            )
+        except Exception as exc:
+            record = ComparisonErrorRecord(
+                name=name,
+                exception_type=type(exc).__name__,
+                traceback_str=_traceback_module.format_exc(),
+            )

        target_steps: set[int] = {info.step for info in bundle_info_pair.y}
        step: Optional[int] = target_steps.pop() if len(target_steps) == 1 else None
@@ -255,24 +271,32 @@ def _compare_bundle_pairs(
 def _consume_comparison_records(
    *,
    comparison_records: Iterator[
-        Union[TensorComparisonRecord, SkipComparisonRecord, NonTensorComparisonRecord]
+        Union[
+            ComparisonTensorRecord,
+            ComparisonSkipRecord,
+            ComparisonNonTensorRecord,
+            ComparisonErrorRecord,
+        ]
    ],
    visualize_per_token: Optional[Path] = None,
-) -> tuple[SummaryRecord, list[str], list[str]]:
-    counts: dict[str, int] = {"passed": 0, "failed": 0, "skipped": 0}
-    collected_comparisons: list[TensorComparisonRecord] = []
+) -> tuple[SummaryRecord, list[str], list[str], list[str]]:
+    counts: dict[str, int] = {"passed": 0, "failed": 0, "skipped": 0, "errored": 0}
+    collected_comparisons: list[ComparisonTensorRecord] = []
    skipped_names: list[str] = []
    failed_names: list[str] = []
+    errored_names: list[str] = []

    for record in comparison_records:
        counts[record.category] += 1
        report_sink.add(record)
-        if isinstance(record, SkipComparisonRecord) and record.category == "skipped":
+        if isinstance(record, ComparisonSkipRecord) and record.category == "skipped":
            skipped_names.append(record.name)
        if record.category == "failed":
            failed_names.append(record.name)
+        if isinstance(record, ComparisonErrorRecord):
+            errored_names.append(record.name)
        if visualize_per_token is not None and isinstance(
-            record, TensorComparisonRecord
+            record, ComparisonTensorRecord
        ):
            collected_comparisons.append(record)

@@ -285,7 +309,7 @@ def _consume_comparison_records(
            output_path=visualize_per_token,
        )

-    return summary, skipped_names, failed_names
+    return summary, skipped_names, failed_names, errored_names


 def parse_args(argv: list[str]) -> argparse.Namespace:
--- a/python/sglang/srt/debug_utils/comparator/output_formatter.py
+++ b/python/sglang/srt/debug_utils/comparator/output_formatter.py
@@ -26,14 +26,15 @@ if TYPE_CHECKING:
    )
    from sglang.srt.debug_utils.comparator.aligner.entrypoint.types import AlignerPlan
    from sglang.srt.debug_utils.comparator.output_types import (
+        ComparisonErrorRecord,
+        ComparisonNonTensorRecord,
+        ComparisonSkipRecord,
+        ComparisonTensorRecord,
        ConfigRecord,
        ErrorLog,
        InfoLog,
        LogRecord,
-        NonTensorComparisonRecord,
-        SkipComparisonRecord,
        SummaryRecord,
-        TensorComparisonRecord,
        _OutputRecord,
        _TableRecord,
    )
@@ -111,15 +112,15 @@ def _format_config_rich_body(
    return Panel("\n".join(lines), title="Comparator Config", border_style="cyan")


-# ── SkipComparisonRecord ─────────────────────────────────────────────
+# ── ComparisonSkipRecord ─────────────────────────────────────────────


-def _format_skip_body(record: SkipComparisonRecord) -> str:
+def _format_skip_body(record: ComparisonSkipRecord) -> str:
    return f"Skip: {record.name}{record._format_location_suffix()} ({record.reason})"


 def _format_skip_rich_body(
-    record: SkipComparisonRecord, verbosity: Verbosity = "normal"
+    record: ComparisonSkipRecord, verbosity: Verbosity = "normal"
 ) -> RenderableType:
    suffix: str = record._format_location_suffix()
    return (
@@ -127,6 +128,30 @@ def _format_skip_rich_body(
    )


+# ── ComparisonErrorRecord ────────────────────────────────────────────
+
+
+def _format_error_body(record: ComparisonErrorRecord) -> str:
+    prefix: str = record._format_location_prefix()
+    return (
+        f"{prefix}Error: {record.name} ({record.exception_type})\n"
+        f"{record.traceback_str}"
+    )
+
+
+def _format_error_rich_body(
+    record: ComparisonErrorRecord, verbosity: Verbosity = "normal"
+) -> RenderableType:
+    prefix: str = record._format_location_prefix_rich()
+    name: str = escape(record.name)
+    header: str = (
+        f"{prefix}[bold red]{name} ── errored ({escape(record.exception_type)})[/]"
+    )
+    if verbosity == "minimal":
+        return header
+    return header + f"\n[dim]{escape(record.traceback_str)}[/]"
+
+
 # ── _TableRecord ─────────────────────────────────────────────────────


@@ -154,10 +179,10 @@ def _format_table_rich_body(
    )


-# ── TensorComparisonRecord ───────────────────────────────────────────
+# ── ComparisonTensorRecord ───────────────────────────────────────────


-def _format_tensor_comparison_body(record: TensorComparisonRecord) -> str:
+def _format_tensor_comparison_body(record: ComparisonTensorRecord) -> str:
    body: str = record._format_location_prefix() + format_comparison(record)
    if record.replicated_checks:
        body += "\n" + format_replicated_checks(record.replicated_checks)
@@ -167,7 +192,7 @@ def _format_tensor_comparison_body(record: TensorComparisonRecord) -> str:


 def _format_tensor_comparison_rich_body(
-    record: TensorComparisonRecord, verbosity: Verbosity = "normal"
+    record: ComparisonTensorRecord, verbosity: Verbosity = "normal"
 ) -> RenderableType:
    from sglang.srt.debug_utils.comparator.tensor_comparator.formatter import (
        format_comparison_rich,
@@ -178,10 +203,10 @@ def _format_tensor_comparison_rich_body(
    )


-# ── NonTensorComparisonRecord ────────────────────────────────────────
+# ── ComparisonNonTensorRecord ────────────────────────────────────────


-def _format_non_tensor_body(record: NonTensorComparisonRecord) -> str:
+def _format_non_tensor_body(record: ComparisonNonTensorRecord) -> str:
    suffix: str = record._format_location_suffix()
    if record.values_equal:
        return f"NonTensor: {record.name}{suffix} = {record.baseline_value} ({record.baseline_type}) [equal]"
@@ -193,7 +218,7 @@ def _format_non_tensor_body(record: NonTensorComparisonRecord) -> str:


 def _format_non_tensor_rich_body(
-    record: NonTensorComparisonRecord, verbosity: Verbosity = "normal"
+    record: ComparisonNonTensorRecord, verbosity: Verbosity = "normal"
 ) -> RenderableType:
    suffix: str = record._format_location_suffix()
    name: str = escape(record.name)
@@ -216,10 +241,13 @@ def _format_non_tensor_rich_body(


 def _format_summary_body(record: SummaryRecord) -> str:
-    return (
+    text: str = (
        f"Summary: {record.passed} passed, {record.failed} failed, "
        f"{record.skipped} skipped (total {record.total})"
    )
+    if record.errored > 0:
+        text += f", {record.errored} errored"
+    return text


 def _format_summary_rich_body(
@@ -231,6 +259,8 @@ def _format_summary_rich_body(
        f"[yellow]{record.skipped} skipped[/] │ "
        f"{record.total} total"
    )
+    if record.errored > 0:
+        text += f" │ [bold red]{record.errored} errored[/]"
    return Panel(text, title="SUMMARY", border_style="bold")


--- a/python/sglang/srt/debug_utils/comparator/output_types.py
+++ b/python/sglang/srt/debug_utils/comparator/output_types.py
@@ -13,6 +13,8 @@ from sglang.srt.debug_utils.comparator.output_formatter import (  # noqa: F401
 from sglang.srt.debug_utils.comparator.output_formatter import (
    _format_config_body,
    _format_config_rich_body,
+    _format_error_body,
+    _format_error_rich_body,
    _format_log_body,
    _format_non_tensor_body,
    _format_non_tensor_rich_body,
@@ -146,8 +148,8 @@ class ConfigRecord(_OutputRecord):
        return _format_config_rich_body(self, verbosity=verbosity)


-class SkipComparisonRecord(_BaseComparisonRecord):
-    type: Literal["skip"] = "skip"
+class ComparisonSkipRecord(_BaseComparisonRecord):
+    type: Literal["comparison_skip"] = "comparison_skip"
    name: str
    reason: str

@@ -164,6 +166,23 @@ class SkipComparisonRecord(_BaseComparisonRecord):
        return _format_skip_rich_body(self, verbosity=verbosity)


+class ComparisonErrorRecord(_BaseComparisonRecord):
+    type: Literal["comparison_error"] = "comparison_error"
+    name: str
+    exception_type: str
+    traceback_str: str
+
+    @property
+    def category(self) -> str:
+        return "errored"
+
+    def _format_body(self) -> str:
+        return _format_error_body(self)
+
+    def _format_rich_body(self, verbosity: Verbosity = "normal") -> RenderableType:
+        return _format_error_rich_body(self, verbosity=verbosity)
+
+
 class _TableRecord(_OutputRecord):
    label: str
    rows: list[dict[str, Any]]
@@ -177,15 +196,6 @@ class _TableRecord(_OutputRecord):
    def _format_rich_body(self, verbosity: Verbosity = "normal") -> RenderableType:
        return _format_table_rich_body(self, verbosity=verbosity)

-        return _format_table_body(self)
-
-    def _format_rich_body(self) -> RenderableType:
-        from sglang.srt.debug_utils.comparator.output_formatter import (
-            _format_table_rich_body,
-        )
-
-        return _format_table_rich_body(self)
-

 class RankInfoRecord(_TableRecord):
    type: Literal["rank_info"] = "rank_info"
@@ -201,10 +211,10 @@ class InputIdsRecord(_TableRecord):
        return f"{self.label} input_ids & positions"


-class TensorComparisonRecord(TensorComparisonInfo, _BaseComparisonRecord):
+class ComparisonTensorRecord(TensorComparisonInfo, _BaseComparisonRecord):
    model_config = ConfigDict(extra="forbid", defer_build=True)

-    type: Literal["comparison"] = "comparison"
+    type: Literal["comparison_tensor"] = "comparison_tensor"
    traced_plan: Optional[TracedAlignerPlan] = None
    replicated_checks: list[ReplicatedCheckResult] = Field(default_factory=list)
    raw_bundle_info: Optional[Pair[BundleSideInfo]] = None
@@ -224,8 +234,8 @@ class TensorComparisonRecord(TensorComparisonInfo, _BaseComparisonRecord):
        return _format_tensor_comparison_rich_body(self, verbosity=verbosity)


-class NonTensorComparisonRecord(_BaseComparisonRecord):
-    type: Literal["non_tensor"] = "non_tensor"
+class ComparisonNonTensorRecord(_BaseComparisonRecord):
+    type: Literal["comparison_non_tensor"] = "comparison_non_tensor"
    name: str
    baseline_value: str
    target_value: str
@@ -245,15 +255,6 @@ class NonTensorComparisonRecord(_BaseComparisonRecord):
    def _format_rich_body(self, verbosity: Verbosity = "normal") -> RenderableType:
        return _format_non_tensor_rich_body(self, verbosity=verbosity)

-        return _format_non_tensor_body(self)
-
-    def _format_rich_body(self) -> RenderableType:
-        from sglang.srt.debug_utils.comparator.output_formatter import (
-            _format_non_tensor_rich_body,
-        )
-
-        return _format_non_tensor_rich_body(self)
-

 class SummaryRecord(_OutputRecord):
    type: Literal["summary"] = "summary"
@@ -261,13 +262,15 @@ class SummaryRecord(_OutputRecord):
    passed: int
    failed: int
    skipped: int
+    errored: int = 0

    @model_validator(mode="after")
    def _validate_totals(self) -> "SummaryRecord":
-        expected: int = self.passed + self.failed + self.skipped
+        expected: int = self.passed + self.failed + self.skipped + self.errored
        if self.total != expected:
            raise ValueError(
-                f"total={self.total} != passed({self.passed}) + failed({self.failed}) + skipped({self.skipped}) = {expected}"
+                f"total={self.total} != passed({self.passed}) + failed({self.failed}) "
+                f"+ skipped({self.skipped}) + errored({self.errored}) = {expected}"
            )
        return self

@@ -277,7 +280,6 @@ class SummaryRecord(_OutputRecord):
    def _format_rich_body(self, verbosity: Verbosity = "normal") -> RenderableType:
        return _format_summary_rich_body(self, verbosity=verbosity)

-        return _format_summary_body(self)

 class LogRecord(_OutputRecord):
    type: Literal["log"] = "log"
@@ -291,9 +293,10 @@ AnyRecord = Annotated[
        ConfigRecord,
        RankInfoRecord,
        InputIdsRecord,
-        SkipComparisonRecord,
-        TensorComparisonRecord,
-        NonTensorComparisonRecord,
+        ComparisonSkipRecord,
+        ComparisonErrorRecord,
+        ComparisonTensorRecord,
+        ComparisonNonTensorRecord,
        SummaryRecord,
        LogRecord,
    ],
--- a/python/sglang/srt/debug_utils/comparator/per_token_visualizer.py
+++ b/python/sglang/srt/debug_utils/comparator/per_token_visualizer.py
@@ -9,12 +9,12 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Optional

-from sglang.srt.debug_utils.comparator.output_types import TensorComparisonRecord
+from sglang.srt.debug_utils.comparator.output_types import ComparisonTensorRecord


 def generate_per_token_heatmap(
    *,
-    records: list[TensorComparisonRecord],
+    records: list[ComparisonTensorRecord],
    output_path: Path,
 ) -> Optional[Path]:
    """Generate a per-token relative difference heatmap PNG.
@@ -31,7 +31,7 @@ def generate_per_token_heatmap(

 def _collect_per_token_data(
    *,
-    records: list[TensorComparisonRecord],
+    records: list[ComparisonTensorRecord],
 ) -> list[tuple[str, list[float]]]:
    rows: list[tuple[str, list[float]]] = []
    for record in records:
--- a/python/sglang/srt/debug_utils/comparator/tensor_comparator/formatter.py
+++ b/python/sglang/srt/debug_utils/comparator/tensor_comparator/formatter.py
@@ -20,9 +20,9 @@ if TYPE_CHECKING:
    from sglang.srt.debug_utils.comparator.aligner.entrypoint.types import AlignerPlan
    from sglang.srt.debug_utils.comparator.output_types import (
        BundleSideInfo,
+        ComparisonTensorRecord,
        ReplicatedCheckResult,
        ShapeSnapshot,
-        TensorComparisonRecord,
    )
    from sglang.srt.debug_utils.comparator.utils import Pair

@@ -207,7 +207,7 @@ def _format_diff(diff: DiffInfo, prefix_text: str = "") -> list[str]:


 def format_comparison_rich(
-    record: TensorComparisonRecord,
+    record: ComparisonTensorRecord,
    verbosity: Verbosity = "normal",
 ) -> str:
    if verbosity == "minimal":
@@ -219,7 +219,7 @@ def format_comparison_rich(
    )


-def _format_comparison_minimal(record: TensorComparisonRecord) -> str:
+def _format_comparison_minimal(record: ComparisonTensorRecord) -> str:
    passed, color, marker = _category_marker(record.category)

    name_part: str = f"[bold {color}]{escape(record.name):30s}[/]"
@@ -233,7 +233,7 @@ def _format_comparison_minimal(record: TensorComparisonRecord) -> str:

 def _format_comparison_normal_or_verbose(
    *,
-    record: TensorComparisonRecord,
+    record: ComparisonTensorRecord,
    verbose: bool,
 ) -> str:
    passed, color, marker = _category_marker(record.category)
--- a/python/sglang/srt/debug_utils/comparator/utils.py
+++ b/python/sglang/srt/debug_utils/comparator/utils.py
@@ -141,10 +141,14 @@ def compute_exit_code(
    skipped_names: list[str],
    allow_failed_pattern: Optional[str],
    failed_names: list[str],
+    errored_names: Optional[list[str]] = None,
 ) -> int:
    if summary.passed == 0:
        return 1

+    if errored_names:
+        return 1
+
    if not _is_all_match_pattern(pattern=allow_failed_pattern, strings=failed_names):
        return 1