chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Enhance error resilience in dump comparator (#19685)

Browse Source
This commit is contained in:
fzyzcjy
2026-03-02 19:08:35 +08:00
committed by GitHub
parent e5ef845cad
commit 7579ab3f33
18 changed files with 495 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py
View File

@@ -34,9 +34,9 @@ from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis, TokenLayou
from sglang.srt.debug_utils.comparator.output_types import (
BundleFileInfo,
BundleSideInfo,
ComparisonTensorRecord,
ReplicatedCheckResult,
ShapeSnapshot,
TensorComparisonRecord,
)
from sglang.srt.debug_utils.comparator.tensor_comparator.formatter import (
_format_abs_diff_percentiles_rich,
@@ -273,9 +273,9 @@ def _make_comparison_record(
replicated_checks: list[ReplicatedCheckResult] | None = None,
raw_bundle_info: Pair[BundleSideInfo] | None = None,
traced_plan: TracedAlignerPlan | None = None,
) -> TensorComparisonRecord:
) -> ComparisonTensorRecord:
s: list[int] = shape if shape is not None else [4, 8]
return TensorComparisonRecord(
return ComparisonTensorRecord(
name=name,
baseline=_make_tensor_info(shape=s, dtype=dtype, sample=sample),
target=_make_tensor_info(shape=s, dtype=dtype, sample=sample),
@@ -417,7 +417,7 @@ class TestFormatComparisonRichMinimal:
"""format_comparison_rich() with verbosity='minimal'."""
def test_passed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(rel_diff=1e-4, passed=True),
)
result: str = format_comparison_rich(record, verbosity="minimal")
@@ -428,7 +428,7 @@ class TestFormatComparisonRichMinimal:
)
def test_failed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(rel_diff=0.5, passed=False),
)
result: str = format_comparison_rich(record, verbosity="minimal")
@@ -439,7 +439,7 @@ class TestFormatComparisonRichMinimal:
)
def test_shape_mismatch(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
shape_mismatch=True,
)
result: str = format_comparison_rich(record, verbosity="minimal")
@@ -450,7 +450,7 @@ class TestFormatComparisonRichMinimal:
)
def test_no_diff(self) -> None:
record: TensorComparisonRecord = _make_comparison_record()
record: ComparisonTensorRecord = _make_comparison_record()
result: str = format_comparison_rich(record, verbosity="minimal")
assert result == ("[red]❌[/] [bold red]hidden_states [/]")
@@ -460,7 +460,7 @@ class TestFormatComparisonRichNormal:
"""format_comparison_rich() with verbosity='normal'."""
def test_passed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(rel_diff=1e-4, passed=True),
)
result: str = format_comparison_rich(record, verbosity="normal")
@@ -477,7 +477,7 @@ class TestFormatComparisonRichNormal:
)
def test_failed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(
rel_diff=0.5, max_abs_diff=1.0, mean_abs_diff=0.3, passed=False
),
@@ -499,7 +499,7 @@ class TestFormatComparisonRichNormal:
)
def test_shape_mismatch(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
shape_mismatch=True,
)
result: str = format_comparison_rich(record, verbosity="normal")
@@ -516,7 +516,7 @@ class TestFormatComparisonRichNormal:
)
def test_with_downcast(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(rel_diff=0.01, passed=False),
diff_downcast=_make_diff(rel_diff=1e-5, passed=True),
downcast_dtype="torch.bfloat16",
@@ -543,7 +543,7 @@ class TestFormatComparisonRichNormal:
x=_make_bundle_side_info(num_files=2, dims="b s h(tp) d"),
y=_make_bundle_side_info(num_files=2, dims="b s h(tp) d"),
)
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(passed=True),
raw_bundle_info=bundle_info,
)
@@ -565,7 +565,7 @@ class TestFormatComparisonRichNormal:
def test_with_plan(self) -> None:
plan: AlignerPlan = _make_simple_aligner_plan(with_unsharder=True)
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(passed=True),
traced_plan=_make_traced_plan(plan),
)
@@ -590,7 +590,7 @@ class TestFormatComparisonRichVerbose:
"""format_comparison_rich() with verbosity='verbose'."""
def test_passed_full_detail(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(rel_diff=1e-4, passed=True),
sample="tensor([0.1, 0.2, ...])",
)
@@ -624,7 +624,7 @@ class TestFormatComparisonRichVerbose:
x=_make_bundle_side_info(num_files=2, with_parallel_info=True),
y=_make_bundle_side_info(num_files=2, with_parallel_info=True),
)
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(passed=True),
raw_bundle_info=bundle_info,
)
@@ -659,7 +659,7 @@ class TestFormatComparisonRichVerbose:
def test_with_plan_and_traces(self) -> None:
plan: AlignerPlan = _make_simple_aligner_plan(with_unsharder=True)
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff(passed=True),
traced_plan=_make_traced_plan(
plan,

34
test/registered/debug_utils/comparator/tensor_comparator/test_types.py
View File

@@ -4,14 +4,14 @@ import sys
import pytest
from sglang.srt.debug_utils.comparator.output_types import (
ComparisonSkipRecord,
ComparisonTensorRecord,
ConfigRecord,
ErrorLog,
InfoLog,
LogRecord,
ReplicatedCheckResult,
SkipComparisonRecord,
SummaryRecord,
TensorComparisonRecord,
parse_record_json,
)
from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
@@ -84,7 +84,7 @@ class TestStrictBase:
class TestRecordTypes:
def test_comparison_record_inherits_tensor_fields(self):
record = TensorComparisonRecord(
record = ComparisonTensorRecord(
name="hidden_states",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -93,7 +93,7 @@ class TestRecordTypes:
diff=_make_diff(),
)
parsed = json.loads(record.model_dump_json())
assert parsed["type"] == "comparison"
assert parsed["type"] == "comparison_tensor"
assert parsed["name"] == "hidden_states"
assert "baseline" in parsed
assert "diff" in parsed
@@ -109,8 +109,8 @@ class TestRecordTypes:
"end_step": 100,
}
),
SkipComparisonRecord(name="attn", reason="no_baseline"),
TensorComparisonRecord(
ComparisonSkipRecord(name="attn", reason="no_baseline"),
ComparisonTensorRecord(
name="mlp",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -149,8 +149,8 @@ def _make_replicated_check(**overrides) -> ReplicatedCheckResult:
class TestWarnings:
def test_comparison_record_failed_when_diff_passed_but_errors(self):
"""TensorComparisonRecord with diff.passed=True but errors → category=='failed'."""
record = TensorComparisonRecord(
"""ComparisonTensorRecord with diff.passed=True but errors → category=='failed'."""
record = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -162,8 +162,8 @@ class TestWarnings:
assert record.category == "failed"
def test_skip_record_failed_when_errors(self):
"""SkipComparisonRecord with errors → category=='failed' instead of 'skipped'."""
record = SkipComparisonRecord(
"""ComparisonSkipRecord with errors → category=='failed' instead of 'skipped'."""
record = ComparisonSkipRecord(
name="x",
reason="no_baseline",
errors=[ErrorLog(category="test", message="some warning")],
@@ -171,8 +171,8 @@ class TestWarnings:
assert record.category == "failed"
def test_replicated_checks_all_passed(self):
"""TensorComparisonRecord with all replicated_checks passed → category=='passed'."""
record = TensorComparisonRecord(
"""ComparisonTensorRecord with all replicated_checks passed → category=='passed'."""
record = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -184,8 +184,8 @@ class TestWarnings:
assert record.category == "passed"
def test_replicated_checks_failed_means_record_failed(self):
"""TensorComparisonRecord with any replicated_check.passed=False → category=='failed'."""
record = TensorComparisonRecord(
"""ComparisonTensorRecord with any replicated_check.passed=False → category=='failed'."""
record = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -197,7 +197,7 @@ class TestWarnings:
assert record.category == "failed"
def test_replicated_check_json_round_trip(self):
"""ReplicatedCheckResult survives JSON round-trip via TensorComparisonRecord."""
"""ReplicatedCheckResult survives JSON round-trip via ComparisonTensorRecord."""
check = _make_replicated_check(
axis="cp",
group_index=2,
@@ -205,7 +205,7 @@ class TestWarnings:
baseline_index=0,
passed=False,
)
record = TensorComparisonRecord(
record = ComparisonTensorRecord(
name="mlp",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -216,7 +216,7 @@ class TestWarnings:
)
restored = parse_record_json(record.model_dump_json())
assert isinstance(restored, TensorComparisonRecord)
assert isinstance(restored, ComparisonTensorRecord)
assert len(restored.replicated_checks) == 1
restored_check: ReplicatedCheckResult = restored.replicated_checks[0]

173
test/registered/debug_utils/comparator/test_entrypoint.py
View File

@@ -7,6 +7,7 @@ from pathlib import Path
import pytest
import torch
import sglang.srt.debug_utils.comparator.entrypoint as _entrypoint_module
import sglang.srt.debug_utils.dumper as _dumper_module
from sglang.srt.debug_utils.comparator.entrypoint import (
parse_args,
@@ -14,14 +15,15 @@ from sglang.srt.debug_utils.comparator.entrypoint import (
)
from sglang.srt.debug_utils.comparator.output_types import (
AnyRecord,
ComparisonErrorRecord,
ComparisonNonTensorRecord,
ComparisonSkipRecord,
ComparisonTensorRecord,
ConfigRecord,
InfoLog,
LogRecord,
NonTensorComparisonRecord,
ReplicatedCheckResult,
SkipComparisonRecord,
SummaryRecord,
TensorComparisonRecord,
_OutputRecord,
parse_record_json,
)
@@ -39,7 +41,7 @@ class TestEntrypointGroupingRaw:
"""Test `--grouping-skip-keys` empty (raw) scenarios"""
def test_run_basic(self, tmp_path, capsys):
"""Two matching tensors produce ConfigRecord, 2 TensorComparisonRecords, and SummaryRecord."""
"""Two matching tensors produce ConfigRecord, 2 ComparisonTensorRecords, and SummaryRecord."""
baseline_path, target_path = _create_dumps(tmp_path, ["tensor_a", "tensor_b"])
argv = _make_argv(baseline_path, target_path, preset="raw")
@@ -54,7 +56,7 @@ class TestEntrypointGroupingRaw:
assert summary.skipped == 0
def test_filter(self, tmp_path, capsys):
"""--filter selects only the matching tensor, producing 1 TensorComparisonRecord."""
"""--filter selects only the matching tensor, producing 1 ComparisonTensorRecord."""
baseline_path, target_path = _create_dumps(tmp_path, ["tensor_a", "tensor_b"])
argv = _make_argv(baseline_path, target_path, filter="tensor_a", preset="raw")
@@ -62,7 +64,7 @@ class TestEntrypointGroupingRaw:
assert len(_get_comparisons(records)) == 1
def test_no_baseline_skip(self, tmp_path, capsys):
"""Target tensor missing from baseline emits a SkipComparisonRecord with reason baseline_load_failed."""
"""Target tensor missing from baseline emits a ComparisonSkipRecord with reason baseline_load_failed."""
baseline_path, target_path = _create_dumps(
tmp_path,
tensor_names=["tensor_a", "tensor_extra"],
@@ -71,7 +73,7 @@ class TestEntrypointGroupingRaw:
argv = _make_argv(baseline_path, target_path, preset="raw")
records, _ = _run_and_parse(argv, capsys)
skips = [r for r in records if isinstance(r, SkipComparisonRecord)]
skips = [r for r in records if isinstance(r, ComparisonSkipRecord)]
assert len(skips) == 1
assert skips[0].reason == "baseline_load_failed"
@@ -100,7 +102,7 @@ class TestEntrypointGroupingRaw:
assert all(isinstance(r, _OutputRecord) for r in records)
def test_comparison_failed(self, tmp_path, capsys):
"""Completely different tensors produce a failed TensorComparisonRecord."""
"""Completely different tensors produce a failed ComparisonTensorRecord."""
torch.manual_seed(42)
baseline_path = _create_rank_dump(
tmp_path / "baseline", rank=0, name="tensor_a", tensor=torch.randn(10, 10)
@@ -251,7 +253,7 @@ class TestEntrypointGroupingRaw:
assert summary.total == 0
def test_raw_multi_rank(self, tmp_path, capsys):
"""Two ranks in raw grouping produce two TensorComparisonRecords (one per rank)."""
"""Two ranks in raw grouping produce two ComparisonTensorRecords (one per rank)."""
torch.manual_seed(42)
tensor = torch.randn(4, 4)
@@ -480,7 +482,7 @@ class TestEntrypointGroupingLogical:
],
)
def test_ambiguous_no_dims_skip(self, tmp_path, capsys, bad_side, expected_reason):
"""Multi-rank without dims on one side produces a SkipComparisonRecord with the appropriate reason."""
"""Multi-rank without dims on one side produces a ComparisonSkipRecord with the appropriate reason."""
torch.manual_seed(42)
tensor = torch.randn(4, 8)
@@ -500,7 +502,7 @@ class TestEntrypointGroupingLogical:
)
records, _ = _run_and_parse(argv, capsys)
skips = [r for r in records if isinstance(r, SkipComparisonRecord)]
skips = [r for r in records if isinstance(r, ComparisonSkipRecord)]
assert len(skips) == 1
assert skips[0].reason == expected_reason
@@ -1092,7 +1094,7 @@ class TestEntrypointPerStepMode:
"""Test per-step comparison mode (sglang_dev preset behavior)."""
def test_multi_step_per_step_comparison(self, tmp_path, capsys):
"""Multiple steps produce one TensorComparisonRecord per step with step field set."""
"""Multiple steps produce one ComparisonTensorRecord per step with step field set."""
torch.manual_seed(42)
baseline_path, target_path = _create_dumps(tmp_path, ["tensor_a"], num_steps=3)
argv = _make_argv(baseline_path, target_path, diff_threshold=0.1)
@@ -1149,7 +1151,7 @@ class TestEntrypointPerStepMode:
assert all(c.baseline.shape == [4, 8] for c in comparisons)
def test_single_step_has_step_field(self, tmp_path, capsys):
"""Single step produces TensorComparisonRecord with location.step=0."""
"""Single step produces ComparisonTensorRecord with location.step=0."""
baseline_path, target_path = _create_dumps(tmp_path, ["tensor_a"], num_steps=1)
argv = _make_argv(baseline_path, target_path)
@@ -1458,7 +1460,7 @@ class TestEntrypointConcatMode:
assert len(comparisons) == 3
def test_concat_aligner_plan_fields(self, tmp_path, capsys):
"""TensorComparisonRecord.traced_plan reports mode='concat' with plan=None."""
"""ComparisonTensorRecord.traced_plan reports mode='concat' with plan=None."""
torch.manual_seed(42)
records = self._run_concat(
@@ -1599,8 +1601,8 @@ class TestEntrypointConcatMode:
)
records, _ = _run_and_parse(argv, capsys)
comparisons: list[TensorComparisonRecord] = _get_comparisons(records)
hidden_comparisons: list[TensorComparisonRecord] = [
comparisons: list[ComparisonTensorRecord] = _get_comparisons(records)
hidden_comparisons: list[ComparisonTensorRecord] = [
c for c in comparisons if c.name == "hidden_states"
]
assert len(hidden_comparisons) >= 1
@@ -2158,7 +2160,7 @@ class TestEntrypointNonTensorValues:
"""Test non-tensor value comparison through the full entrypoint pipeline."""
def test_non_tensor_float_same_value(self, tmp_path: Path, capsys) -> None:
"""Two sides dump the same float → NonTensorComparisonRecord with values_equal=True, category=passed."""
"""Two sides dump the same float → ComparisonNonTensorRecord with values_equal=True, category=passed."""
baseline_path, target_path = _create_non_tensor_dumps(
tmp_path, name="sm_scale", baseline_value=0.125, target_value=0.125
)
@@ -2177,7 +2179,7 @@ class TestEntrypointNonTensorValues:
assert summary.failed == 0
def test_non_tensor_float_different_value(self, tmp_path: Path, capsys) -> None:
"""Two sides dump different floats → NonTensorComparisonRecord with values_equal=False, category=failed."""
"""Two sides dump different floats → ComparisonNonTensorRecord with values_equal=False, category=failed."""
baseline_path, target_path = _create_non_tensor_dumps(
tmp_path, name="sm_scale", baseline_value=0.125, target_value=0.25
)
@@ -2262,7 +2264,7 @@ class TestEntrypointNonTensorValues:
assert non_tensors[0].target_type == "dict"
def test_non_tensor_none_value(self, tmp_path: Path, capsys) -> None:
"""Dumping None is displayed as NonTensorComparisonRecord, not skipped as load failure."""
"""Dumping None is displayed as ComparisonNonTensorRecord, not skipped as load failure."""
baseline_path, target_path = _create_non_tensor_dumps(
tmp_path, name="optional_param", baseline_value=None, target_value=None
)
@@ -2278,7 +2280,7 @@ class TestEntrypointNonTensorValues:
assert non_tensors[0].category == "passed"
def test_non_tensor_json_roundtrip(self, tmp_path: Path, capsys) -> None:
"""NonTensorComparisonRecord JSON output can be parsed back correctly."""
"""ComparisonNonTensorRecord JSON output can be parsed back correctly."""
baseline_path, target_path = _create_non_tensor_dumps(
tmp_path, name="sm_scale", baseline_value=0.125, target_value=0.125
)
@@ -2290,7 +2292,7 @@ class TestEntrypointNonTensorValues:
json_str: str = non_tensors[0].model_dump_json()
roundtripped = parse_record_json(json_str)
assert isinstance(roundtripped, NonTensorComparisonRecord)
assert isinstance(roundtripped, ComparisonNonTensorRecord)
assert roundtripped.name == "sm_scale"
assert roundtripped.values_equal is True
@@ -2344,17 +2346,17 @@ class TestEntrypointVisualize:
# --------------------------- Assertion helpers -------------------
def _get_comparisons(records: list[AnyRecord]) -> list[TensorComparisonRecord]:
return [r for r in records if isinstance(r, TensorComparisonRecord)]
def _get_comparisons(records: list[AnyRecord]) -> list[ComparisonTensorRecord]:
return [r for r in records if isinstance(r, ComparisonTensorRecord)]
def _get_non_tensors(records: list[AnyRecord]) -> list[NonTensorComparisonRecord]:
return [r for r in records if isinstance(r, NonTensorComparisonRecord)]
def _get_non_tensors(records: list[AnyRecord]) -> list[ComparisonNonTensorRecord]:
return [r for r in records if isinstance(r, ComparisonNonTensorRecord)]
def _assert_single_comparison_passed(
records: list[AnyRecord],
) -> TensorComparisonRecord:
) -> ComparisonTensorRecord:
comparisons = _get_comparisons(records)
assert len(comparisons) == 1
assert comparisons[0].diff is not None
@@ -3235,8 +3237,8 @@ class TestEntrypointThdCpZigzag:
)
records, _ = _run_and_parse(argv, capsys)
comparisons: list[TensorComparisonRecord] = _get_comparisons(records)
hidden_comparisons: list[TensorComparisonRecord] = [
comparisons: list[ComparisonTensorRecord] = _get_comparisons(records)
hidden_comparisons: list[ComparisonTensorRecord] = [
c for c in comparisons if c.name == "hidden_states"
]
assert len(hidden_comparisons) >= 1
@@ -3287,8 +3289,8 @@ class TestEntrypointThdCpZigzag:
records, _ = _run_and_parse(argv, capsys)
# hidden_states should pass comparison (after unshard + reorder)
comparisons: list[TensorComparisonRecord] = _get_comparisons(records)
hidden_comparisons: list[TensorComparisonRecord] = [
comparisons: list[ComparisonTensorRecord] = _get_comparisons(records)
hidden_comparisons: list[ComparisonTensorRecord] = [
c for c in comparisons if c.name == "hidden_states"
]
assert len(hidden_comparisons) >= 1
@@ -3355,7 +3357,7 @@ class TestEntrypointDpFilter:
)
records, _ = _run_and_parse(argv, capsys)
comparison: TensorComparisonRecord = _assert_single_comparison_passed(records)
comparison: ComparisonTensorRecord = _assert_single_comparison_passed(records)
assert comparison.name == "hidden"
def test_dp2_megatron_both_sides(self, tmp_path: Path, capsys) -> None:
@@ -3410,7 +3412,7 @@ class TestEntrypointDpFilter:
)
records, _ = _run_and_parse(argv, capsys)
comparison: TensorComparisonRecord = _assert_single_comparison_passed(records)
comparison: ComparisonTensorRecord = _assert_single_comparison_passed(records)
assert comparison.name == "hidden"
def test_dp2_tp2_sglang(self, tmp_path: Path, capsys) -> None:
@@ -3458,7 +3460,7 @@ class TestEntrypointDpFilter:
)
records, _ = _run_and_parse(argv, capsys)
comparison: TensorComparisonRecord = _assert_single_comparison_passed(records)
comparison: ComparisonTensorRecord = _assert_single_comparison_passed(records)
assert comparison.name == "hidden"
def test_dp2_both_nonempty_raises(self, tmp_path: Path, capsys) -> None:
@@ -3496,10 +3498,12 @@ class TestEntrypointDpFilter:
diff_threshold=1e-3,
)
with pytest.raises(
AssertionError, match="Expected exactly 1 non-empty dp_rank"
):
_run_and_parse(argv, capsys)
records, exit_code = _run_and_parse(argv, capsys)
errors = [r for r in records if isinstance(r, ComparisonErrorRecord)]
assert len(errors) == 1
assert errors[0].exception_type == "AssertionError"
assert "Expected exactly 1 non-empty dp_rank" in errors[0].traceback_str
assert exit_code == 1
class TestEntrypointDpGroupAlias:
@@ -3542,7 +3546,7 @@ class TestEntrypointDpGroupAlias:
)
records, _ = _run_and_parse(argv, capsys)
comparison: TensorComparisonRecord = _assert_single_comparison_passed(records)
comparison: ComparisonTensorRecord = _assert_single_comparison_passed(records)
assert comparison.name == "hidden"
def test_dp_alias_via_override_dims(self, tmp_path: Path, capsys) -> None:
@@ -3599,7 +3603,7 @@ class TestEntrypointDpGroupAlias:
)
records, _ = _run_and_parse(argv, capsys)
comparison: TensorComparisonRecord = _assert_single_comparison_passed(records)
comparison: ComparisonTensorRecord = _assert_single_comparison_passed(records)
assert comparison.name == "hidden"
def test_dp_alias_with_real_alias_group_filters(
@@ -3640,7 +3644,7 @@ class TestEntrypointDpGroupAlias:
)
records, _ = _run_and_parse(argv, capsys)
comparison: TensorComparisonRecord = _assert_single_comparison_passed(records)
comparison: ComparisonTensorRecord = _assert_single_comparison_passed(records)
assert comparison.name == "hidden"
@@ -3679,7 +3683,7 @@ class TestEntrypointMetaOverride:
records: list[AnyRecord], *, expected_count: int = 1
) -> None:
"""Assert that exactly expected_count comparisons exist and all passed."""
comparisons: list[TensorComparisonRecord] = _get_comparisons(records)
comparisons: list[ComparisonTensorRecord] = _get_comparisons(records)
assert len(comparisons) == expected_count
assert all(c.diff is not None and c.diff.passed for c in comparisons)
@@ -3975,14 +3979,14 @@ class TestEntrypointMetaOverride:
)
records, _ = _run_and_parse(argv, capsys)
non_tensors: list[NonTensorComparisonRecord] = [
r for r in records if isinstance(r, NonTensorComparisonRecord)
non_tensors: list[ComparisonNonTensorRecord] = [
r for r in records if isinstance(r, ComparisonNonTensorRecord)
]
assert len(non_tensors) == 1
assert non_tensors[0].name == "sm_scale"
assert non_tensors[0].values_equal
comparisons: list[TensorComparisonRecord] = _get_comparisons(records)
comparisons: list[ComparisonTensorRecord] = _get_comparisons(records)
assert len(comparisons) == 1
assert comparisons[0].name == "hidden"
@@ -4356,12 +4360,9 @@ class TestEntrypointDpAttentionMissingAlias:
assert exit_code == 1
comparisons: list[TensorComparisonRecord] = _get_comparisons(records)
assert len(comparisons) == 1
comparison: TensorComparisonRecord = comparisons[0]
assert comparison.shape_mismatch is True
assert comparison.diff is None
assert comparison.category == "failed"
errors = [r for r in records if isinstance(r, ComparisonErrorRecord)]
assert len(errors) == 1
assert errors[0].category == "errored"
class TestEntrypointAutoDescend:
@@ -4460,5 +4461,77 @@ class TestEntrypointAutoDescend:
run(parse_args(argv))
class TestErrorResilience:
"""Bundle comparison exception → continue with remaining bundles."""
def test_one_bundle_errors_others_continue(self, tmp_path, capsys, monkeypatch):
"""One bundle raises exception → other bundles still compared, summary correct."""
baseline_path, target_path = _create_dumps(
tmp_path, ["tensor_a", "tensor_b", "tensor_c"]
)
argv = _make_argv(baseline_path, target_path, preset="raw")
original = _entrypoint_module.compare_bundle_pair
def _patched(**kwargs):
if kwargs["name"] == "tensor_b":
raise RuntimeError("intentional test error")
return original(**kwargs)
monkeypatch.setattr(_entrypoint_module, "compare_bundle_pair", _patched)
records, exit_code = _run_and_parse(argv, capsys)
comparisons = _get_comparisons(records)
assert len(comparisons) == 2
errors = [r for r in records if isinstance(r, ComparisonErrorRecord)]
assert len(errors) == 1
assert errors[0].name == "tensor_b"
assert errors[0].exception_type == "RuntimeError"
assert "intentional test error" in errors[0].traceback_str
summary = records[-1]
assert isinstance(summary, SummaryRecord)
assert summary.errored == 1
assert summary.passed == 2
assert summary.total == 3
assert exit_code == 1
def test_all_bundles_error_exits_one(self, tmp_path, capsys, monkeypatch):
"""All bundles error → exit 1, summary all errored."""
baseline_path, target_path = _create_dumps(tmp_path, ["tensor_a"])
argv = _make_argv(baseline_path, target_path, preset="raw")
def _always_raise(**kwargs):
raise ValueError("always fail")
monkeypatch.setattr(_entrypoint_module, "compare_bundle_pair", _always_raise)
records, exit_code = _run_and_parse(argv, capsys)
summary = records[-1]
assert isinstance(summary, SummaryRecord)
assert summary.errored == 1
assert summary.passed == 0
assert exit_code == 1
def test_error_record_json_roundtrip_in_output(self, tmp_path, capsys, monkeypatch):
"""ComparisonErrorRecord correctly serializes and deserializes in output."""
baseline_path, target_path = _create_dumps(tmp_path, ["tensor_a"])
argv = _make_argv(baseline_path, target_path, preset="raw")
def _raise(**kwargs):
raise TypeError("bad type")
monkeypatch.setattr(_entrypoint_module, "compare_bundle_pair", _raise)
records, _ = _run_and_parse(argv, capsys)
errors = [r for r in records if isinstance(r, ComparisonErrorRecord)]
assert len(errors) == 1
assert errors[0].exception_type == "TypeError"
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))

12
test/registered/debug_utils/comparator/test_manually_verify.py
View File

@@ -208,7 +208,7 @@ class TestPerTokenHeatmapManualVerify:
rows for different tensor names. Colorbar shows log10 scale.
"""
from sglang.srt.debug_utils.comparator.output_types import (
TensorComparisonRecord,
ComparisonTensorRecord,
)
from sglang.srt.debug_utils.comparator.per_token_visualizer import (
generate_per_token_heatmap,
@@ -222,7 +222,7 @@ class TestPerTokenHeatmapManualVerify:
hidden_dim: int = 128
num_tensors: int = 5
records: list[TensorComparisonRecord] = []
records: list[ComparisonTensorRecord] = []
for i in range(num_tensors):
baseline: torch.Tensor = torch.randn(seq_len, hidden_dim)
noise_scale: torch.Tensor = torch.linspace(
@@ -237,7 +237,7 @@ class TestPerTokenHeatmapManualVerify:
diff_threshold=1e-3,
seq_dim=0,
)
records.append(TensorComparisonRecord(**info.model_dump()))
records.append(ComparisonTensorRecord(**info.model_dump()))
output_path: Path = tmp_path / "per_token_increasing_diff.png"
result = generate_per_token_heatmap(records=records, output_path=output_path)
@@ -253,7 +253,7 @@ class TestPerTokenHeatmapManualVerify:
rest is dark/cold.
"""
from sglang.srt.debug_utils.comparator.output_types import (
TensorComparisonRecord,
ComparisonTensorRecord,
)
from sglang.srt.debug_utils.comparator.per_token_visualizer import (
generate_per_token_heatmap,
@@ -268,7 +268,7 @@ class TestPerTokenHeatmapManualVerify:
spike_pos: int = 32
num_tensors: int = 4
records: list[TensorComparisonRecord] = []
records: list[ComparisonTensorRecord] = []
for i in range(num_tensors):
baseline: torch.Tensor = torch.randn(seq_len, hidden_dim)
target: torch.Tensor = baseline.clone()
@@ -281,7 +281,7 @@ class TestPerTokenHeatmapManualVerify:
diff_threshold=1e-3,
seq_dim=0,
)
records.append(TensorComparisonRecord(**info.model_dump()))
records.append(ComparisonTensorRecord(**info.model_dump()))
output_path: Path = tmp_path / "per_token_single_spike.png"
result = generate_per_token_heatmap(records=records, output_path=output_path)

138
test/registered/debug_utils/comparator/test_model_validation.py
View File

@@ -4,6 +4,12 @@ import sys
import pytest
from pydantic import ValidationError
from sglang.srt.debug_utils.comparator.aligner.entrypoint.traced_types import (
TracedAlignerPlan,
TracedSidePlan,
TracedStepPlan,
TracedSubPlan,
)
from sglang.srt.debug_utils.comparator.aligner.entrypoint.types import (
AlignerPerStepPlan,
AlignerPlan,
@@ -22,11 +28,12 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import (
)
from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis, TokenLayout
from sglang.srt.debug_utils.comparator.output_types import (
ComparisonErrorRecord,
ComparisonNonTensorRecord,
ComparisonSkipRecord,
ComparisonTensorRecord,
ErrorLog,
NonTensorComparisonRecord,
SkipComparisonRecord,
SummaryRecord,
TensorComparisonRecord,
parse_record_json,
)
from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
@@ -156,6 +163,14 @@ class TestSummaryRecord:
with pytest.raises(ValidationError, match="total=10"):
SummaryRecord(total=10, passed=5, failed=2, skipped=1)
def test_valid_with_errored(self):
record = SummaryRecord(total=10, passed=6, failed=2, skipped=1, errored=1)
assert record.errored == 1
def test_total_mismatch_with_errored(self):
with pytest.raises(ValidationError, match="total=10"):
SummaryRecord(total=10, passed=6, failed=2, skipped=1, errored=0)
class TestAxisInfo:
def test_valid(self):
@@ -208,9 +223,9 @@ def _make_comparison_record(
*,
diff: DiffInfo | None,
errors: list | None = None,
) -> TensorComparisonRecord:
) -> ComparisonTensorRecord:
ti: TensorInfo = _make_tensor_info()
return TensorComparisonRecord(
return ComparisonTensorRecord(
name="t",
baseline=ti,
target=ti,
@@ -223,7 +238,7 @@ def _make_comparison_record(
class TestOutputRecordCategories:
def test_skip_record_with_errors_is_failed(self) -> None:
record = SkipComparisonRecord(
record = ComparisonSkipRecord(
name="t",
reason="test",
errors=[ErrorLog(category="c", message="m")],
@@ -231,28 +246,28 @@ class TestOutputRecordCategories:
assert record.category == "failed"
def test_skip_record_no_warnings_is_skipped(self) -> None:
record = SkipComparisonRecord(name="t", reason="test")
record = ComparisonSkipRecord(name="t", reason="test")
assert record.category == "skipped"
def test_comparison_record_diff_none_is_failed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(diff=None)
record: ComparisonTensorRecord = _make_comparison_record(diff=None)
assert record.category == "failed"
def test_comparison_record_passed_with_errors_is_failed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff_info(passed=True),
errors=[ErrorLog(category="c", message="m")],
)
assert record.category == "failed"
def test_comparison_record_passed_no_warnings_is_passed(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff_info(passed=True),
)
assert record.category == "passed"
def test_non_tensor_record_equal_is_passed(self) -> None:
record = NonTensorComparisonRecord(
record = ComparisonNonTensorRecord(
name="sm_scale",
baseline_value="0.125",
target_value="0.125",
@@ -263,7 +278,7 @@ class TestOutputRecordCategories:
assert record.category == "passed"
def test_non_tensor_record_different_is_failed(self) -> None:
record = NonTensorComparisonRecord(
record = ComparisonNonTensorRecord(
name="sm_scale",
baseline_value="0.125",
target_value="0.25",
@@ -274,7 +289,7 @@ class TestOutputRecordCategories:
assert record.category == "failed"
def test_non_tensor_record_with_errors_is_failed(self) -> None:
record = NonTensorComparisonRecord(
record = ComparisonNonTensorRecord(
name="sm_scale",
baseline_value="0.125",
target_value="0.125",
@@ -286,7 +301,7 @@ class TestOutputRecordCategories:
assert record.category == "failed"
def test_non_tensor_record_json_roundtrip(self) -> None:
record = NonTensorComparisonRecord(
record = ComparisonNonTensorRecord(
name="sm_scale",
baseline_value="0.125",
target_value="0.25",
@@ -296,14 +311,14 @@ class TestOutputRecordCategories:
)
json_str: str = record.model_dump_json()
roundtripped = parse_record_json(json_str)
assert isinstance(roundtripped, NonTensorComparisonRecord)
assert isinstance(roundtripped, ComparisonNonTensorRecord)
assert roundtripped.name == "sm_scale"
assert roundtripped.values_equal is False
assert roundtripped.baseline_value == "0.125"
assert roundtripped.target_value == "0.25"
def test_non_tensor_record_text_format_equal(self) -> None:
record = NonTensorComparisonRecord(
record = ComparisonNonTensorRecord(
name="sm_scale",
baseline_value="0.125",
target_value="0.125",
@@ -316,7 +331,7 @@ class TestOutputRecordCategories:
assert "[equal]" in text
def test_non_tensor_record_text_format_different(self) -> None:
record = NonTensorComparisonRecord(
record = ComparisonNonTensorRecord(
name="sm_scale",
baseline_value="0.125",
target_value="0.25",
@@ -328,14 +343,38 @@ class TestOutputRecordCategories:
assert "baseline" in text
assert "target" in text
def test_error_record_category_is_errored(self) -> None:
record = ComparisonErrorRecord(
name="t", exception_type="ValueError", traceback_str="..."
)
assert record.category == "errored"
def _make_aligner_plan() -> AlignerPlan:
def test_error_record_json_roundtrip(self) -> None:
record = ComparisonErrorRecord(
name="t", exception_type="ValueError", traceback_str="traceback..."
)
json_str: str = record.model_dump_json()
roundtripped = parse_record_json(json_str)
assert isinstance(roundtripped, ComparisonErrorRecord)
assert roundtripped.name == "t"
assert roundtripped.exception_type == "ValueError"
def test_error_record_text_format(self) -> None:
record = ComparisonErrorRecord(
name="t", exception_type="RuntimeError", traceback_str="Traceback..."
)
text: str = record.to_text()
assert "RuntimeError" in text
assert "Traceback" in text
def _make_traced_aligner_plan() -> TracedAlignerPlan:
unsharder = UnsharderPlan(
axis=ParallelAxis.TP,
params=ConcatParams(dim_name="h"),
groups=[[0, 1]],
)
return AlignerPlan(
plan = AlignerPlan(
per_step_plans=Pair(
x=[
AlignerPerStepPlan(
@@ -349,54 +388,67 @@ def _make_aligner_plan() -> AlignerPlan:
],
),
)
traced_sub = TracedSubPlan(plan=unsharder, snapshot=None)
traced_step = TracedStepPlan(
step=0, input_object_indices=[0, 1], sub_plans=[traced_sub]
)
return TracedAlignerPlan(
plan=plan,
per_side=Pair(
x=TracedSidePlan(step_plans=[traced_step]),
y=TracedSidePlan(step_plans=[traced_step]),
),
)
class TestAlignerPlanInTensorComparisonRecord:
def test_comparison_record_with_aligner_plan(self) -> None:
plan: AlignerPlan = _make_aligner_plan()
record: TensorComparisonRecord = _make_comparison_record(
class TestAlignerPlanInComparisonTensorRecord:
def test_comparison_record_with_traced_plan(self) -> None:
traced_plan: TracedAlignerPlan = _make_traced_aligner_plan()
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff_info(passed=True),
)
record_with_plan = record.model_copy(update={"aligner_plan": plan})
assert record_with_plan.aligner_plan is not None
assert record_with_plan.aligner_plan.per_step_plans.x[0].step == 0
record_with_plan = record.model_copy(update={"traced_plan": traced_plan})
assert record_with_plan.traced_plan is not None
assert record_with_plan.traced_plan.per_side.x.step_plans[0].step == 0
def test_aligner_plan_json_roundtrip(self) -> None:
plan: AlignerPlan = _make_aligner_plan()
record: TensorComparisonRecord = _make_comparison_record(
def test_traced_plan_json_roundtrip(self) -> None:
traced_plan: TracedAlignerPlan = _make_traced_aligner_plan()
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff_info(passed=True),
)
record_with_plan = record.model_copy(update={"aligner_plan": plan})
record_with_plan = record.model_copy(update={"traced_plan": traced_plan})
json_str: str = record_with_plan.model_dump_json()
parsed = json.loads(json_str)
assert "aligner_plan" in parsed
assert "traced_plan" in parsed
assert (
parsed["aligner_plan"]["per_step_plans"]["x"][0]["sub_plans"][0]["type"]
parsed["traced_plan"]["per_side"]["x"]["step_plans"][0]["sub_plans"][0][
"plan"
]["type"]
== "unsharder"
)
roundtripped: TensorComparisonRecord = parse_record_json(json_str)
assert roundtripped.aligner_plan is not None
roundtripped: ComparisonTensorRecord = parse_record_json(json_str)
assert roundtripped.traced_plan is not None
assert (
roundtripped.aligner_plan.per_step_plans.x[0].sub_plans[0].type
roundtripped.traced_plan.per_side.x.step_plans[0].sub_plans[0].plan.type
== "unsharder"
)
def test_comparison_record_without_aligner_plan(self) -> None:
record: TensorComparisonRecord = _make_comparison_record(
def test_comparison_record_without_traced_plan(self) -> None:
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff_info(passed=True),
)
json_str: str = record.model_dump_json()
roundtripped: TensorComparisonRecord = parse_record_json(json_str)
assert roundtripped.aligner_plan is None
roundtripped: ComparisonTensorRecord = parse_record_json(json_str)
assert roundtripped.traced_plan is None
def test_aligner_plan_text_format(self) -> None:
plan: AlignerPlan = _make_aligner_plan()
record: TensorComparisonRecord = _make_comparison_record(
def test_traced_plan_text_format(self) -> None:
traced_plan: TracedAlignerPlan = _make_traced_aligner_plan()
record: ComparisonTensorRecord = _make_comparison_record(
diff=_make_diff_info(passed=True),
)
record_with_plan = record.model_copy(update={"aligner_plan": plan})
record_with_plan = record.model_copy(update={"traced_plan": traced_plan})
text: str = record_with_plan.to_text()
assert "Aligner Plan:" in text

52
test/registered/debug_utils/comparator/test_output_types.py
View File

@@ -34,15 +34,15 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import (
)
from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis, TokenLayout
from sglang.srt.debug_utils.comparator.output_types import (
ComparisonNonTensorRecord,
ComparisonSkipRecord,
ComparisonTensorRecord,
ConfigRecord,
ErrorLog,
InfoLog,
LogRecord,
NonTensorComparisonRecord,
RecordLocation,
SkipComparisonRecord,
SummaryRecord,
TensorComparisonRecord,
_format_aligner_plan,
_split_logs,
)
@@ -150,20 +150,20 @@ class TestConfigRecord:
# ---------------------------------------------------------------------------
# SkipComparisonRecord
# ComparisonSkipRecord
# ---------------------------------------------------------------------------
class TestSkipComparisonRecord:
class TestComparisonSkipRecord:
def test_format_body_no_step(self) -> None:
record: SkipComparisonRecord = SkipComparisonRecord(
record: ComparisonSkipRecord = ComparisonSkipRecord(
name="layer.weight",
reason="zero-dim tensor",
)
assert record._format_body() == "Skip: layer.weight (zero-dim tensor)"
def test_format_body_with_step(self) -> None:
record: SkipComparisonRecord = SkipComparisonRecord(
record: ComparisonSkipRecord = ComparisonSkipRecord(
name="layer.weight",
reason="scalar",
location=RecordLocation(step=3),
@@ -171,7 +171,7 @@ class TestSkipComparisonRecord:
assert record._format_body() == "Skip: layer.weight (step=3) (scalar)"
def test_format_rich_body(self) -> None:
record: SkipComparisonRecord = SkipComparisonRecord(
record: ComparisonSkipRecord = ComparisonSkipRecord(
name="attn.qkv",
reason="no baseline",
)
@@ -179,14 +179,14 @@ class TestSkipComparisonRecord:
assert body == "[dim]⊘ attn.qkv ── skipped (no baseline)[/]"
def test_category_skipped(self) -> None:
record: SkipComparisonRecord = SkipComparisonRecord(
record: ComparisonSkipRecord = ComparisonSkipRecord(
name="x",
reason="r",
)
assert record.category == "skipped"
def test_category_failed(self) -> None:
record: SkipComparisonRecord = SkipComparisonRecord(
record: ComparisonSkipRecord = ComparisonSkipRecord(
name="x",
reason="r",
errors=[ErrorLog(category="e", message="boom")],
@@ -195,13 +195,13 @@ class TestSkipComparisonRecord:
# ---------------------------------------------------------------------------
# NonTensorComparisonRecord
# ComparisonNonTensorRecord
# ---------------------------------------------------------------------------
class TestNonTensorComparisonRecord:
class TestComparisonNonTensorRecord:
def test_format_body_equal(self) -> None:
record: NonTensorComparisonRecord = NonTensorComparisonRecord(
record: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="config.lr",
baseline_value="0.001",
target_value="0.001",
@@ -212,7 +212,7 @@ class TestNonTensorComparisonRecord:
assert record._format_body() == "NonTensor: config.lr = 0.001 (float) [equal]"
def test_format_body_not_equal(self) -> None:
record: NonTensorComparisonRecord = NonTensorComparisonRecord(
record: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="config.lr",
baseline_value="0.001",
target_value="0.01",
@@ -227,7 +227,7 @@ class TestNonTensorComparisonRecord:
)
def test_format_rich_body_equal(self) -> None:
record: NonTensorComparisonRecord = NonTensorComparisonRecord(
record: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="config.lr",
baseline_value="0.001",
target_value="0.001",
@@ -238,7 +238,7 @@ class TestNonTensorComparisonRecord:
assert record._format_rich_body() == ("═ config.lr = 0.001 (float) [green]✓[/]")
def test_format_rich_body_not_equal(self) -> None:
record: NonTensorComparisonRecord = NonTensorComparisonRecord(
record: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="config.lr",
baseline_value="0.001",
target_value="0.01",
@@ -253,7 +253,7 @@ class TestNonTensorComparisonRecord:
)
def test_with_step(self) -> None:
record: NonTensorComparisonRecord = NonTensorComparisonRecord(
record: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="bias",
baseline_value="True",
target_value="True",
@@ -265,7 +265,7 @@ class TestNonTensorComparisonRecord:
assert "(step=5)" in record._format_body()
def test_category(self) -> None:
passed: NonTensorComparisonRecord = NonTensorComparisonRecord(
passed: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="x",
baseline_value="1",
target_value="1",
@@ -273,7 +273,7 @@ class TestNonTensorComparisonRecord:
target_type="int",
values_equal=True,
)
failed: NonTensorComparisonRecord = NonTensorComparisonRecord(
failed: ComparisonNonTensorRecord = ComparisonNonTensorRecord(
name="x",
baseline_value="1",
target_value="2",
@@ -328,13 +328,13 @@ class TestSummaryRecord:
# ---------------------------------------------------------------------------
# TensorComparisonRecord._format_body
# ComparisonTensorRecord._format_body
# ---------------------------------------------------------------------------
class TestTensorComparisonRecordFormatBody:
class TestComparisonTensorRecordFormatBody:
def test_basic(self) -> None:
record: TensorComparisonRecord = TensorComparisonRecord(
record: ComparisonTensorRecord = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -365,7 +365,7 @@ class TestTensorComparisonRecordFormatBody:
def test_with_replicated_checks(self) -> None:
from sglang.srt.debug_utils.comparator.output_types import ReplicatedCheckResult
record: TensorComparisonRecord = TensorComparisonRecord(
record: ComparisonTensorRecord = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -420,7 +420,7 @@ class TestTensorComparisonRecordFormatBody:
y=TracedSidePlan(step_plans=[]),
),
)
record: TensorComparisonRecord = TensorComparisonRecord(
record: ComparisonTensorRecord = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -453,7 +453,7 @@ class TestTensorComparisonRecordFormatBody:
)
def test_with_step(self) -> None:
record: TensorComparisonRecord = TensorComparisonRecord(
record: ComparisonTensorRecord = ComparisonTensorRecord(
name="hidden",
baseline=_make_tensor_info(),
target=_make_tensor_info(),
@@ -679,7 +679,7 @@ class TestOutputRecordLogAttachment:
assert text == "Config: {'a': 1}\n ✗ err1\n ℹ note1"
def test_to_rich_string_body(self) -> None:
record: SkipComparisonRecord = SkipComparisonRecord(
record: ComparisonSkipRecord = ComparisonSkipRecord(
name="x",
reason="r",
errors=[ErrorLog(category="e", message="oops")],

14
test/registered/debug_utils/comparator/test_per_token_visualizer.py
View File

@@ -9,7 +9,7 @@ from pathlib import Path
import pytest
import torch
from sglang.srt.debug_utils.comparator.output_types import TensorComparisonRecord
from sglang.srt.debug_utils.comparator.output_types import ComparisonTensorRecord
from sglang.srt.debug_utils.comparator.tensor_comparator.comparator import (
compare_tensor_pair,
)
@@ -31,8 +31,8 @@ def _make_comparison_record(
baseline: torch.Tensor,
target: torch.Tensor,
seq_dim: int = 0,
) -> TensorComparisonRecord:
"""Build a TensorComparisonRecord with per-token data from raw tensors."""
) -> ComparisonTensorRecord:
"""Build a ComparisonTensorRecord with per-token data from raw tensors."""
info = compare_tensor_pair(
x_baseline=baseline,
x_target=target,
@@ -40,7 +40,7 @@ def _make_comparison_record(
diff_threshold=1e-3,
seq_dim=seq_dim,
)
return TensorComparisonRecord(**info.model_dump())
return ComparisonTensorRecord(**info.model_dump())
class TestPerTokenVisualizer:
@@ -68,7 +68,7 @@ class TestPerTokenVisualizer:
name="no_per_token",
diff_threshold=1e-3,
)
record = TensorComparisonRecord(**info.model_dump())
record = ComparisonTensorRecord(**info.model_dump())
output_path: Path = tmp_path / "no_data.png"
result = generate_per_token_heatmap(records=[record], output_path=output_path)
@@ -82,7 +82,7 @@ class TestPerTokenVisualizer:
)
torch.manual_seed(42)
records: list[TensorComparisonRecord] = [
records: list[ComparisonTensorRecord] = [
_make_comparison_record(
name=f"tensor_{i}",
baseline=torch.randn(16, 32),
@@ -108,7 +108,7 @@ class TestPerTokenVisualizer:
)
torch.manual_seed(42)
records: list[TensorComparisonRecord] = [
records: list[ComparisonTensorRecord] = [
_make_comparison_record(
name="short",
baseline=torch.randn(4, 8),

30
test/registered/debug_utils/comparator/test_utils.py
View File

@@ -410,6 +410,36 @@ class TestComputeExitCode:
== 1
)
def test_errored_with_passed_exits_one(self):
"""Has errored bundle even with passed → exit 1."""
summary = SummaryRecord(total=3, passed=2, failed=0, skipped=0, errored=1)
assert (
compute_exit_code(
summary,
allow_skipped_pattern=".*",
skipped_names=[],
allow_failed_pattern=None,
failed_names=[],
errored_names=["broken_tensor"],
)
== 1
)
def test_errored_only_exits_one(self):
"""All errored → exit 1 (passed==0 already exits 1, but errored also independently triggers)."""
summary = SummaryRecord(total=1, passed=0, failed=0, skipped=0, errored=1)
assert (
compute_exit_code(
summary,
allow_skipped_pattern=".*",
skipped_names=[],
allow_failed_pattern=None,
failed_names=[],
errored_names=["broken_tensor"],
)
== 1
)
def _make_pt(directory: Path) -> None:
directory.mkdir(parents=True, exist_ok=True)

6
test/registered/debug_utils/comparator/testing_helpers.py
View File

@@ -4,6 +4,12 @@ from __future__ import annotations
from typing import Optional
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(
est_time=0, suite="default", nightly=True, disabled="helper module, no tests"
)
from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
DiffInfo,
TensorInfo,