Support sequence parallel and trivial dims in dump comparator (#19460)

This commit is contained in:
fzyzcjy
2026-02-27 08:11:15 +08:00
committed by GitHub
parent eb2ada3804
commit f2d1b7c4ad
4 changed files with 58 additions and 3 deletions

View File

@@ -31,9 +31,15 @@ def compute_unsharder_plan(
sharded_axis_infos: dict[ParallelAxis, DimSpec] = {
spec.parallel: spec for spec in dim_specs if spec.parallel is not None
}
sharded_axes: set[ParallelAxis] = set(sharded_axis_infos)
sharded_axes_raw: set[ParallelAxis] = set(sharded_axis_infos)
all_axes: set[ParallelAxis] = {axis for info in parallel_infos for axis in info}
# axis annotated in dims but absent from all parallel_infos -> axis_size=1, skip
sharded_axes: set[ParallelAxis] = sharded_axes_raw & all_axes
sharded_axis_infos = {
k: v for k, v in sharded_axis_infos.items() if k in sharded_axes
}
replicated_axes: set[ParallelAxis] = all_axes - sharded_axes
if not sharded_axes and not replicated_axes:

View File

@@ -1238,6 +1238,18 @@ class _MegatronPlugin(_FrameworkPlugin):
except (AttributeError, AssertionError):
info["megatron_error"] = True
# Megatron sequence parallel reuses the TP group (no dedicated parallel state API).
# When sequence_parallel=True, inject sp_rank/sp_size for the comparator unsharder.
try:
from megatron.training.global_vars import get_args
args = get_args()
if getattr(args, "sequence_parallel", False) and "tp_rank" in info:
info["sp_rank"] = info["tp_rank"]
info["sp_size"] = info["tp_size"]
except (ImportError, AssertionError, AttributeError):
pass
return info
def convert_value(