From d3c0f4376a12e3c51836902f7fed02be60e1bf90 Mon Sep 17 00:00:00 2001 From: Du Bin Date: Mon, 16 Mar 2026 22:38:59 +0800 Subject: [PATCH] Fix AssertionError crash in disagg prefill inflight queue with PP (#20686) --- python/sglang/srt/disaggregation/prefill.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 381e92590..7d97f123b 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -569,7 +569,20 @@ class SchedulerDisaggregationPrefillMixin: undone_reqs.append(req) continue - assert poll == KVPoll.Success or poll == KVPoll.Failed + # In PP mode, the previous rank may have reached a terminal + # state (Success/Failed) while this rank's local poll is still + # in a transient state due to clock skew or propagation delay. + # Treat non-terminal states as undone instead of crashing. + if poll not in ( + KVPoll.Success, + KVPoll.Failed, + ): + logger.warning( + f"PP rank {self.pp_rank}: unexpected poll state {poll} for rid {req.rid} " + f"from consensus; treating as undone" + ) + undone_reqs.append(req) + continue if poll in [KVPoll.WaitingForInput, KVPoll.Transferring]: undone_reqs.append(req) @@ -597,7 +610,11 @@ class SchedulerDisaggregationPrefillMixin: if self.enable_metrics: self.metrics_collector.increment_transfer_failed_reqs() else: - assert False, f"Unexpected polling state {poll=}" + logger.warning( + f"Unexpected polling state {poll} for rid {req.rid} in inflight queue; " + f"treating as undone" + ) + undone_reqs.append(req) for req in done_reqs: req.time_stats.set_completion_time()