Protect CP shared-KV cache-hit correctness under batched FP8 reuse

Cache-hit GSM8K regressions only appeared after the second pass reused request-specific suffix pages, so this change adds fail-fast transfer validation, masks stale rectangular page-table tails, and extends CUDA/unit coverage across FP8 CP shared-KV write, load, top-k, and materialization paths. The temporary ledger records eliminated hypotheses to prevent re-debugging the same L2 and persistent-cache paths.\n\nConstraint: CP shared KV stores physical pages but scheduler-visible semantics must remain valid-token/page-bounded.\nConstraint: bs>1 FP8 prefill must preserve existing CP shared-KV fast paths without silent fallback.\nRejected: Blame raw HiCache L2 load without tests | L2 KV and index backup/load/materialize roundtrips pass on remote CUDA.\nRejected: Disable current/partial reuse broadly | hides the cache-hit contract regression and costs performance.\nConfidence: medium\nScope-risk: moderate\nDirective: Do not weaken CP shared-KV fail-fast or rectangular-tail masking without rerunning second-pass cache-hit accuracy tests.\nTested: remote CUDA pytest for fused FP8 MLA store, fused persistent index store, L2-loaded FP8 KV materialize, L2-loaded index materialize, ragged top-k offset, TAI batched index MQA prepare.\nTested: local py_compile for touched test files and git diff --check.\nNot-tested: full second-pass GSM8K accuracy after these diagnostic tests; root cause remains under investigation.

python/sglang/srt/disaggregation/prefill.py

@@ -524,6 +524,8 @@ class PrefillBootstrapQueue:
             [req.disagg_kv_sender for req in self.queue],
             self.scheduler.attn_cp_cpu_group,
             self.scheduler.attn_tp_cpu_group,
+            debug_label="bootstrap",
+            debug_ids=[req.rid for req in self.queue],
         )
         _cp_shared_kv_bs_gt1_prefill_timing(
             "bootstrap_poll_done",
@@ -937,6 +939,8 @@ class SchedulerDisaggregationPrefillMixin:
             [req.disagg_kv_sender for req in self.disagg_prefill_inflight_queue],
             self.attn_cp_cpu_group,
             self.attn_tp_cpu_group,
+            debug_label="inflight",
+            debug_ids=[req.rid for req in self.disagg_prefill_inflight_queue],
         )
         _cp_shared_kv_bs_gt1_prefill_timing(
             "inflight_poll_done",
@@ -1050,6 +1054,8 @@ class SchedulerDisaggregationPrefillMixin:
             [req.disagg_kv_sender for req in self.disagg_prefill_inflight_queue],
             self.attn_cp_cpu_group,
             self.attn_tp_cpu_group,
+            debug_label="get_transferred_rids",
+            debug_ids=[req.rid for req in self.disagg_prefill_inflight_queue],
         )
 
         transferred_rids: List[str] = []

python/sglang/srt/disaggregation/utils.py

@@ -63,11 +63,85 @@ def poll_and_all_reduce(pollers, gloo_group: dist.ProcessGroup):
     return tensor_to_reduce.tolist()
 
 
+def _cp_shared_kv_poll_debug_enabled() -> bool:
+    return envs.SGLANG_CP_SHARED_KV_BS_GT1_DEBUG.get()
+
+
+def _poll_queue_debug_hash(debug_ids: Optional[list[str]]) -> int:
+    if not debug_ids:
+        return 0
+
+    # Stable bounded FNV-1a style digest.  Keep it in signed-int64 range because
+    # the debug consensus uses torch.int64 CPU collectives.
+    value = 1469598103934665603
+    mask = (1 << 63) - 1
+    for item in debug_ids:
+        for byte in str(item).encode("utf-8", errors="replace"):
+            value ^= byte
+            value = (value * 1099511628211) & mask
+        value ^= 0xFF
+        value = (value * 1099511628211) & mask
+    return int(value)
+
+
+def _validate_poll_queue_consensus(
+    *,
+    label: str,
+    scope: str,
+    local_len: int,
+    debug_ids: Optional[list[str]],
+    group: dist.ProcessGroup,
+) -> None:
+    local_hash = _poll_queue_debug_hash(debug_ids)
+    signature = torch.tensor([local_len, local_hash], dtype=torch.int64, device="cpu")
+    min_signature = signature.clone()
+    max_signature = signature.clone()
+
+    dist.all_reduce(min_signature, op=dist.ReduceOp.MIN, group=group)
+    dist.all_reduce(max_signature, op=dist.ReduceOp.MAX, group=group)
+
+    if torch.equal(min_signature, max_signature):
+        return
+
+    ids_head = list(debug_ids[:8]) if debug_ids is not None else None
+    message = (
+        "[CP_SHARED_KV_FAIL_FAST][poll_queue] "
+        f"label={label} scope={scope} local_len={local_len} "
+        f"min_len={int(min_signature[0].item())} "
+        f"max_len={int(max_signature[0].item())} "
+        f"local_hash={local_hash} min_hash={int(min_signature[1].item())} "
+        f"max_hash={int(max_signature[1].item())} ids_head={ids_head}"
+    )
+    logger.error(message)
+    raise RuntimeError(message)
+
+
 def poll_and_all_reduce_attn_cp_tp_group(
     pollers,
     attn_cp_cpu_group: dist.ProcessGroup,
     attn_tp_cpu_group: dist.ProcessGroup,
+    *,
+    debug_label: Optional[str] = None,
+    debug_ids: Optional[list[str]] = None,
 ):
+    if _cp_shared_kv_poll_debug_enabled():
+        label = debug_label or "unknown"
+        local_len = len(pollers)
+        _validate_poll_queue_consensus(
+            label=label,
+            scope="attn_tp",
+            local_len=local_len,
+            debug_ids=debug_ids,
+            group=attn_tp_cpu_group,
+        )
+        _validate_poll_queue_consensus(
+            label=label,
+            scope="attn_cp",
+            local_len=local_len,
+            debug_ids=debug_ids,
+            group=attn_cp_cpu_group,
+        )
+
     # First sync across attn-tp ranks so all TP participants for a given (dp, cp)
     # shard observe the same status transitions.
     polls = poll_and_all_reduce(pollers, attn_tp_cpu_group)