diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 1b9cb50c9..407d4ce7c 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -1753,6 +1753,35 @@ class HiCacheController: phys=rng(physical_chunks[-1]), draft_host_idx=rng(draft_host_indices), ) + # MAKE-OR-BREAK reload invariant (the code above SKIPS this for + # perf): every page this rank reloads into must be owned by THIS + # rank, i.e. owner_for_logical_pages((loc)//page_size) == cp_rank. + # A nonzero bad_owner = alloc_pages_with_owners/padding produced a + # per-page owner permutation -> this rank loads its host bytes into + # pages owned by another rank -> attention reads garbage (the + # aggregate byte-hash can't see this). + try: + _lay = self.cp_shared_kv_layout + _ps = int(getattr(_lay, "page_size", getattr(self, "page_size", 0))) + if _ps > 0: + _pages = selected_logical_locs // _ps + _own = _lay.owner_for_logical_pages(_pages) + _bad = int((_own != _lay.cp_rank).sum().item()) + cptrace( + 1, + "owner_check", + node_id=getattr(node, "id", -1), + cprank=_lay.cp_rank, + owned=int(owned_positions.numel()), + bad_owner=_bad, + valid_len=valid_len, + padded_len=padded_len, + tail_pad=padded_len - valid_len, + pages=rng(_pages), + owners=rng(_own), + ) + except Exception: + pass record_stage("build_chunks") visible_device_indices = (