From 1d54b0b7abf87a0b22ec8b98e65e6f01f4a50881 Mon Sep 17 00:00:00 2001 From: leavelet Date: Wed, 17 Jun 2026 04:52:17 +0000 Subject: [PATCH] CP HiCache trace: reload per-page owner-check (the make-or-break invariant) Two metadata agents verified that all cache-hit-extend metadata (positions, page_table, cu_seqlens, topk offset, MoE row-selection) is correct and prefix-aware. The one reload-specific, unguarded thing is the per-page OWNER replay: load_cp builds selected_logical_locs = node_device_indices[owned_positions] and explicitly SKIPS re-validating that those pages are owned by this rank (perf; the allocator's owner assert is gated behind debug_mode). A per-page owner permutation (correct bytes, wrong logical page) passes the aggregate host-keyed value-hash but makes attention read the right bytes for the wrong positions -> reload-only repeated-token garbage. Add owner_check (level 1): at each reloaded node, owner_for_logical_pages( selected_logical_locs // page_size) must all == cp_rank; log bad_owner count + valid/padded/tail_pad. bad_owner>0 = the per-page owner/padding permutation = root cause, which the aggregate byte-hash cannot detect. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sglang/srt/managers/cache_controller.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 1b9cb50c9..407d4ce7c 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -1753,6 +1753,35 @@ class HiCacheController: phys=rng(physical_chunks[-1]), draft_host_idx=rng(draft_host_indices), ) + # MAKE-OR-BREAK reload invariant (the code above SKIPS this for + # perf): every page this rank reloads into must be owned by THIS + # rank, i.e. owner_for_logical_pages((loc)//page_size) == cp_rank. + # A nonzero bad_owner = alloc_pages_with_owners/padding produced a + # per-page owner permutation -> this rank loads its host bytes into + # pages owned by another rank -> attention reads garbage (the + # aggregate byte-hash can't see this). + try: + _lay = self.cp_shared_kv_layout + _ps = int(getattr(_lay, "page_size", getattr(self, "page_size", 0))) + if _ps > 0: + _pages = selected_logical_locs // _ps + _own = _lay.owner_for_logical_pages(_pages) + _bad = int((_own != _lay.cp_rank).sum().item()) + cptrace( + 1, + "owner_check", + node_id=getattr(node, "id", -1), + cprank=_lay.cp_rank, + owned=int(owned_positions.numel()), + bad_owner=_bad, + valid_len=valid_len, + padded_len=padded_len, + tail_pad=padded_len - valid_len, + pages=rng(_pages), + owners=rng(_own), + ) + except Exception: + pass record_stage("build_chunks") visible_device_indices = (