diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 407d4ce7c..6fdb0ac32 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -1705,6 +1705,26 @@ class HiCacheController: node_device_indices = device_indices[offset : offset + padded_len] offset += padded_len visible_chunks.append(node_device_indices[:valid_len]) + if trace_enabled(1): + # CROSS-RANK invariant: the visible logical-loc sequence for this + # node MUST be byte-identical on every CP rank — the cross-rank + # gather reads token j from its owner rank at physical(loc_j), so + # if rank A and rank R disagree on loc_j, A reads R's slot for a + # DIFFERENT token (right bytes, wrong place). Emitted for EVERY + # node (incl. zero-owned, before the continue below) so the + # analyzer can diff this hash across ranks by node_id. Divergence + # here is produced by wall-clock-LRU device eviction giving ranks + # different free buckets in alloc_pages_with_owners. + _vis = node_device_indices[:valid_len] + cptrace( + 1, + "visible_locs_hash", + node_id=getattr(node, "id", -1), + cprank=self.cp_shared_kv_layout.cp_rank, + n=int(_vis.numel()), + locs_hash=khash(_vis), + locs=rng(_vis), + ) if self.has_draft_hicache: draft_host_indices = getattr( meta, "draft_host_indices", None diff --git a/python/sglang/srt/mem_cache/allocator.py b/python/sglang/srt/mem_cache/allocator.py index 1cdd7cd2b..b1fd6ea6e 100644 --- a/python/sglang/srt/mem_cache/allocator.py +++ b/python/sglang/srt/mem_cache/allocator.py @@ -1093,6 +1093,20 @@ class CPSharedPagedTokenToKVPoolAllocator(PagedTokenToKVPoolAllocator): page_size, dtype=torch.int64, device=self.device ).unsqueeze(0) out_indices = (base + offsets).reshape(-1) + if trace_enabled(1) and any(c > 0 for c in selected_release_counts): + # Tapped the DEFERRED-FREE (release) bucket because a lane's free + # bucket was exhausted — only under eviction pressure. A release page + # may still be read by an in-flight kernel (deferred free exists to + # prevent exactly this), so handing it to an H2D reload here is a + # use-after-free / aliasing risk the byte-round-trip hash can't see. + cptrace( + 1, + "release_draw", + cprank=self.cp_rank, + pages=len(page_compute_owners), + free_by_owner=selected_free_counts, + release_by_owner=selected_release_counts, + ) self._consume_owner_bucket_prefix( release=False, counts_by_owner=selected_free_counts ) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 464b6f13c..c8be24dd2 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -1310,6 +1310,25 @@ class HiRadixCache(RadixCache): ) -> CpLoadBackPlan: evict_start_time = time.perf_counter() eviction_plan = self._plan_cp_load_back_owner_lane_evictions(plan) + if trace_enabled(1): + # ROOT-CAUSE signal: device load-back eviction victim selection is + # keyed on wall-clock last_access_time (per-rank monotonic clock, NOT + # replicated across CP ranks — unlike host eviction which uses the + # deterministic (priority, node.id)). So ranks can pick DIFFERENT + # victims -> free DIFFERENT pages -> diverge alloc_pages_with_owners + # -> req_to_token disagrees across ranks. Log the victim id set per + # rank for this load op; the analyzer diffs it across ranks. A + # cross-rank mismatch confirms the divergence trigger. + cptrace( + 1, + "victim_set", + node_id=node_id, + cprank=self._cp_hicache_cp_rank(), + nvictims=len(eviction_plan.victims), + victims=sorted( + int(getattr(v, "id", -1)) for v in eviction_plan.victims + ), + ) plan_elapsed_ms = (time.perf_counter() - evict_start_time) * 1000.0 if plan_elapsed_ms >= 1000.0: logger.warning(