From ff4717c2fb48ed1e4c41648d1423658322a38a5e Mon Sep 17 00:00:00 2001 From: laoyao0822 Date: Sun, 7 Jun 2026 23:56:04 +0800 Subject: [PATCH] Keep L1 free-room eviction off the CP load-back hot path CP load-back already handles exact owner-lane deficits before admitting L2->L1 reloads. Treating free-room target misses as synchronous evict work adds avoidable latency to cache-hit requests even when the current load can fit. The load-back path now records the free-room miss as advisory and leaves proactive replenishment to non-critical eviction paths. Constraint: GSM8K/cache-hit runs are sensitive to CPU/control-path latency during L2->L1 reload. Rejected: Evict to free-room on every load-back | preserves room but puts synchronous eviction on the latency-sensitive cache-hit path. Confidence: medium Scope-risk: moderate Directive: Exact owner-lane deficits remain mandatory; free-room targets must not become blocking in load-back without ETE latency evidence. Tested: python -m py_compile on changed runtime files. Not-tested: ETE latency after this isolated change not rerun in this commit step. (cherry picked from commit 2f42acba0d748d2d747f0531963267d86c483cde) --- python/sglang/srt/mem_cache/hiradix_cache.py | 37 ++++---------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 78fac7e2a..8184f3804 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -3438,36 +3438,13 @@ class HiRadixCache(RadixCache): ) record_stage("owner_lane_evict") - if not any(v > 0 for v in load_back_plan.deficit_by_owner) and any( - v > 0 for v in load_back_plan.free_room_deficit_by_owner - ): - # Best-effort L1 free-room maintenance for CP load-back. Exact - # admission may still succeed with a nearly empty owner lane - # (common for one-page host hits that all map to owner 0), but - # letting those loads proceed without replenishing free room can - # starve the next extend batch. Free-room target misses are not - # fatal; exact deficits remain the strict condition below. - free_room_deficits = [ - int(v) for v in load_back_plan.free_room_deficit_by_owner - ] - free_room_pages = sum(max(0, v) for v in free_room_deficits) - if free_room_pages > 0: - evict_result = self.evict( - EvictParams( - num_tokens=free_room_pages * self.page_size, - owner_lane_deficits=free_room_deficits, - ) - ) - logger.debug( - "[HiCache-load] owner-lane free-room eviction before CP " - "load-back: node_id=%d free_room_deficit_by_owner=%s " - "num_tokens_evicted=%d", - last_hit_node.id, - free_room_deficits, - getattr(evict_result, "num_tokens_evicted", 0), - ) - load_back_plan = self._refresh_cp_load_back_plan(load_back_plan) - record_stage("owner_lane_free_room_evict") + if any(v > 0 for v in load_back_plan.free_room_deficit_by_owner): + # L1 free-room is an advisory signal for future extend batches, + # not a reason to put synchronous eviction on the CP load-back + # hot path when exact owner-lane admission already succeeds. + # Exact deficits are handled above; proactive/free-room refill + # should happen outside this latency-sensitive L2->L1 path. + record_stage("owner_lane_free_room_advisory") if any(v > 0 for v in load_back_plan.deficit_by_owner): self.dec_lock_ref(ancester_node)