From e5982dcceb2b7f0a4b2be7695d3903dcacb49390 Mon Sep 17 00:00:00 2001 From: leavelet Date: Sat, 23 May 2026 10:20:54 +0000 Subject: [PATCH] fix: call _update_leaf_status in inc/dec_node_lock_ref to prevent phantom evictable nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit inc_node_lock_ref/dec_node_lock_ref adjusted evictable_size_ but did not call _update_leaf_status, so nodes could become "phantoms" — counted in evictable_size_ but missing from evictable_leaves. Under load with write-behind, this caused eviction to return 0 despite large evictable_size_, leading to OOM: Available tokens: 778624 (evictable_size=706880) evict_result=(num_tokens_evicted=0) The race: write-behind locks node X via inc_node_lock_ref (X stays in evictable_leaves). A request path touches X via inc_lock_ref, which calls _update_leaf_status and removes X from evictable_leaves. Request finishes, dec_lock_ref keeps X out (lock_ref still >0). Write ack calls dec_node_lock_ref dropping lock_ref to 0, but never calls _update_leaf_status — X is permanently lost. Co-Authored-By: Claude Opus 4.6 (1M context) --- python/sglang/srt/mem_cache/radix_cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 40e8bee43..f06ab7803 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -665,6 +665,7 @@ class RadixCache(BasePrefixCache): self.evictable_size_ -= len(node.key) self.protected_size_ += len(node.key) node.lock_ref += 1 + self._update_leaf_status(node) def dec_node_lock_ref(self, node: TreeNode): """Decrement lock_ref on a single node without walking to root. @@ -679,6 +680,7 @@ class RadixCache(BasePrefixCache): self.evictable_size_ += len(node.key) self.protected_size_ -= len(node.key) node.lock_ref -= 1 + self._update_leaf_status(node) def evictable_size(self): return self.evictable_size_