chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[BUGFIX] fix radix cache memory consumption to avoid OOM (#17191)

Browse Source 
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
Yi Zhang
2026-01-17 16:47:37 +08:00
committed by GitHub
parent dc743fe4ba
commit 737a1183d6
5 changed files with 45 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/mem_cache/hiradix_cache.py
View File

@@ -839,11 +839,11 @@ class HiRadixCache(RadixCache):
if child.evicted:
new_node.value = None
else:
new_node.value = child.value[:split_len]
child.value = child.value[split_len:]
new_node.value = child.value[:split_len].clone()
child.value = child.value[split_len:].clone()
if child.backuped:
new_node.host_value = child.host_value[:split_len]
child.host_value = child.host_value[split_len:]
new_node.host_value = child.host_value[:split_len].clone()
child.host_value = child.host_value[split_len:].clone()
new_node.hash_value, child.hash_value = split_node_hash_value(
child.hash_value, split_len, self.page_size

4
python/sglang/srt/mem_cache/mamba_radix_cache.py
View File

@@ -982,7 +982,7 @@ class MambaRadixCache(BasePrefixCache):
new_node.full_lock_ref = child.full_lock_ref
new_node.mamba_lock_ref = 0
new_node.key = child.key[:split_len]
new_node.value = child.value[:split_len]
new_node.value = child.value[:split_len].clone()
# child time should be later than parent's time for mamba tombstone
child.last_access_time = get_last_access_time()
@@ -992,7 +992,7 @@ class MambaRadixCache(BasePrefixCache):
self.mamba_lru_list.remove_node(child)
child.parent = new_node
child.key = child.key[split_len:]
child.value = child.value[split_len:]
child.value = child.value[split_len:].clone()
new_node.parent.children[self.get_child_key_fn(key)] = new_node
# insert the new node and child into the lru lists, insert

6
python/sglang/srt/mem_cache/radix_cache.py
View File

@@ -654,10 +654,10 @@ class RadixCache(BasePrefixCache):
new_node.parent = child.parent
new_node.lock_ref = child.lock_ref
new_node.key = child.key[:split_len]
new_node.value = child.value[:split_len]
new_node.value = child.value[:split_len].clone()
child.parent = new_node
child.key = child.key[split_len:]
child.value = child.value[split_len:]
child.value = child.value[split_len:].clone()
new_node.parent.children[self.get_child_key_fn(key)] = new_node
# Split hash_value if it was already computed, otherwise leave as None
@@ -703,7 +703,7 @@ class RadixCache(BasePrefixCache):
new_node = TreeNode(priority=priority)
new_node.parent = node
new_node.key = key
new_node.value = value
new_node.value = value.clone()
node.children[child_key] = new_node
self.evictable_size_ += len(key)
# Hash will be computed lazily during event emission

4
python/sglang/srt/mem_cache/swa_radix_cache.py
View File

@@ -878,7 +878,7 @@ class SWARadixCache(BasePrefixCache):
new_node.full_lock_ref = child.full_lock_ref
new_node.swa_lock_ref = child.swa_lock_ref
new_node.key = child.key[:split_len]
new_node.value = child.value[:split_len]
new_node.value = child.value[:split_len].clone()
# parent inherits the swa_uuid from child for swa lock ref
new_node.swa_uuid = child.swa_uuid
child.swa_uuid = None
@@ -891,7 +891,7 @@ class SWARadixCache(BasePrefixCache):
self.swa_lru_list.remove_node(child)
child.parent = new_node
child.key = child.key[split_len:]
child.value = child.value[split_len:]
child.value = child.value[split_len:].clone()
new_node.parent.children[self.get_child_key_fn(key)] = new_node
# insert the new node and child into the lru lists, insert