Unify memory management across (overlap, non-overlap) x (page>=1) x (spec, non-spec, spec v2) x (retract, finished) (#12224)

This commit is contained in:
Liangsheng Yin
2025-11-11 02:56:22 +08:00
committed by GitHub
parent 838bcb0d93
commit 665416f6dd
24 changed files with 193 additions and 156 deletions
+14 -29
View File
@@ -51,6 +51,7 @@ from sglang.srt.managers.schedule_batch import FINISH_ABORT, RequestStage, Sched
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.common import release_kv_cache
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
HybridReqToTokenPool,
@@ -618,37 +619,21 @@ class DecodePreallocQueue:
req.req_pool_idx = req_pool_indices[0]
# Alloc all tokens for the prebuilt req (except for the reserved input token for decoding)
fill_len = len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0)
req.kv_allocated_len = fill_len
req.kv_committed_len = fill_len
if self.token_to_kv_pool_allocator.page_size == 1:
kv_loc = self.token_to_kv_pool_allocator.alloc(
len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0)
)
kv_loc = self.token_to_kv_pool_allocator.alloc(fill_len)
else:
num_tokens = len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0)
device = self.token_to_kv_pool_allocator.device
kv_loc = self.token_to_kv_pool_allocator.alloc_extend(
prefix_lens=torch.tensor(
[0],
dtype=torch.int64,
device=self.token_to_kv_pool_allocator.device,
),
prefix_lens_cpu=torch.tensor(
[0],
dtype=torch.int64,
),
seq_lens=torch.tensor(
[num_tokens],
dtype=torch.int64,
device=self.token_to_kv_pool_allocator.device,
),
seq_lens_cpu=torch.tensor(
[num_tokens],
dtype=torch.int64,
),
last_loc=torch.tensor(
[-1],
dtype=torch.int64,
device=self.token_to_kv_pool_allocator.device,
),
extend_num_tokens=num_tokens,
prefix_lens=torch.tensor([0], dtype=torch.int64, device=device),
prefix_lens_cpu=torch.tensor([0], dtype=torch.int64),
seq_lens=torch.tensor([fill_len], dtype=torch.int64, device=device),
seq_lens_cpu=torch.tensor([fill_len], dtype=torch.int64),
last_loc=torch.tensor([-1], dtype=torch.int64, device=device),
extend_num_tokens=fill_len,
)
assert (
@@ -763,7 +748,7 @@ class DecodeTransferQueue:
[decode_req.req], decode_req.req.return_logprob
)
# release pre-allocated kv cache, but don't insert into the tree since it's failed
self.tree_cache.cache_finished_req(decode_req.req, is_insert=False)
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
indices_to_remove.add(i)
if self.scheduler.enable_metrics:
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
@@ -1,6 +1,9 @@
from __future__ import annotations
import logging
import threading
import time
from typing import TYPE_CHECKING
import torch
@@ -18,6 +21,9 @@ from sglang.srt.mem_cache.memory_pool_host import (
)
from sglang.srt.server_args import ServerArgs
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
logger = logging.getLogger(__name__)
@@ -177,14 +183,14 @@ class DecodeKVCacheOffloadManager:
)
finish_count -= 1
def _release_finished_req(self, req, prefill_offloaded_len):
def _release_finished_req(self, req: Req, prefill_offloaded_len: int):
# FIXME: not sure which length to use here: kv_allocated_len or kv_committed_len
kv_indices = self.req_to_token_pool.req_to_token[
req.req_pool_idx,
: len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0),
req.req_pool_idx, prefill_offloaded_len : req.kv_allocated_len
]
# Free the incremental part of the request
self.token_to_kv_pool_allocator.free(kv_indices[prefill_offloaded_len:])
self.token_to_kv_pool_allocator.free(kv_indices)
self.req_to_token_pool.free(req.req_pool_idx)
def _check_backup_progress(self, finish_count):
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
import torch
from sglang.srt.disaggregation.utils import prepare_abort
from sglang.srt.mem_cache.common import release_kv_cache
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
@@ -119,7 +120,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
# Grammar accept_token can raise ValueError if the token is not in the grammar.
# This can happen if the grammar is not set correctly or the token is invalid.
error_message = f"Grammar accept_token failed for req {req.rid} with token {req.output_ids[-1]}: {e}"
self.tree_cache.cache_finished_req(req)
release_kv_cache(req, self.tree_cache)
prepare_abort(
req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR
)
+4 -3
View File
@@ -48,6 +48,7 @@ from sglang.srt.managers.schedule_batch import (
RequestStage,
ScheduleBatch,
)
from sglang.srt.mem_cache.common import release_kv_cache
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
NSATokenToKVPool,
@@ -468,7 +469,7 @@ class SchedulerDisaggregationPrefillMixin:
# Grammar accept_token can raise ValueError if the token is not in the grammar.
# This can happen if the grammar is not set correctly or the token is invalid.
error_message = f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
self.tree_cache.cache_finished_req(req)
release_kv_cache(req, self.tree_cache)
prepare_abort(
req,
error_message,
@@ -534,7 +535,7 @@ class SchedulerDisaggregationPrefillMixin:
if poll in [KVPoll.WaitingForInput, KVPoll.Transferring]:
undone_reqs.append(req)
elif poll == KVPoll.Success: # transfer done
self.tree_cache.cache_finished_req(req) # unlock the tree
release_kv_cache(req, self.tree_cache) # unlock the tree
req.finished_reason = FINISH_LENGTH(length=0)
# FIXME: clean up req's data in transfer engine
if hasattr(req.disagg_kv_sender, "clear"):
@@ -547,7 +548,7 @@ class SchedulerDisaggregationPrefillMixin:
except Exception as e:
error_message += f" with exception {e}"
logger.warning(error_message)
self.tree_cache.cache_finished_req(req) # unlock the tree
release_kv_cache(req, self.tree_cache) # unlock the tree
prepare_abort(
req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR
)