Unify memory management across (overlap, non-overlap) x (page>=1) x (spec, non-spec, spec v2) x (retract, finished) (#12224)
This commit is contained in:
@@ -51,6 +51,7 @@ from sglang.srt.managers.schedule_batch import FINISH_ABORT, RequestStage, Sched
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.common import release_kv_cache
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
HybridReqToTokenPool,
|
||||
@@ -618,37 +619,21 @@ class DecodePreallocQueue:
|
||||
|
||||
req.req_pool_idx = req_pool_indices[0]
|
||||
|
||||
# Alloc all tokens for the prebuilt req (except for the reserved input token for decoding)
|
||||
fill_len = len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0)
|
||||
req.kv_allocated_len = fill_len
|
||||
req.kv_committed_len = fill_len
|
||||
if self.token_to_kv_pool_allocator.page_size == 1:
|
||||
kv_loc = self.token_to_kv_pool_allocator.alloc(
|
||||
len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0)
|
||||
)
|
||||
kv_loc = self.token_to_kv_pool_allocator.alloc(fill_len)
|
||||
else:
|
||||
num_tokens = len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0)
|
||||
device = self.token_to_kv_pool_allocator.device
|
||||
kv_loc = self.token_to_kv_pool_allocator.alloc_extend(
|
||||
prefix_lens=torch.tensor(
|
||||
[0],
|
||||
dtype=torch.int64,
|
||||
device=self.token_to_kv_pool_allocator.device,
|
||||
),
|
||||
prefix_lens_cpu=torch.tensor(
|
||||
[0],
|
||||
dtype=torch.int64,
|
||||
),
|
||||
seq_lens=torch.tensor(
|
||||
[num_tokens],
|
||||
dtype=torch.int64,
|
||||
device=self.token_to_kv_pool_allocator.device,
|
||||
),
|
||||
seq_lens_cpu=torch.tensor(
|
||||
[num_tokens],
|
||||
dtype=torch.int64,
|
||||
),
|
||||
last_loc=torch.tensor(
|
||||
[-1],
|
||||
dtype=torch.int64,
|
||||
device=self.token_to_kv_pool_allocator.device,
|
||||
),
|
||||
extend_num_tokens=num_tokens,
|
||||
prefix_lens=torch.tensor([0], dtype=torch.int64, device=device),
|
||||
prefix_lens_cpu=torch.tensor([0], dtype=torch.int64),
|
||||
seq_lens=torch.tensor([fill_len], dtype=torch.int64, device=device),
|
||||
seq_lens_cpu=torch.tensor([fill_len], dtype=torch.int64),
|
||||
last_loc=torch.tensor([-1], dtype=torch.int64, device=device),
|
||||
extend_num_tokens=fill_len,
|
||||
)
|
||||
|
||||
assert (
|
||||
@@ -763,7 +748,7 @@ class DecodeTransferQueue:
|
||||
[decode_req.req], decode_req.req.return_logprob
|
||||
)
|
||||
# release pre-allocated kv cache, but don't insert into the tree since it's failed
|
||||
self.tree_cache.cache_finished_req(decode_req.req, is_insert=False)
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
indices_to_remove.add(i)
|
||||
if self.scheduler.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -18,6 +21,9 @@ from sglang.srt.mem_cache.memory_pool_host import (
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -177,14 +183,14 @@ class DecodeKVCacheOffloadManager:
|
||||
)
|
||||
finish_count -= 1
|
||||
|
||||
def _release_finished_req(self, req, prefill_offloaded_len):
|
||||
def _release_finished_req(self, req: Req, prefill_offloaded_len: int):
|
||||
# FIXME: not sure which length to use here: kv_allocated_len or kv_committed_len
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.req_pool_idx,
|
||||
: len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0),
|
||||
req.req_pool_idx, prefill_offloaded_len : req.kv_allocated_len
|
||||
]
|
||||
|
||||
# Free the incremental part of the request
|
||||
self.token_to_kv_pool_allocator.free(kv_indices[prefill_offloaded_len:])
|
||||
self.token_to_kv_pool_allocator.free(kv_indices)
|
||||
self.req_to_token_pool.free(req.req_pool_idx)
|
||||
|
||||
def _check_backup_progress(self, finish_count):
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
from sglang.srt.disaggregation.utils import prepare_abort
|
||||
from sglang.srt.mem_cache.common import release_kv_cache
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
|
||||
@@ -119,7 +120,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
# Grammar accept_token can raise ValueError if the token is not in the grammar.
|
||||
# This can happen if the grammar is not set correctly or the token is invalid.
|
||||
error_message = f"Grammar accept_token failed for req {req.rid} with token {req.output_ids[-1]}: {e}"
|
||||
self.tree_cache.cache_finished_req(req)
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
prepare_abort(
|
||||
req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
@@ -48,6 +48,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
RequestStage,
|
||||
ScheduleBatch,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import release_kv_cache
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
NSATokenToKVPool,
|
||||
@@ -468,7 +469,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# Grammar accept_token can raise ValueError if the token is not in the grammar.
|
||||
# This can happen if the grammar is not set correctly or the token is invalid.
|
||||
error_message = f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
|
||||
self.tree_cache.cache_finished_req(req)
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
prepare_abort(
|
||||
req,
|
||||
error_message,
|
||||
@@ -534,7 +535,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if poll in [KVPoll.WaitingForInput, KVPoll.Transferring]:
|
||||
undone_reqs.append(req)
|
||||
elif poll == KVPoll.Success: # transfer done
|
||||
self.tree_cache.cache_finished_req(req) # unlock the tree
|
||||
release_kv_cache(req, self.tree_cache) # unlock the tree
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
# FIXME: clean up req's data in transfer engine
|
||||
if hasattr(req.disagg_kv_sender, "clear"):
|
||||
@@ -547,7 +548,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
except Exception as e:
|
||||
error_message += f" with exception {e}"
|
||||
logger.warning(error_message)
|
||||
self.tree_cache.cache_finished_req(req) # unlock the tree
|
||||
release_kv_cache(req, self.tree_cache) # unlock the tree
|
||||
prepare_abort(
|
||||
req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user