Bound CP prefill batching by estimated temp memory
CP shared-KV bs>1 batching was only bounded by request count, extend tokens, and cached tokens. That left temporary GPU buffers such as MLA/index materialization, remap metadata, logits windows, and transfer descriptors implicit, and raw extend-token limits could exceed the active chunked-prefill budget.\n\nThis adds an explicit max-buffer-size admission gate with a CPU-only stream-aware estimator, wires it through PrefillAdder/Scheduler, performs a startup CUDA smoke allocation when configured, and reports the estimate in the scheduler admission benchmark. When chunked prefill is active, the effective CP extend-token limit is capped by the current chunk budget so the CP path does not advertise unreachable batch capacity or lift max-prefill-tokens too far.\n\nConstraint: Admission estimation must stay CPU-only on the scheduler hot path; CUDA allocation is limited to startup smoke checking.\nConstraint: Single oversized requests must still be allowed to run alone to avoid scheduler deadlock.\nRejected: Rely only on --max-prefill-tokens | it does not reliably bound the first oversized request and does not model cache-hit/load-back pressure.\nRejected: Let CP extend limit exceed chunked-prefill size | it creates an unreachable effective capacity and misleading budget lift.\nConfidence: medium\nScope-risk: moderate\nDirective: If bs>1 L1 prefetch is enabled later, update CPSharedKVPrefillBufferEstimatorContext.bs_gt1_l1_prefetch_enabled and include the live prefetch dense buffers in overlap windows.\nTested: local py_compile for touched files\nTested: local PYTHONPATH=python pytest -q test/registered/unit/managers/test_cp_shared_kv_prefill_buffer_estimator.py (4 passed)\nTested: remote cjy-glm5-new targeted pytest for new server_args, PrefillAdder, estimator, and benchmark cases (10 passed)\nTested: remote cjy-glm5-new PYTHONPATH=python pytest -q test/registered/unit/managers/test_cp_shared_kv_prefill_buffer_estimator.py test/registered/unit/managers/test_prefill_adder.py test/registered/unit/managers/test_prefill_scheduler_admission_bench.py (29 passed before chunk cap, then test_prefill_adder.py 21 passed after chunk cap)\nNot-tested: full server_args suite because existing TestPrepareServerArgs tries to reach HuggingFace and fails under container DNS/network\nNot-tested: GLM5 ETE smoke with --cp-shared-kv-prefill-max-buffer-size
This commit is contained in:
@@ -143,6 +143,13 @@ class SchedulerBenchConfig:
|
||||
cp_shared_kv_prefill_max_batch_requests: Optional[int] = None
|
||||
cp_shared_kv_prefill_max_total_extend_tokens: Optional[int] = None
|
||||
cp_shared_kv_prefill_max_total_cached_tokens: Optional[int] = None
|
||||
cp_shared_kv_prefill_max_buffer_size: Optional[int] = None
|
||||
kv_cache_dim: int = 656
|
||||
kv_dtype_bytes: int = 1
|
||||
index_head_dim: int = 128
|
||||
vocab_size: int = 128_000
|
||||
tp_size: int = 8
|
||||
enable_bs_gt1_prefetch_estimate: bool = False
|
||||
max_ticks: int = 1
|
||||
consume_l2_load_back_capacity: bool = True
|
||||
|
||||
@@ -198,6 +205,8 @@ class TickResult:
|
||||
log_input_tokens: int
|
||||
allocator_available_after_tick: int
|
||||
load_back_events: list[LoadBackEvent]
|
||||
cp_estimated_peak_buffer_bytes: int
|
||||
cp_buffer_breakdown: dict[str, int]
|
||||
duration_us: float
|
||||
|
||||
|
||||
@@ -211,8 +220,15 @@ class TraceResult:
|
||||
|
||||
|
||||
class FakeTokenAllocator:
|
||||
def __init__(self, available_tokens: int):
|
||||
def __init__(self, available_tokens: int, cfg: SchedulerBenchConfig):
|
||||
self.available_tokens = int(available_tokens)
|
||||
self.kvcache = SimpleNamespace(
|
||||
kv_cache_dim=cfg.kv_cache_dim,
|
||||
store_dtype=SimpleNamespace(itemsize=cfg.kv_dtype_bytes),
|
||||
index_head_dim=cfg.index_head_dim,
|
||||
quant_block_size=128,
|
||||
index_k_with_scale_buffer_dtype=SimpleNamespace(itemsize=1),
|
||||
)
|
||||
|
||||
def available_size(self) -> int:
|
||||
return self.available_tokens
|
||||
@@ -231,6 +247,9 @@ class FakeTokenAllocator:
|
||||
self.available_tokens -= tokens
|
||||
return True
|
||||
|
||||
def get_kvcache(self):
|
||||
return self.kvcache
|
||||
|
||||
|
||||
class FakeTreeCache:
|
||||
def __init__(
|
||||
@@ -382,6 +401,9 @@ def _configure_scheduler_globals(enable_cp_context: bool) -> None:
|
||||
|
||||
def _make_prefill_adder(cfg: SchedulerBenchConfig, tree_cache: FakeTreeCache, allocator: FakeTokenAllocator):
|
||||
_install_sgl_kernel_stubs()
|
||||
from sglang.srt.managers.cp_shared_kv_prefill_buffer_estimator import (
|
||||
CPSharedKVPrefillBufferEstimatorContext,
|
||||
)
|
||||
from sglang.srt.managers.schedule_policy import PrefillAdder
|
||||
|
||||
return PrefillAdder(
|
||||
@@ -398,6 +420,18 @@ def _make_prefill_adder(cfg: SchedulerBenchConfig, tree_cache: FakeTreeCache, al
|
||||
cp_shared_kv_prefill_max_batch_requests=cfg.cp_shared_kv_prefill_max_batch_requests,
|
||||
cp_shared_kv_prefill_max_total_extend_tokens=cfg.cp_shared_kv_prefill_max_total_extend_tokens,
|
||||
cp_shared_kv_prefill_max_total_cached_tokens=cfg.cp_shared_kv_prefill_max_total_cached_tokens,
|
||||
cp_shared_kv_prefill_max_buffer_size=cfg.cp_shared_kv_prefill_max_buffer_size,
|
||||
cp_shared_kv_prefill_buffer_estimator_context=(
|
||||
CPSharedKVPrefillBufferEstimatorContext(
|
||||
kvcache=allocator.get_kvcache(),
|
||||
model_config=SimpleNamespace(vocab_size=cfg.vocab_size),
|
||||
tp_size=cfg.tp_size,
|
||||
page_size=cfg.page_size,
|
||||
logprob_chunk_enabled=False,
|
||||
logprob_chunk_size=2048,
|
||||
bs_gt1_l1_prefetch_enabled=cfg.enable_bs_gt1_prefetch_estimate,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -426,7 +460,7 @@ def run_scheduler_admission_trace(
|
||||
|
||||
pending = list(requests)
|
||||
ticks: list[TickResult] = []
|
||||
allocator = FakeTokenAllocator(cfg.available_tokens)
|
||||
allocator = FakeTokenAllocator(cfg.available_tokens, cfg)
|
||||
tree_cache = FakeTreeCache(
|
||||
allocator=allocator,
|
||||
page_size=cfg.page_size,
|
||||
@@ -484,6 +518,22 @@ def run_scheduler_admission_trace(
|
||||
accepted_rids = {req.rid for req in accepted}
|
||||
pending = [spec for spec in pending if spec.rid not in accepted_rids]
|
||||
|
||||
peak_estimate = adder.cp_shared_kv_prefill_last_buffer_estimate
|
||||
peak_breakdown = (
|
||||
{
|
||||
"layer_forward_peak_bytes": peak_estimate.layer_forward_peak_bytes,
|
||||
"logits_window_peak_bytes": peak_estimate.logits_window_peak_bytes,
|
||||
"load_back_window_peak_bytes": peak_estimate.load_back_window_peak_bytes,
|
||||
"materialize_peak_bytes": peak_estimate.materialize_peak_bytes,
|
||||
"prefetch_peak_bytes": peak_estimate.prefetch_peak_bytes,
|
||||
"logits_peak_bytes": peak_estimate.logits_peak_bytes,
|
||||
"remap_peak_bytes": peak_estimate.remap_peak_bytes,
|
||||
"transfer_descriptor_peak_bytes": peak_estimate.transfer_descriptor_peak_bytes,
|
||||
"backup_descriptor_peak_bytes": peak_estimate.backup_descriptor_peak_bytes,
|
||||
}
|
||||
if peak_estimate is not None
|
||||
else {}
|
||||
)
|
||||
ticks.append(
|
||||
TickResult(
|
||||
tick=tick_idx,
|
||||
@@ -499,6 +549,10 @@ def run_scheduler_admission_trace(
|
||||
log_input_tokens=int(adder.log_input_tokens),
|
||||
allocator_available_after_tick=allocator.available_size(),
|
||||
load_back_events=list(tree_cache.load_back_events[load_event_start:]),
|
||||
cp_estimated_peak_buffer_bytes=int(
|
||||
adder.cp_shared_kv_prefill_estimated_peak_buffer_bytes
|
||||
),
|
||||
cp_buffer_breakdown=peak_breakdown,
|
||||
duration_us=duration_us,
|
||||
)
|
||||
)
|
||||
@@ -566,7 +620,8 @@ def _print_text(trace: TraceResult) -> None:
|
||||
f"page_size={trace.config.page_size} available={trace.config.available_tokens} "
|
||||
f"evictable={trace.config.evictable_tokens} max_prefill={trace.config.max_prefill_tokens} "
|
||||
f"cp_extend_limit={trace.config.cp_shared_kv_prefill_max_total_extend_tokens} "
|
||||
f"cp_cached_limit={trace.config.cp_shared_kv_prefill_max_total_cached_tokens}"
|
||||
f"cp_cached_limit={trace.config.cp_shared_kv_prefill_max_total_cached_tokens} "
|
||||
f"cp_buffer_limit={trace.config.cp_shared_kv_prefill_max_buffer_size}"
|
||||
)
|
||||
for tick in trace.ticks:
|
||||
accepted = ",".join(
|
||||
@@ -578,6 +633,7 @@ def _print_text(trace: TraceResult) -> None:
|
||||
f"tick={tick.tick} bs={len(tick.accepted)} accepted=[{accepted}] "
|
||||
f"stop={tick.stopped_on_rid}:{tick.stopped_result} "
|
||||
f"cp_extend={tick.cp_total_extend_tokens} cp_cached={tick.cp_total_cached_tokens} "
|
||||
f"cp_peak_buffer={tick.cp_estimated_peak_buffer_bytes} "
|
||||
f"log_hit={tick.log_hit_tokens} "
|
||||
f"log_input={tick.log_input_tokens} rem_input={tick.rem_input_tokens_after_tick} "
|
||||
f"rem_total={tick.rem_total_tokens_after_tick:.1f} "
|
||||
@@ -618,6 +674,18 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--cp-max-batch-requests", type=int, default=8)
|
||||
parser.add_argument("--cp-max-total-extend-tokens", type=int, default=65_536)
|
||||
parser.add_argument("--cp-max-total-cached-tokens", type=int, default=None)
|
||||
parser.add_argument(
|
||||
"--cp-max-buffer-size",
|
||||
type=float,
|
||||
default=None,
|
||||
help="CP shared-KV estimated temp-buffer gate in decimal GB.",
|
||||
)
|
||||
parser.add_argument("--kv-cache-dim", type=int, default=656)
|
||||
parser.add_argument("--kv-dtype-bytes", type=int, default=1)
|
||||
parser.add_argument("--index-head-dim", type=int, default=128)
|
||||
parser.add_argument("--vocab-size", type=int, default=128_000)
|
||||
parser.add_argument("--tp-size", type=int, default=8)
|
||||
parser.add_argument("--enable-bs-gt1-prefetch-estimate", action="store_true")
|
||||
parser.add_argument("--no-consume-l2-load-back-capacity", action="store_true")
|
||||
parser.add_argument("--output", choices=("text", "json"), default="text")
|
||||
return parser
|
||||
@@ -642,6 +710,17 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
cp_shared_kv_prefill_max_batch_requests=args.cp_max_batch_requests,
|
||||
cp_shared_kv_prefill_max_total_extend_tokens=args.cp_max_total_extend_tokens,
|
||||
cp_shared_kv_prefill_max_total_cached_tokens=args.cp_max_total_cached_tokens,
|
||||
cp_shared_kv_prefill_max_buffer_size=(
|
||||
None
|
||||
if args.cp_max_buffer_size is None
|
||||
else int(args.cp_max_buffer_size * 1e9)
|
||||
),
|
||||
kv_cache_dim=args.kv_cache_dim,
|
||||
kv_dtype_bytes=args.kv_dtype_bytes,
|
||||
index_head_dim=args.index_head_dim,
|
||||
vocab_size=args.vocab_size,
|
||||
tp_size=args.tp_size,
|
||||
enable_bs_gt1_prefetch_estimate=args.enable_bs_gt1_prefetch_estimate,
|
||||
max_ticks=args.max_ticks,
|
||||
consume_l2_load_back_capacity=not args.no_consume_l2_load_back_capacity,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user