Bound CP prefill batching by estimated temp memory

CP shared-KV bs>1 batching was only bounded by request count, extend tokens, and cached tokens. That left temporary GPU buffers such as MLA/index materialization, remap metadata, logits windows, and transfer descriptors implicit, and raw extend-token limits could exceed the active chunked-prefill budget.\n\nThis adds an explicit max-buffer-size admission gate with a CPU-only stream-aware estimator, wires it through PrefillAdder/Scheduler, performs a startup CUDA smoke allocation when configured, and reports the estimate in the scheduler admission benchmark. When chunked prefill is active, the effective CP extend-token limit is capped by the current chunk budget so the CP path does not advertise unreachable batch capacity or lift max-prefill-tokens too far.\n\nConstraint: Admission estimation must stay CPU-only on the scheduler hot path; CUDA allocation is limited to startup smoke checking.\nConstraint: Single oversized requests must still be allowed to run alone to avoid scheduler deadlock.\nRejected: Rely only on --max-prefill-tokens | it does not reliably bound the first oversized request and does not model cache-hit/load-back pressure.\nRejected: Let CP extend limit exceed chunked-prefill size | it creates an unreachable effective capacity and misleading budget lift.\nConfidence: medium\nScope-risk: moderate\nDirective: If bs>1 L1 prefetch is enabled later, update CPSharedKVPrefillBufferEstimatorContext.bs_gt1_l1_prefetch_enabled and include the live prefetch dense buffers in overlap windows.\nTested: local py_compile for touched files\nTested: local PYTHONPATH=python pytest -q test/registered/unit/managers/test_cp_shared_kv_prefill_buffer_estimator.py (4 passed)\nTested: remote cjy-glm5-new targeted pytest for new server_args, PrefillAdder, estimator, and benchmark cases (10 passed)\nTested: remote cjy-glm5-new PYTHONPATH=python pytest -q test/registered/unit/managers/test_cp_shared_kv_prefill_buffer_estimator.py test/registered/unit/managers/test_prefill_adder.py test/registered/unit/managers/test_prefill_scheduler_admission_bench.py (29 passed before chunk cap, then test_prefill_adder.py 21 passed after chunk cap)\nNot-tested: full server_args suite because existing TestPrepareServerArgs tries to reach HuggingFace and fails under container DNS/network\nNot-tested: GLM5 ETE smoke with --cp-shared-kv-prefill-max-buffer-size
This commit is contained in:
laoyao0822
2026-06-11 01:33:28 +08:00
parent 9a9893e571
commit 3a43727216
10 changed files with 1806 additions and 3 deletions
@@ -143,6 +143,13 @@ class SchedulerBenchConfig:
cp_shared_kv_prefill_max_batch_requests: Optional[int] = None
cp_shared_kv_prefill_max_total_extend_tokens: Optional[int] = None
cp_shared_kv_prefill_max_total_cached_tokens: Optional[int] = None
cp_shared_kv_prefill_max_buffer_size: Optional[int] = None
kv_cache_dim: int = 656
kv_dtype_bytes: int = 1
index_head_dim: int = 128
vocab_size: int = 128_000
tp_size: int = 8
enable_bs_gt1_prefetch_estimate: bool = False
max_ticks: int = 1
consume_l2_load_back_capacity: bool = True
@@ -198,6 +205,8 @@ class TickResult:
log_input_tokens: int
allocator_available_after_tick: int
load_back_events: list[LoadBackEvent]
cp_estimated_peak_buffer_bytes: int
cp_buffer_breakdown: dict[str, int]
duration_us: float
@@ -211,8 +220,15 @@ class TraceResult:
class FakeTokenAllocator:
def __init__(self, available_tokens: int):
def __init__(self, available_tokens: int, cfg: SchedulerBenchConfig):
self.available_tokens = int(available_tokens)
self.kvcache = SimpleNamespace(
kv_cache_dim=cfg.kv_cache_dim,
store_dtype=SimpleNamespace(itemsize=cfg.kv_dtype_bytes),
index_head_dim=cfg.index_head_dim,
quant_block_size=128,
index_k_with_scale_buffer_dtype=SimpleNamespace(itemsize=1),
)
def available_size(self) -> int:
return self.available_tokens
@@ -231,6 +247,9 @@ class FakeTokenAllocator:
self.available_tokens -= tokens
return True
def get_kvcache(self):
return self.kvcache
class FakeTreeCache:
def __init__(
@@ -382,6 +401,9 @@ def _configure_scheduler_globals(enable_cp_context: bool) -> None:
def _make_prefill_adder(cfg: SchedulerBenchConfig, tree_cache: FakeTreeCache, allocator: FakeTokenAllocator):
_install_sgl_kernel_stubs()
from sglang.srt.managers.cp_shared_kv_prefill_buffer_estimator import (
CPSharedKVPrefillBufferEstimatorContext,
)
from sglang.srt.managers.schedule_policy import PrefillAdder
return PrefillAdder(
@@ -398,6 +420,18 @@ def _make_prefill_adder(cfg: SchedulerBenchConfig, tree_cache: FakeTreeCache, al
cp_shared_kv_prefill_max_batch_requests=cfg.cp_shared_kv_prefill_max_batch_requests,
cp_shared_kv_prefill_max_total_extend_tokens=cfg.cp_shared_kv_prefill_max_total_extend_tokens,
cp_shared_kv_prefill_max_total_cached_tokens=cfg.cp_shared_kv_prefill_max_total_cached_tokens,
cp_shared_kv_prefill_max_buffer_size=cfg.cp_shared_kv_prefill_max_buffer_size,
cp_shared_kv_prefill_buffer_estimator_context=(
CPSharedKVPrefillBufferEstimatorContext(
kvcache=allocator.get_kvcache(),
model_config=SimpleNamespace(vocab_size=cfg.vocab_size),
tp_size=cfg.tp_size,
page_size=cfg.page_size,
logprob_chunk_enabled=False,
logprob_chunk_size=2048,
bs_gt1_l1_prefetch_enabled=cfg.enable_bs_gt1_prefetch_estimate,
)
),
)
@@ -426,7 +460,7 @@ def run_scheduler_admission_trace(
pending = list(requests)
ticks: list[TickResult] = []
allocator = FakeTokenAllocator(cfg.available_tokens)
allocator = FakeTokenAllocator(cfg.available_tokens, cfg)
tree_cache = FakeTreeCache(
allocator=allocator,
page_size=cfg.page_size,
@@ -484,6 +518,22 @@ def run_scheduler_admission_trace(
accepted_rids = {req.rid for req in accepted}
pending = [spec for spec in pending if spec.rid not in accepted_rids]
peak_estimate = adder.cp_shared_kv_prefill_last_buffer_estimate
peak_breakdown = (
{
"layer_forward_peak_bytes": peak_estimate.layer_forward_peak_bytes,
"logits_window_peak_bytes": peak_estimate.logits_window_peak_bytes,
"load_back_window_peak_bytes": peak_estimate.load_back_window_peak_bytes,
"materialize_peak_bytes": peak_estimate.materialize_peak_bytes,
"prefetch_peak_bytes": peak_estimate.prefetch_peak_bytes,
"logits_peak_bytes": peak_estimate.logits_peak_bytes,
"remap_peak_bytes": peak_estimate.remap_peak_bytes,
"transfer_descriptor_peak_bytes": peak_estimate.transfer_descriptor_peak_bytes,
"backup_descriptor_peak_bytes": peak_estimate.backup_descriptor_peak_bytes,
}
if peak_estimate is not None
else {}
)
ticks.append(
TickResult(
tick=tick_idx,
@@ -499,6 +549,10 @@ def run_scheduler_admission_trace(
log_input_tokens=int(adder.log_input_tokens),
allocator_available_after_tick=allocator.available_size(),
load_back_events=list(tree_cache.load_back_events[load_event_start:]),
cp_estimated_peak_buffer_bytes=int(
adder.cp_shared_kv_prefill_estimated_peak_buffer_bytes
),
cp_buffer_breakdown=peak_breakdown,
duration_us=duration_us,
)
)
@@ -566,7 +620,8 @@ def _print_text(trace: TraceResult) -> None:
f"page_size={trace.config.page_size} available={trace.config.available_tokens} "
f"evictable={trace.config.evictable_tokens} max_prefill={trace.config.max_prefill_tokens} "
f"cp_extend_limit={trace.config.cp_shared_kv_prefill_max_total_extend_tokens} "
f"cp_cached_limit={trace.config.cp_shared_kv_prefill_max_total_cached_tokens}"
f"cp_cached_limit={trace.config.cp_shared_kv_prefill_max_total_cached_tokens} "
f"cp_buffer_limit={trace.config.cp_shared_kv_prefill_max_buffer_size}"
)
for tick in trace.ticks:
accepted = ",".join(
@@ -578,6 +633,7 @@ def _print_text(trace: TraceResult) -> None:
f"tick={tick.tick} bs={len(tick.accepted)} accepted=[{accepted}] "
f"stop={tick.stopped_on_rid}:{tick.stopped_result} "
f"cp_extend={tick.cp_total_extend_tokens} cp_cached={tick.cp_total_cached_tokens} "
f"cp_peak_buffer={tick.cp_estimated_peak_buffer_bytes} "
f"log_hit={tick.log_hit_tokens} "
f"log_input={tick.log_input_tokens} rem_input={tick.rem_input_tokens_after_tick} "
f"rem_total={tick.rem_total_tokens_after_tick:.1f} "
@@ -618,6 +674,18 @@ def build_arg_parser() -> argparse.ArgumentParser:
parser.add_argument("--cp-max-batch-requests", type=int, default=8)
parser.add_argument("--cp-max-total-extend-tokens", type=int, default=65_536)
parser.add_argument("--cp-max-total-cached-tokens", type=int, default=None)
parser.add_argument(
"--cp-max-buffer-size",
type=float,
default=None,
help="CP shared-KV estimated temp-buffer gate in decimal GB.",
)
parser.add_argument("--kv-cache-dim", type=int, default=656)
parser.add_argument("--kv-dtype-bytes", type=int, default=1)
parser.add_argument("--index-head-dim", type=int, default=128)
parser.add_argument("--vocab-size", type=int, default=128_000)
parser.add_argument("--tp-size", type=int, default=8)
parser.add_argument("--enable-bs-gt1-prefetch-estimate", action="store_true")
parser.add_argument("--no-consume-l2-load-back-capacity", action="store_true")
parser.add_argument("--output", choices=("text", "json"), default="text")
return parser
@@ -642,6 +710,17 @@ def main(argv: Optional[list[str]] = None) -> int:
cp_shared_kv_prefill_max_batch_requests=args.cp_max_batch_requests,
cp_shared_kv_prefill_max_total_extend_tokens=args.cp_max_total_extend_tokens,
cp_shared_kv_prefill_max_total_cached_tokens=args.cp_max_total_cached_tokens,
cp_shared_kv_prefill_max_buffer_size=(
None
if args.cp_max_buffer_size is None
else int(args.cp_max_buffer_size * 1e9)
),
kv_cache_dim=args.kv_cache_dim,
kv_dtype_bytes=args.kv_dtype_bytes,
index_head_dim=args.index_head_dim,
vocab_size=args.vocab_size,
tp_size=args.tp_size,
enable_bs_gt1_prefetch_estimate=args.enable_bs_gt1_prefetch_estimate,
max_ticks=args.max_ticks,
consume_l2_load_back_capacity=not args.no_consume_l2_load_back_capacity,
)