Expose gated evidence for CP shared-KV bs>1 debugging
The bs>1 prefill path has multiple coupled stages: scheduler admission, page-aligned batch planning, tensor splitting, direct cache writes, index top-k, MLA reuse, and disaggregated KV handoff. Add a default-off, rate-limited debug channel so production ETE runs can identify where batching or metadata semantics diverge without permanently increasing hot-path log volume. Constraint: Logs must be default-off and rate-limited because these paths execute per-rank and often per-layer. Rejected: Always-on INFO logs | would flood logs and add CPU overhead during normal prefill. Rejected: Only scheduler-side logging | insufficient to distinguish planner, index, MLA, and transfer handoff failures. Confidence: medium Scope-risk: moderate Directive: Keep bs>1 debug evidence env-gated; do not add unconditional per-layer or per-token logs in these paths. Tested: Local py_compile for touched files Tested: git diff --check Tested: Remote py_compile and targeted NSA CP utility tests: 5 passed Not-tested: Full ETE correctness with debug disabled
This commit is contained in:
@@ -68,6 +68,24 @@ def _cp_draft_shared_kv_debug(message: str, *args) -> None:
|
||||
logger.info("[CP_DRAFT_SHARED_KV] " + message, *args)
|
||||
|
||||
|
||||
_CP_SHARED_KV_BS_GT1_PREFILL_DEBUG_COUNTS = {}
|
||||
|
||||
|
||||
def _cp_shared_kv_bs_gt1_prefill_debug(
|
||||
key: str,
|
||||
message: str,
|
||||
*args,
|
||||
) -> None:
|
||||
if not envs.SGLANG_CP_SHARED_KV_BS_GT1_DEBUG.get():
|
||||
return
|
||||
limit = int(envs.SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT.get())
|
||||
count = _CP_SHARED_KV_BS_GT1_PREFILL_DEBUG_COUNTS.get(key, 0)
|
||||
if limit > 0 and count >= limit:
|
||||
return
|
||||
_CP_SHARED_KV_BS_GT1_PREFILL_DEBUG_COUNTS[key] = count + 1
|
||||
logger.info("[CP_SHARED_KV_BS_GT1_DEBUG] event=%s " + message, key, *args)
|
||||
|
||||
|
||||
def _seq_summary(values) -> str:
|
||||
if values is None:
|
||||
return "None"
|
||||
@@ -622,6 +640,28 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
logprob_pt = 0
|
||||
# Transfer kv for prefill completed requests and add it into disagg_prefill_inflight_queue
|
||||
next_token_ids = result.next_token_ids.tolist()
|
||||
if envs.SGLANG_CP_SHARED_KV_BS_GT1_DEBUG.get():
|
||||
spec_info = getattr(batch, "spec_info", None)
|
||||
spec_hidden = getattr(spec_info, "hidden_states", None)
|
||||
spec_topk = getattr(spec_info, "topk_index", None)
|
||||
_cp_shared_kv_bs_gt1_prefill_debug(
|
||||
"prefill_result_handoff",
|
||||
"bs=%s rids=%s extend_lens=%s prefix_lens=%s next_token_ids=%s "
|
||||
"has_spec=%s hidden_shape=%s topk_shape=%s out_cache_tokens=%s "
|
||||
"inflight_before=%s",
|
||||
len(batch.reqs),
|
||||
[req.rid for req in batch.reqs[:8]],
|
||||
list(getattr(batch, "extend_lens", []) or []),
|
||||
list(getattr(batch, "prefix_lens", []) or []),
|
||||
next_token_ids[:8],
|
||||
spec_info is not None,
|
||||
tuple(spec_hidden.shape) if spec_hidden is not None else None,
|
||||
tuple(spec_topk.shape) if spec_topk is not None else None,
|
||||
int(batch.out_cache_loc.numel())
|
||||
if getattr(batch, "out_cache_loc", None) is not None
|
||||
else None,
|
||||
len(self.disagg_prefill_inflight_queue),
|
||||
)
|
||||
if batch.return_logprob:
|
||||
if logits_output.next_token_logprobs is not None:
|
||||
logits_output.next_token_logprobs = (
|
||||
@@ -989,6 +1029,28 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
getattr(req, "already_computed", None),
|
||||
draft_prefix_overlap,
|
||||
)
|
||||
if envs.SGLANG_CP_SHARED_KV_BS_GT1_DEBUG.get():
|
||||
_cp_shared_kv_bs_gt1_prefill_debug(
|
||||
"send_kv_chunk",
|
||||
"rid=%s room=%s start_idx=%s end_idx=%s last_chunk=%s "
|
||||
"page_size=%s pages=%s state_pages=%s prefix_len=%s "
|
||||
"host_hit_length=%s extend_input_len=%s fill_len=%s "
|
||||
"origin_input_len=%s has_draft_pool=%s",
|
||||
req.rid,
|
||||
req.bootstrap_room,
|
||||
start_idx,
|
||||
end_idx,
|
||||
last_chunk,
|
||||
page_size,
|
||||
_seq_summary(page_indices),
|
||||
_seq_summary(state_indices),
|
||||
prefix_len,
|
||||
host_hit_length,
|
||||
getattr(req, "extend_input_len", None),
|
||||
len(req.fill_ids),
|
||||
len(req.origin_input_ids),
|
||||
has_draft_pool,
|
||||
)
|
||||
if has_draft_pool and draft_prefix_overlap > 0:
|
||||
_cp_draft_shared_kv_debug(
|
||||
"prefill_send_cachehit_draft_prefix rid=%s room=%s "
|
||||
|
||||
Reference in New Issue
Block a user