From e4e0784387ef643ab021e427e3dd52721247f5b6 Mon Sep 17 00:00:00 2001 From: leavelet Date: Wed, 17 Jun 2026 11:20:54 +0000 Subject: [PATCH] CP HiCache trace: content-key in fwd_hash for L1-hit vs L2-reload differential Adds a per-request content fingerprint (hash of extend input-ids + seq_len) to fwd_hash so the SAME request forwarded from an L1-hit (known-good) and an L2-reload (suspect) can be JOINED across the log without rid (the Rust PD gateway strips the client rid). Gated to bs<=1 forwards (the join is only meaningful single-request, and this skips the c=24 flood forwards so the level-3 log stays small). The analyzer joins by ck and reports the first DETERMINISTIC stage (topk/attn) that diverges = corruption localized. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sglang/srt/mem_cache/cp_hicache_trace.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/python/sglang/srt/mem_cache/cp_hicache_trace.py b/python/sglang/srt/mem_cache/cp_hicache_trace.py index f944fbc96..a65a6b191 100644 --- a/python/sglang/srt/mem_cache/cp_hicache_trace.py +++ b/python/sglang/srt/mem_cache/cp_hicache_trace.py @@ -95,6 +95,37 @@ def knz(t) -> int: return -1 +def _content_key(forward_batch) -> int: + """Stable per-request content fingerprint so the SAME request forwarded from + an L1-hit (known-good) and an L2-reload (suspect) can be JOINED across the + log WITHOUT rid (the Rust PD gateway strips the client rid -> server mints a + UUID). Derived from the extend input-ids + total seq length, which are + identical for the same content regardless of where the prefix KV came from. + Cached on the ForwardBatch (one compute, shared by every layer/stage). + Only meaningful for a SINGLE-request (bs=1) forward; for bs>1 it mixes + requests and simply won't join (harmless).""" + ck = getattr(forward_batch, "_cp_content_key", None) + if ck is not None: + return ck + ck = -1 + try: + import torch + + ids = getattr(forward_batch, "input_ids", None) + sl = getattr(forward_batch, "seq_lens", None) + h = khash(ids) if isinstance(ids, torch.Tensor) else 0 + slsum = int(sl.sum().item()) if isinstance(sl, torch.Tensor) else 0 + nreq = len(getattr(forward_batch, "rids", []) or []) + ck = ((h ^ (slsum * 1000003) ^ (nreq * 998244353)) & 0x7FFFFFFFFFFFFFFF) + except Exception: + ck = -1 + try: + forward_batch._cp_content_key = ck + except Exception: + pass + return ck + + def fwd_hash(forward_batch, layer_id, stage, t, *, level: int = 3) -> None: """Hash a per-layer forward tensor (attn-in/out, topk_indices, MoE-in) to localize where a reload forward diverges from a fresh one. Level 3 (so the @@ -109,6 +140,13 @@ def fwd_hash(forward_batch, layer_id, stage, t, *, level: int = 3) -> None: fm = getattr(forward_batch, "forward_mode", None) if fm is not None and hasattr(fm, "is_extend") and not fm.is_extend(): return + # Single-request (bs<=1) forwards only: the content-key join (L1-hit vs + # L2-reload of the SAME content) is meaningful only at bs==1 (bs>1 mixes + # requests -> ck is a blend), and this also skips the c=24 flood-evict + # forwards so the level-3 log stays small and focused on the comparison. + _rids0 = getattr(forward_batch, "rids", None) + if _rids0 is not None and len(_rids0) > 1: + return if isinstance(t, torch.Tensor): h, nz, rows = khash(t), knz(t), int(t.shape[0]) if t.dim() else 0 elif t is None: @@ -122,6 +160,7 @@ def fwd_hash(forward_batch, layer_id, stage, t, *, level: int = 3) -> None: "fwd_hash", rid=(rids[0] if rids else "?"), nreq=(len(rids) if rids else 0), + ck=_content_key(forward_batch), layer=layer_id, stage=stage, h=h,