CP HiCache trace: content-key in fwd_hash for L1-hit vs L2-reload differential

Adds a per-request content fingerprint (hash of extend input-ids + seq_len) to
fwd_hash so the SAME request forwarded from an L1-hit (known-good) and an
L2-reload (suspect) can be JOINED across the log without rid (the Rust PD
gateway strips the client rid). Gated to bs<=1 forwards (the join is only
meaningful single-request, and this skips the c=24 flood forwards so the
level-3 log stays small). The analyzer joins by ck and reports the first
DETERMINISTIC stage (topk/attn) that diverges = corruption localized.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

python/sglang/srt/mem_cache/cp_hicache_trace.py

@@ -95,6 +95,37 @@ def knz(t) -> int:
         return -1
 
 
+def _content_key(forward_batch) -> int:
+    """Stable per-request content fingerprint so the SAME request forwarded from
+    an L1-hit (known-good) and an L2-reload (suspect) can be JOINED across the
+    log WITHOUT rid (the Rust PD gateway strips the client rid -> server mints a
+    UUID). Derived from the extend input-ids + total seq length, which are
+    identical for the same content regardless of where the prefix KV came from.
+    Cached on the ForwardBatch (one compute, shared by every layer/stage).
+    Only meaningful for a SINGLE-request (bs=1) forward; for bs>1 it mixes
+    requests and simply won't join (harmless)."""
+    ck = getattr(forward_batch, "_cp_content_key", None)
+    if ck is not None:
+        return ck
+    ck = -1
+    try:
+        import torch
+
+        ids = getattr(forward_batch, "input_ids", None)
+        sl = getattr(forward_batch, "seq_lens", None)
+        h = khash(ids) if isinstance(ids, torch.Tensor) else 0
+        slsum = int(sl.sum().item()) if isinstance(sl, torch.Tensor) else 0
+        nreq = len(getattr(forward_batch, "rids", []) or [])
+        ck = ((h ^ (slsum * 1000003) ^ (nreq * 998244353)) & 0x7FFFFFFFFFFFFFFF)
+    except Exception:
+        ck = -1
+    try:
+        forward_batch._cp_content_key = ck
+    except Exception:
+        pass
+    return ck
+
+
 def fwd_hash(forward_batch, layer_id, stage, t, *, level: int = 3) -> None:
     """Hash a per-layer forward tensor (attn-in/out, topk_indices, MoE-in) to
     localize where a reload forward diverges from a fresh one. Level 3 (so the
@@ -109,6 +140,13 @@ def fwd_hash(forward_batch, layer_id, stage, t, *, level: int = 3) -> None:
         fm = getattr(forward_batch, "forward_mode", None)
         if fm is not None and hasattr(fm, "is_extend") and not fm.is_extend():
             return
+        # Single-request (bs<=1) forwards only: the content-key join (L1-hit vs
+        # L2-reload of the SAME content) is meaningful only at bs==1 (bs>1 mixes
+        # requests -> ck is a blend), and this also skips the c=24 flood-evict
+        # forwards so the level-3 log stays small and focused on the comparison.
+        _rids0 = getattr(forward_batch, "rids", None)
+        if _rids0 is not None and len(_rids0) > 1:
+            return
         if isinstance(t, torch.Tensor):
             h, nz, rows = khash(t), knz(t), int(t.shape[0]) if t.dim() else 0
         elif t is None:
@@ -122,6 +160,7 @@ def fwd_hash(forward_batch, layer_id, stage, t, *, level: int = 3) -> None:
             "fwd_hash",
             rid=(rids[0] if rids else "?"),
             nreq=(len(rids) if rids else 0),
+            ck=_content_key(forward_batch),
             layer=layer_id,
             stage=stage,
             h=h,