CP HiCache L3 3.4: cold-start (restart durability) — load|clear via SGLANG_CP_L3_COLD_START

On restart the LMDB index + disk-slab blobs persist, but CpL3Store.from_config built the slot
pools all-free and the GC LRU empty -> the next spill re-hands-out a slot the durable index still
references (clobbers a live blob) and GC never reclaims the carried-over entries. Neither a clean
start nor a durable reload was actually realized.

connect() now applies a cold-start policy BEFORE the bg threads start (the write thread is the sole
pool/GC owner, so the single-threaded reconcile must precede it):
  - clear (default): wipe the persisted index + reset the pools/GC -> genuinely empty start (disk
    blobs are inert, overwritten lazily on slot reuse).
  - load: rebuild this rank's slot free-list + GC LRU from the durable disk blobs. Drive the scan
    from the rank's OWN slab file (header-only reads) so it never inspects another rank's slots even
    when ranks share a disk; a slot is LIVE iff its blob header parses AND the shared index still maps
    that content hash back to this exact slot -> occupy + seed the GC LRU with the durable last_access;
    orphan/unwritten slots stay free. Header-only (no payload CRC); reload-time verify-on-read still
    fail-softs a torn payload. The L3 durable floor now survives a process restart.

Primitives: CpL3SlotPool.rebuild_from_allocated (O(num_slots) bulk-occupy) + CpL3DiskSlab.read_header
(one aligned block, not the multi-MB slot). Env SGLANG_CP_L3_COLD_START (default "clear"), read in
_maybe_init_cp_l3 and passed to connect(). Tests: 4 cold-start e2e (load rebuilds the floor + no slot
collision after a fresh spill; GC LRU rebuilt; clear starts empty; unknown mode fails loud) + 2 unit
(rebuild_from_allocated, read_header). 23 L3 store/disk/posix tests green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

python/sglang/srt/environ.py

@@ -406,6 +406,14 @@ class Envs:
     # full radix tree and asserts the running per-rank counts match (fail loud on
     # drift). Expensive full-walk per snapshot -> default off; on in bring-up/CI.
     SGLANG_CP_HICACHE_VERIFY_SNAPSHOT = EnvBool(False)
+    # CP HiCache L3 (disk durable floor) cold-start policy at connect():
+    #   "clear" (default) -> wipe the persisted LMDB index + reset the slot pools so
+    #     the process starts with an empty L3 (the durable blobs on disk are inert and
+    #     overwritten lazily on reuse). Safe + matches "fresh boot" semantics.
+    #   "load" -> rebuild the slot free-list + the GC LRU from the persisted disk-slab
+    #     blob headers (cross-checked against the durable index), so the L3 durable
+    #     floor SURVIVES a process restart. Opt-in.
+    SGLANG_CP_L3_COLD_START = EnvStr("clear")
 
     # Mooncake KV Transfer
     SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)

python/sglang/srt/mem_cache/cp_l3_disk.py

@@ -216,3 +216,19 @@ class CpL3SlotPool:
         self._free = list(range(self._num_slots - 1, -1, -1))
         self._allocated = set()
 
+    def rebuild_from_allocated(self, allocated) -> None:
+        """Cold-rebuild (restart durability): set the occupied set in bulk from a scan of the durable disk
+        blobs, deriving the free list as the complement. O(num_slots) (one pass), vs O(n^2) for per-slot
+        ``free.remove``. Free order stays deterministic (lowest index first). Fail loud on an out-of-range or
+        duplicate slot (a corrupt/inconsistent rebuild input)."""
+        alloc: set[int] = set()
+        for idx in allocated:
+            idx = int(idx)
+            if idx < 0 or idx >= self._num_slots:
+                _l3_failfast(f"rebuild slot index {idx} out of range [0,{self._num_slots})")
+            if idx in alloc:
+                _l3_failfast(f"rebuild duplicate L3 slot {idx}")
+            alloc.add(idx)
+        self._allocated = alloc
+        self._free = [i for i in range(self._num_slots - 1, -1, -1) if i not in alloc]
+

python/sglang/srt/mem_cache/cp_l3_posix.py

@@ -20,7 +20,10 @@ import os
 from typing import Optional
 
 from sglang.srt.mem_cache.cp_l3_disk import (
+    CP_L3_DIO_ALIGN,
     CP_L3_HEADER_BYTES,
+    CpL3BlobHeader,
+    align_up,
     compute_crc,
     pack_blob_header,
     slot_bytes_for_page,
@@ -126,6 +129,26 @@ class CpL3DiskSlab:
             return None
         return payload
 
+    def read_header(self, slot_idx: int) -> Optional[CpL3BlobHeader]:
+        """Read + parse ONLY the 64B blob header of a slot (cold-rebuild scan). Reads one O_DIRECT-aligned
+        block (the header lives at offset 0 of the slot) instead of the whole multi-MB slot, so scanning all
+        slots costs ~num_slots * 4 KiB. Returns None for an unwritten/torn/foreign slot (bad magic/version);
+        does NOT verify the payload CRC (that is deferred to reload-time verify-on-read -- a header-only scan is
+        the cheap rebuild path, and reload still fail-softs on a bad payload)."""
+        self._check_slot(slot_idx)
+        nbytes = align_up(CP_L3_HEADER_BYTES, CP_L3_DIO_ALIGN)  # one aligned block (>= 64B)
+        buf = mmap.mmap(-1, nbytes)
+        try:
+            n = os.preadv(self._fd, [buf], slot_idx * self.slot_bytes)
+            if n < CP_L3_HEADER_BYTES:
+                return None
+            try:
+                return unpack_blob_header(buf[:CP_L3_HEADER_BYTES])
+            except ValueError:
+                return None  # unwritten (zero), torn, or foreign slot -> treat as free
+        finally:
+            buf.close()
+
     def read_into(self, slot_idx: int, dst_buf: mmap.mmap, *, expect_hash: Optional[bytes] = None) -> bool:
         """Read a slot into a caller-provided aligned buffer (zero-copy reload). Returns True iff intact;
         on True the payload is at dst_buf[CP_L3_HEADER_BYTES:CP_L3_HEADER_BYTES+page_bytes]."""

python/sglang/srt/mem_cache/cp_l3_store.py

@@ -196,8 +196,11 @@ class CpL3Store:
             index=index, slabs=slabs, pools=pools, accessors=accessors,
         )
 
-    def connect(self, cfg: Optional[CpL3Config] = None) -> None:
-        """Start the bg threads. If cfg.require_plp, fail loud on a non-PLP disk (durability gate)."""
+    def connect(self, cfg: Optional[CpL3Config] = None, *, cold_start: str = "clear") -> None:
+        """Open the store for use. If cfg.require_plp, fail loud on a non-PLP disk (durability gate). Apply the
+        cold-start policy (``clear`` | ``load``) to the persisted state BEFORE the bg threads start (the write
+        thread is the sole pool/GC owner, so the single-threaded rebuild must precede it), then start the
+        threads."""
         if cfg is not None and cfg.require_plp:
             is_plp, median_us = probe_disk_plp(self.disk_dir)
             if not is_plp:
@@ -206,8 +209,79 @@ class CpL3Store:
                     f"{median_us:.1f}us > threshold); durability is unsafe. Set require_plp=false only if "
                     f"you accept data loss on power failure."
                 )
+        self._cold_start(cold_start)
         self._start_threads()
 
+    # ---------- cold start (restart durability) ----------
+    def _cold_start(self, mode: str) -> None:
+        """Reconcile the in-memory slot pools + GC LRU with the persisted index + disk blobs at connect().
+
+        ``clear`` (default): wipe the persisted index + reset the pools/GC -> the store starts empty (the disk
+        blobs are inert and overwritten lazily as slots are reused). ``load``: rebuild the slot free-list + the
+        GC LRU from this rank's durable disk-slab blob headers (cross-checked vs the index) so the L3 durable
+        floor survives a restart. Either way the post-condition is a CONSISTENT pool<->index<->disk state (the
+        bug is doing neither: from_config builds the pools all-free while the durable index still references
+        occupied slots -> the next spill re-hands-out a live slot)."""
+        mode = (mode or "clear").strip().lower()
+        if mode == "clear":
+            self._cold_clear()
+        elif mode == "load":
+            self._cold_rebuild()
+        else:
+            _l3_failfast(f"unknown cold_start mode {mode!r} (expected 'clear' or 'load')")
+
+    def _cold_clear(self) -> None:
+        """Fresh, empty L3: reset the slot pools + GC, drop every persisted index entry. The on-disk blobs are
+        left as-is (inert: nothing references them once the index is empty; they are overwritten on slot reuse)."""
+        for pool in self.pools.values():
+            pool.reset()
+        for pk in self.payloads:
+            self._gc_heap[pk].clear()
+            self._gc_current[pk].clear()
+        self._gc_seq = 0
+        self.index.clear()
+        logger.info("[CP-L3] cold-start=clear rank=%d: index wiped, %d slot pools reset",
+                    self.cp_rank, len(self.pools))
+
+    def _cold_rebuild(self) -> None:
+        """Rebuild this rank's slot free-list + GC LRU from the durable disk blobs (restart durability).
+
+        Per payload, scan this rank's own slab file slot-by-slot (cheap header-only reads). A slot is LIVE iff
+        its blob header parses AND the shared index still maps that content hash back to THIS exact slot on THIS
+        disk -- then occupy it and seed the GC LRU with the index's durable last_access. Everything else stays
+        free: an unwritten slot, OR an ORPHAN blob (index deleted/moved -> the blob is unreachable and may be
+        overwritten). Driven by the rank's own slab (not the shared index) so it never inspects another rank's
+        slots, even when ranks share a disk. Header-only (no payload CRC) -> reload-time verify-on-read still
+        fail-softs a torn payload."""
+        total_live = 0
+        for pk in self.payloads:
+            slab = self.slabs[pk]
+            allocated: List[int] = []
+            n_live = n_orphan = 0
+            for slot in range(slab.num_slots):
+                hdr = slab.read_header(slot)
+                if hdr is None:
+                    continue  # unwritten / torn / foreign -> free
+                if hdr.payload_kind != pk:
+                    n_orphan += 1
+                    continue  # per-payload file; a foreign kind = stale -> free
+                h = hdr.content_hash  # 32 raw bytes
+                entry = self.index.get(h, pk)
+                if entry is None or entry.slot_idx != slot or entry.disk_id != self.disk_id:
+                    n_orphan += 1
+                    continue  # index doesn't map this hash back to this slot -> orphan blob, leave free
+                allocated.append(slot)
+                self._gc_add(pk, h, slot, entry.last_access)
+                n_live += 1
+            self.pools[pk].rebuild_from_allocated(allocated)
+            total_live += n_live
+            logger.info(
+                "[CP-L3] cold-start=load rank=%d payload=%s: %d live / %d orphan of %d slots restored",
+                self.cp_rank, pk, n_live, n_orphan, slab.num_slots,
+            )
+        logger.info("[CP-L3] cold-start=load rank=%d: %d total live pages restored (durable floor survives restart)",
+                    self.cp_rank, total_live)
+
     def _start_threads(self) -> None:
         if self._started:
             return

python/sglang/srt/mem_cache/hiradix_cache.py

@@ -1307,10 +1307,11 @@ class HiRadixCache(RadixCache):
         self.cp_l3_store = CpL3Store.from_config(
             cfg, cp_rank=cp_rank, cp_size=cp_size, accessors=accessors
         )
-        self.cp_l3_store.connect(cfg)
+        cold_start = envs.SGLANG_CP_L3_COLD_START.get()
+        self.cp_l3_store.connect(cfg, cold_start=cold_start)
         logger.info(
-            "[CP-L3] enabled rank=%d/%d payloads=%s disk_dir=%s",
-            cp_rank, cp_size, list(accessors), self.cp_l3_store.disk_dir,
+            "[CP-L3] enabled rank=%d/%d payloads=%s disk_dir=%s cold_start=%s",
+            cp_rank, cp_size, list(accessors), self.cp_l3_store.disk_dir, cold_start,
         )
         # (CP-HiCache metrics are created lazily in _cp_maybe_collect_metrics, gated on L2 pooling -- so they
         # cover PURE-L2-without-L3 too, not just this L3 path.)