Gate CP HiCache write-backup on hard deficit, not the free-room watermark

A long cold-request workload flooded the prefill log (~177MB / 32K lines in
8min, all 8 CP ranks) with cp_host_reservation_plan_insufficient +
prepare_write_backup_reservation_failed even though the host pool was <42%
used and the writes physically fit.

Root cause: _free_room_deficit folds the proactive 20% free-room target
(hicache_host_free_room_ratio) into the admission deficit once a lane dips
below the 10% trigger. _evict_cp_host_for_write_admission then failed -- and
evicted nothing -- whenever it could not reclaim that full 20% target, which
is impossible while the residual is locked/pending. The lane never drained,
so every subsequent backup re-issued the same impossible demand and re-logged
on every tick. The reservation failure is itself a graceful skip, so there is
no crash; the symptom is the log storm.

Gate write-backup admission on the HARD deficit -- max(0, required-available)
over the target and draft lanes (draft_available is 2**62 with no draft pool,
a no-op) -- and evict toward the watermark best-effort, admitting whenever the
write itself fits. This drains the stuck lane and backs the request up instead
of skipping+flooding. The proactive watermark stays as the eviction target
(deficit_by_owner unchanged); it is no longer a hard admission gate. The
decision is computed from rank-replicated state on the collective-free reserve
path, so every CP rank makes the same admit/skip choice (opus-reviewed
rank-safe).

Also rate-limit the four host-reservation fallback warnings (once per 10s per
fallback_name, with a suppressed count) so any genuine exhaustion degrades
quietly instead of producing a log storm.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

python/sglang/srt/mem_cache/hiradix_cache.py

@@ -2135,21 +2135,45 @@ class HiRadixCache(RadixCache):
         self, admission: CpWriteAdmission, *, node_id: int, phase: str
     ) -> bool:
         eviction_plan = admission.eviction_plan
-        if any(v > 0 for v in eviction_plan.remaining_deficit):
-            logger.warning(
-                "[CP_HICACHE_FALLBACK][cp_host_reservation_plan_insufficient] "
-                "reason=insufficient_evictable_host_slots node_id=%d phase=%s "
-                "required=%s target_avail=%s draft_avail=%s deficit=%s "
-                "planned_freed=%s remaining_deficit=%s victims=%s",
-                node_id,
-                phase,
+        # Only the HARD deficit -- what the backup actually needs in order to fit
+        # -- may block admission. admission.deficit_by_owner (hence
+        # eviction_plan.remaining_deficit) also folds in the PROACTIVE free-room
+        # watermark (hicache_host_free_room_ratio): once a lane dips below the
+        # trigger watermark that eviction target balloons to ~20% of capacity.
+        # Refusing a backup that physically fits, just because a lane is below
+        # that advisory target, turned a ~1k-slot write into a 20%-of-capacity
+        # reclaim demand; when locked residual made the target unreachable it
+        # skipped the backup AND re-fired every tick on every rank (a 177MB log
+        # storm with the host pool <42% used, 2026-06-19). Evict toward the
+        # target best-effort, but admit whenever the write itself fits.
+        # draft_available is 2**62 when there is no draft pool, so the draft term
+        # is a no-op in that case.
+        hard_deficit = [
+            max(max(0, int(req) - int(tgt)), max(0, int(req) - int(drf)))
+            for req, tgt, drf in zip(
                 admission.required_by_owner,
                 admission.target_available_by_owner,
                 admission.draft_available_by_owner,
-                admission.deficit_by_owner,
-                eviction_plan.planned_freed,
-                eviction_plan.remaining_deficit,
-                [getattr(node, "id", None) for node in eviction_plan.victims],
+            )
+        ]
+        remaining_hard = [
+            max(0, hard - int(freed))
+            for hard, freed in zip(hard_deficit, eviction_plan.planned_freed)
+        ]
+        if any(v > 0 for v in remaining_hard):
+            self._warn_cp_hicache_fallback(
+                "cp_host_reservation_plan_insufficient",
+                "insufficient_evictable_host_slots",
+                rate_limit_s=10.0,
+                node_id=node_id,
+                phase=phase,
+                hard_deficit=hard_deficit,
+                required=admission.required_by_owner,
+                target_avail=admission.target_available_by_owner,
+                draft_avail=admission.draft_available_by_owner,
+                planned_freed=eviction_plan.planned_freed,
+                remaining_hard=remaining_hard,
+                victims=len(eviction_plan.victims),
             )
             return False
 
@@ -2316,10 +2340,35 @@ class HiRadixCache(RadixCache):
         )
         self.cache_controller.evict_cp_host(prepared.metadata)
 
-    def _warn_cp_hicache_fallback(self, fallback_name: str, reason: str, **kwargs):
+    def _warn_cp_hicache_fallback(
+        self, fallback_name: str, reason: str, *, rate_limit_s: float = 0.0, **kwargs
+    ):
         details = " ".join(f"{key}={value}" for key, value in kwargs.items())
         if details:
             details = " " + details
+        if rate_limit_s > 0.0:
+            # Host-reservation shortfalls are emitted per backup-attempt per rank;
+            # under genuine host pressure that is thousands of identical lines a
+            # minute (a 177MB log storm, 2026-06-19). Collapse repeats of the same
+            # fallback_name within the window so a real shortfall stays loud but
+            # bounded -- the first line of each window carries the count suppressed
+            # since the previous emission.
+            warn_state = getattr(self, "_cp_warn_state", None)
+            if warn_state is None:
+                warn_state = {}
+                self._cp_warn_state = warn_state
+            now = time.monotonic()
+            entry = warn_state.get(fallback_name)
+            if entry is not None and (now - entry[0]) < rate_limit_s:
+                entry[1] += 1
+                return
+            suppressed = entry[1] if entry is not None else 0
+            warn_state[fallback_name] = [now, 0]
+            if suppressed:
+                details = (
+                    f"{details} suppressed_since_last={suppressed} "
+                    f"window_s={int(rate_limit_s)}"
+                )
         logger.warning(
             "[CP_HICACHE_FALLBACK][%s] reason=%s%s",
             fallback_name,
@@ -2667,6 +2716,7 @@ class HiRadixCache(RadixCache):
             self._warn_cp_hicache_fallback(
                 "prepare_write_backup_reservation_failed",
                 "host_reservation_failed",
+                rate_limit_s=10.0,
                 node_id=node_id,
                 rid=getattr(req, "rid", "<unknown>"),
                 logical_len=len(kv_indices),
@@ -2757,6 +2807,7 @@ class HiRadixCache(RadixCache):
                     self._warn_cp_hicache_fallback(
                         "prepare_write_backups_batch_admission_failed",
                         "host_reservation_failed",
+                        rate_limit_s=10.0,
                         batch_size=len(candidates),
                         required_by_owner=required,
                         remaining_deficit=admission.eviction_plan.remaining_deficit,
@@ -2801,6 +2852,7 @@ class HiRadixCache(RadixCache):
                 self._warn_cp_hicache_fallback(
                     "post_forward_catch_up_backup_failed",
                     "host_reservation_failed",
+                    rate_limit_s=10.0,
                     node_id=node.id,
                     logical_len=len(node.value) if node.value is not None else 0,
                     required_host_slots=result.required_host_slots,