From 8d73919d02c50a5fb89a25a1b44b9f292b4ced38 Mon Sep 17 00:00:00 2001 From: leavelet Date: Fri, 19 Jun 2026 12:14:36 +0000 Subject: [PATCH] Gate CP HiCache write-backup on hard deficit, not the free-room watermark A long cold-request workload flooded the prefill log (~177MB / 32K lines in 8min, all 8 CP ranks) with cp_host_reservation_plan_insufficient + prepare_write_backup_reservation_failed even though the host pool was <42% used and the writes physically fit. Root cause: _free_room_deficit folds the proactive 20% free-room target (hicache_host_free_room_ratio) into the admission deficit once a lane dips below the 10% trigger. _evict_cp_host_for_write_admission then failed -- and evicted nothing -- whenever it could not reclaim that full 20% target, which is impossible while the residual is locked/pending. The lane never drained, so every subsequent backup re-issued the same impossible demand and re-logged on every tick. The reservation failure is itself a graceful skip, so there is no crash; the symptom is the log storm. Gate write-backup admission on the HARD deficit -- max(0, required-available) over the target and draft lanes (draft_available is 2**62 with no draft pool, a no-op) -- and evict toward the watermark best-effort, admitting whenever the write itself fits. This drains the stuck lane and backs the request up instead of skipping+flooding. The proactive watermark stays as the eviction target (deficit_by_owner unchanged); it is no longer a hard admission gate. The decision is computed from rank-replicated state on the collective-free reserve path, so every CP rank makes the same admit/skip choice (opus-reviewed rank-safe). Also rate-limit the four host-reservation fallback warnings (once per 10s per fallback_name, with a suppressed count) so any genuine exhaustion degrades quietly instead of producing a log storm. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/sglang/srt/mem_cache/hiradix_cache.py | 78 +++++++++++++++---- .../mem_cache/test_cp_hicache_metadata.py | 73 +++++++++++++++++ 2 files changed, 138 insertions(+), 13 deletions(-) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 76e27d51c..3e6059c7c 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -2135,21 +2135,45 @@ class HiRadixCache(RadixCache): self, admission: CpWriteAdmission, *, node_id: int, phase: str ) -> bool: eviction_plan = admission.eviction_plan - if any(v > 0 for v in eviction_plan.remaining_deficit): - logger.warning( - "[CP_HICACHE_FALLBACK][cp_host_reservation_plan_insufficient] " - "reason=insufficient_evictable_host_slots node_id=%d phase=%s " - "required=%s target_avail=%s draft_avail=%s deficit=%s " - "planned_freed=%s remaining_deficit=%s victims=%s", - node_id, - phase, + # Only the HARD deficit -- what the backup actually needs in order to fit + # -- may block admission. admission.deficit_by_owner (hence + # eviction_plan.remaining_deficit) also folds in the PROACTIVE free-room + # watermark (hicache_host_free_room_ratio): once a lane dips below the + # trigger watermark that eviction target balloons to ~20% of capacity. + # Refusing a backup that physically fits, just because a lane is below + # that advisory target, turned a ~1k-slot write into a 20%-of-capacity + # reclaim demand; when locked residual made the target unreachable it + # skipped the backup AND re-fired every tick on every rank (a 177MB log + # storm with the host pool <42% used, 2026-06-19). Evict toward the + # target best-effort, but admit whenever the write itself fits. + # draft_available is 2**62 when there is no draft pool, so the draft term + # is a no-op in that case. + hard_deficit = [ + max(max(0, int(req) - int(tgt)), max(0, int(req) - int(drf))) + for req, tgt, drf in zip( admission.required_by_owner, admission.target_available_by_owner, admission.draft_available_by_owner, - admission.deficit_by_owner, - eviction_plan.planned_freed, - eviction_plan.remaining_deficit, - [getattr(node, "id", None) for node in eviction_plan.victims], + ) + ] + remaining_hard = [ + max(0, hard - int(freed)) + for hard, freed in zip(hard_deficit, eviction_plan.planned_freed) + ] + if any(v > 0 for v in remaining_hard): + self._warn_cp_hicache_fallback( + "cp_host_reservation_plan_insufficient", + "insufficient_evictable_host_slots", + rate_limit_s=10.0, + node_id=node_id, + phase=phase, + hard_deficit=hard_deficit, + required=admission.required_by_owner, + target_avail=admission.target_available_by_owner, + draft_avail=admission.draft_available_by_owner, + planned_freed=eviction_plan.planned_freed, + remaining_hard=remaining_hard, + victims=len(eviction_plan.victims), ) return False @@ -2316,10 +2340,35 @@ class HiRadixCache(RadixCache): ) self.cache_controller.evict_cp_host(prepared.metadata) - def _warn_cp_hicache_fallback(self, fallback_name: str, reason: str, **kwargs): + def _warn_cp_hicache_fallback( + self, fallback_name: str, reason: str, *, rate_limit_s: float = 0.0, **kwargs + ): details = " ".join(f"{key}={value}" for key, value in kwargs.items()) if details: details = " " + details + if rate_limit_s > 0.0: + # Host-reservation shortfalls are emitted per backup-attempt per rank; + # under genuine host pressure that is thousands of identical lines a + # minute (a 177MB log storm, 2026-06-19). Collapse repeats of the same + # fallback_name within the window so a real shortfall stays loud but + # bounded -- the first line of each window carries the count suppressed + # since the previous emission. + warn_state = getattr(self, "_cp_warn_state", None) + if warn_state is None: + warn_state = {} + self._cp_warn_state = warn_state + now = time.monotonic() + entry = warn_state.get(fallback_name) + if entry is not None and (now - entry[0]) < rate_limit_s: + entry[1] += 1 + return + suppressed = entry[1] if entry is not None else 0 + warn_state[fallback_name] = [now, 0] + if suppressed: + details = ( + f"{details} suppressed_since_last={suppressed} " + f"window_s={int(rate_limit_s)}" + ) logger.warning( "[CP_HICACHE_FALLBACK][%s] reason=%s%s", fallback_name, @@ -2667,6 +2716,7 @@ class HiRadixCache(RadixCache): self._warn_cp_hicache_fallback( "prepare_write_backup_reservation_failed", "host_reservation_failed", + rate_limit_s=10.0, node_id=node_id, rid=getattr(req, "rid", ""), logical_len=len(kv_indices), @@ -2757,6 +2807,7 @@ class HiRadixCache(RadixCache): self._warn_cp_hicache_fallback( "prepare_write_backups_batch_admission_failed", "host_reservation_failed", + rate_limit_s=10.0, batch_size=len(candidates), required_by_owner=required, remaining_deficit=admission.eviction_plan.remaining_deficit, @@ -2801,6 +2852,7 @@ class HiRadixCache(RadixCache): self._warn_cp_hicache_fallback( "post_forward_catch_up_backup_failed", "host_reservation_failed", + rate_limit_s=10.0, node_id=node.id, logical_len=len(node.value) if node.value is not None else 0, required_host_slots=result.required_host_slots, diff --git a/test/registered/unit/mem_cache/test_cp_hicache_metadata.py b/test/registered/unit/mem_cache/test_cp_hicache_metadata.py index 4884bd311..cd5fac5c4 100644 --- a/test/registered/unit/mem_cache/test_cp_hicache_metadata.py +++ b/test/registered/unit/mem_cache/test_cp_hicache_metadata.py @@ -805,6 +805,79 @@ class TestCpHiCacheFreeRoom(CustomTestCase): # required=128, target_room=128, available=96. self.assertEqual(admission.deficit_by_owner, (128, 0)) + def test_evict_admission_admits_write_that_fits_below_watermark(self): + # Regression (2026-06-19 host-reservation log storm): a backup that + # physically fits (required <= available, hard deficit 0) must be ADMITTED + # even when the lane is below the proactive free-room watermark and that + # advisory eviction target (deficit_by_owner / remaining_deficit) cannot be + # met. The old gate keyed off remaining_deficit and returned False here, + # skipping the backup AND re-firing the warning every tick on every rank. + cache = HiRadixCache.__new__(HiRadixCache) + admission = CpWriteAdmission( + node_id=1, + phase="unit", + required_by_owner=(100, 100), + target_available_by_owner=(500, 500), # write fits -> hard deficit 0 + draft_available_by_owner=(2**62, 2**62), # no draft pool sentinel + deficit_by_owner=(700000, 700000), # proactive watermark target + eviction_plan=CpHiCacheEvictionPlan( + victims=(), + planned_freed=(0, 0), + remaining_deficit=(700000, 700000), # watermark unreachable + ), + ) + self.assertTrue( + cache._evict_cp_host_for_write_admission( + admission, node_id=1, phase="unit" + ) + ) + + def test_evict_admission_fails_when_write_does_not_fit(self): + # When the backup genuinely does not fit (required > available) and + # eviction cannot free enough, admission must still fail (skip the backup). + cache = HiRadixCache.__new__(HiRadixCache) + admission = CpWriteAdmission( + node_id=2, + phase="unit", + required_by_owner=(1000, 0), + target_available_by_owner=(100, 500), # hard deficit 900 on lane 0 + draft_available_by_owner=(2**62, 2**62), + deficit_by_owner=(900, 0), + eviction_plan=CpHiCacheEvictionPlan( + victims=(), + planned_freed=(0, 0), # freed nothing + remaining_deficit=(900, 0), + ), + ) + self.assertFalse( + cache._evict_cp_host_for_write_admission( + admission, node_id=2, phase="unit" + ) + ) + + def test_evict_admission_admits_when_eviction_covers_hard_deficit_only(self): + # Eviction that reaches the HARD deficit (required-available) but not the + # full proactive watermark target must still admit (best-effort drain). + cache = HiRadixCache.__new__(HiRadixCache) + admission = CpWriteAdmission( + node_id=3, + phase="unit", + required_by_owner=(1000, 0), + target_available_by_owner=(100, 500), # hard deficit 900 on lane 0 + draft_available_by_owner=(2**62, 2**62), + deficit_by_owner=(700000, 0), # watermark target + eviction_plan=CpHiCacheEvictionPlan( + victims=(), + planned_freed=(900, 0), # freed exactly the hard deficit + remaining_deficit=(699100, 0), # watermark not reached + ), + ) + self.assertTrue( + cache._evict_cp_host_for_write_admission( + admission, node_id=3, phase="unit" + ) + ) + def test_prepare_write_backups_for_reqs_runs_one_batch_admission(self): cache = HiRadixCache.__new__(HiRadixCache) cache.disable = False