Keep CP HiCache free room from collapsing under cache reuse

Repeated GSM8K/cache-hit traffic can drain a CP owner lane while exact
allocation still succeeds. This changes L1/L2 free-room logic so the trigger
accounts for the pending allocation, then evicts a minimum target chunk or the
exact shortage, whichever is larger. CP load-back also performs best-effort
owner-lane free-room eviction before admitting tiny cache hits that would
otherwise skew one lane.

The commit also adds gated bs>1 prefill timing markers around
recv/bootstrap/batch/inflight polling so future hangs expose the scheduler
boundary instead of only surfacing as a watchdog shutdown. The post-fix repeat
GSM8K failure is recorded as an active regression to continue investigating,
not as old-process noise.

Constraint: Free-room policy must reduce repeated eviction and owner-lane starvation without reserving required+target pages after every allocation
Rejected: Evict to required+target availability | wastes L1/L2 residency under many short cache hits
Rejected: Treat free-room misses as fatal on load-back | exact capacity should remain the strict admission condition
Confidence: medium
Scope-risk: moderate
Directive: Do not remove the gated bs>1 timing markers until repeat-GSM8K hangs have a direct failing boundary
Tested: git diff --check
Tested: python -m py_compile for touched Python files
Tested: remote earlier test_cp_shared_kv_layout.py and test_cp_hicache_metadata.py passed 157 tests after this free-room formula
Not-tested: Two full repeat GSM8K runs after this commit; latest repeat still killed prefill and requires follow-up root-cause work
This commit is contained in:
laoyao0822
2026-06-07 01:23:34 +08:00
parent 68defa3923
commit b17976b60d
9 changed files with 642 additions and 24 deletions
@@ -631,7 +631,22 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
trigger_ratio=0.25,
)
self.assertEqual(deficit, 96)
self.assertEqual(deficit, 128)
def test_free_room_deficit_trigger_accounts_for_pending_reservation(self):
deficit = hiradix_cache._free_room_deficit(
required=64,
available=96,
capacity=256,
page_size=64,
target_ratio=0.5,
trigger_ratio=0.25,
)
# available=96 is above trigger_room=64, but after reserving required=64
# only 32 tokens would remain. Trigger checks available - required,
# then evicts at least target_room=128 tokens.
self.assertEqual(deficit, 128)
def test_free_room_deficit_rounds_room_to_page(self):
deficit = hiradix_cache._free_room_deficit(
@@ -643,7 +658,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
trigger_ratio=0.0,
)
self.assertEqual(deficit, 128)
self.assertEqual(deficit, 64)
def test_cp_host_write_admission_uses_trigger_target_room(self):
cache = HiRadixCache.__new__(HiRadixCache)
@@ -672,7 +687,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
)
self.assertEqual(admission.target_available_by_owner, (96, 256))
self.assertEqual(admission.deficit_by_owner, (96, 0))
self.assertEqual(admission.deficit_by_owner, (128, 0))
def test_cp_host_write_admission_does_not_evict_when_trigger_room_fits(self):
cache = HiRadixCache.__new__(HiRadixCache)
@@ -706,7 +721,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
def test_cp_host_write_admission_uses_draft_room_deficit(self):
cache = HiRadixCache.__new__(HiRadixCache)
cache.page_size = 64
cache.hicache_host_free_room_ratio = 0.25
cache.hicache_host_free_room_ratio = 0.5
cache.hicache_host_free_room_trigger_ratio = 0.25
cache._cp_required_host_tokens_by_rank = lambda _indices: (64, 0)
cache._cp_host_capacity_snapshot = lambda: CpHiCacheCapacitySnapshot(
@@ -731,7 +746,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
self.assertEqual(admission.target_available_by_owner, (256, 256))
self.assertEqual(admission.draft_available_by_owner, (64, 256))
self.assertEqual(admission.deficit_by_owner, (64, 0))
self.assertEqual(admission.deficit_by_owner, (128, 0))
def test_cp_host_write_batch_admission_uses_aggregate_required(self):
cache = HiRadixCache.__new__(HiRadixCache)
@@ -761,7 +776,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
self.assertEqual(admission.required_by_owner, (128, 0))
self.assertEqual(admission.target_available_by_owner, (96, 256))
# required=128, target_room=128, available=96.
self.assertEqual(admission.deficit_by_owner, (160, 0))
self.assertEqual(admission.deficit_by_owner, (128, 0))
def test_prepare_write_backups_for_reqs_runs_one_batch_admission(self):
cache = HiRadixCache.__new__(HiRadixCache)
@@ -2873,7 +2888,7 @@ class TestHiRadixCacheCPLoadBack(CustomTestCase):
self.assertEqual(plan.deficit_by_owner, [1, 0])
# The free-room target is still reported for observability/proactive policy.
# required=1 page, available=0, target_room=ceil(8*0.5)=4 pages.
self.assertEqual(plan.free_room_deficit_by_owner, [5, 0])
self.assertEqual(plan.free_room_deficit_by_owner, [4, 0])
def test_cp_load_back_uses_host_len_not_host_value(self):
cache = HiRadixCache.__new__(HiRadixCache)