Keep CP HiCache free room from collapsing under cache reuse
Repeated GSM8K/cache-hit traffic can drain a CP owner lane while exact allocation still succeeds. This changes L1/L2 free-room logic so the trigger accounts for the pending allocation, then evicts a minimum target chunk or the exact shortage, whichever is larger. CP load-back also performs best-effort owner-lane free-room eviction before admitting tiny cache hits that would otherwise skew one lane. The commit also adds gated bs>1 prefill timing markers around recv/bootstrap/batch/inflight polling so future hangs expose the scheduler boundary instead of only surfacing as a watchdog shutdown. The post-fix repeat GSM8K failure is recorded as an active regression to continue investigating, not as old-process noise. Constraint: Free-room policy must reduce repeated eviction and owner-lane starvation without reserving required+target pages after every allocation Rejected: Evict to required+target availability | wastes L1/L2 residency under many short cache hits Rejected: Treat free-room misses as fatal on load-back | exact capacity should remain the strict admission condition Confidence: medium Scope-risk: moderate Directive: Do not remove the gated bs>1 timing markers until repeat-GSM8K hangs have a direct failing boundary Tested: git diff --check Tested: python -m py_compile for touched Python files Tested: remote earlier test_cp_shared_kv_layout.py and test_cp_hicache_metadata.py passed 157 tests after this free-room formula Not-tested: Two full repeat GSM8K runs after this commit; latest repeat still killed prefill and requires follow-up root-cause work
This commit is contained in:
@@ -631,7 +631,22 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
|
||||
trigger_ratio=0.25,
|
||||
)
|
||||
|
||||
self.assertEqual(deficit, 96)
|
||||
self.assertEqual(deficit, 128)
|
||||
|
||||
def test_free_room_deficit_trigger_accounts_for_pending_reservation(self):
|
||||
deficit = hiradix_cache._free_room_deficit(
|
||||
required=64,
|
||||
available=96,
|
||||
capacity=256,
|
||||
page_size=64,
|
||||
target_ratio=0.5,
|
||||
trigger_ratio=0.25,
|
||||
)
|
||||
|
||||
# available=96 is above trigger_room=64, but after reserving required=64
|
||||
# only 32 tokens would remain. Trigger checks available - required,
|
||||
# then evicts at least target_room=128 tokens.
|
||||
self.assertEqual(deficit, 128)
|
||||
|
||||
def test_free_room_deficit_rounds_room_to_page(self):
|
||||
deficit = hiradix_cache._free_room_deficit(
|
||||
@@ -643,7 +658,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
|
||||
trigger_ratio=0.0,
|
||||
)
|
||||
|
||||
self.assertEqual(deficit, 128)
|
||||
self.assertEqual(deficit, 64)
|
||||
|
||||
def test_cp_host_write_admission_uses_trigger_target_room(self):
|
||||
cache = HiRadixCache.__new__(HiRadixCache)
|
||||
@@ -672,7 +687,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(admission.target_available_by_owner, (96, 256))
|
||||
self.assertEqual(admission.deficit_by_owner, (96, 0))
|
||||
self.assertEqual(admission.deficit_by_owner, (128, 0))
|
||||
|
||||
def test_cp_host_write_admission_does_not_evict_when_trigger_room_fits(self):
|
||||
cache = HiRadixCache.__new__(HiRadixCache)
|
||||
@@ -706,7 +721,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
|
||||
def test_cp_host_write_admission_uses_draft_room_deficit(self):
|
||||
cache = HiRadixCache.__new__(HiRadixCache)
|
||||
cache.page_size = 64
|
||||
cache.hicache_host_free_room_ratio = 0.25
|
||||
cache.hicache_host_free_room_ratio = 0.5
|
||||
cache.hicache_host_free_room_trigger_ratio = 0.25
|
||||
cache._cp_required_host_tokens_by_rank = lambda _indices: (64, 0)
|
||||
cache._cp_host_capacity_snapshot = lambda: CpHiCacheCapacitySnapshot(
|
||||
@@ -731,7 +746,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
|
||||
|
||||
self.assertEqual(admission.target_available_by_owner, (256, 256))
|
||||
self.assertEqual(admission.draft_available_by_owner, (64, 256))
|
||||
self.assertEqual(admission.deficit_by_owner, (64, 0))
|
||||
self.assertEqual(admission.deficit_by_owner, (128, 0))
|
||||
|
||||
def test_cp_host_write_batch_admission_uses_aggregate_required(self):
|
||||
cache = HiRadixCache.__new__(HiRadixCache)
|
||||
@@ -761,7 +776,7 @@ class TestCpHiCacheFreeRoom(CustomTestCase):
|
||||
self.assertEqual(admission.required_by_owner, (128, 0))
|
||||
self.assertEqual(admission.target_available_by_owner, (96, 256))
|
||||
# required=128, target_room=128, available=96.
|
||||
self.assertEqual(admission.deficit_by_owner, (160, 0))
|
||||
self.assertEqual(admission.deficit_by_owner, (128, 0))
|
||||
|
||||
def test_prepare_write_backups_for_reqs_runs_one_batch_admission(self):
|
||||
cache = HiRadixCache.__new__(HiRadixCache)
|
||||
@@ -2873,7 +2888,7 @@ class TestHiRadixCacheCPLoadBack(CustomTestCase):
|
||||
self.assertEqual(plan.deficit_by_owner, [1, 0])
|
||||
# The free-room target is still reported for observability/proactive policy.
|
||||
# required=1 page, available=0, target_room=ceil(8*0.5)=4 pages.
|
||||
self.assertEqual(plan.free_room_deficit_by_owner, [5, 0])
|
||||
self.assertEqual(plan.free_room_deficit_by_owner, [4, 0])
|
||||
|
||||
def test_cp_load_back_uses_host_len_not_host_value(self):
|
||||
cache = HiRadixCache.__new__(HiRadixCache)
|
||||
|
||||
Reference in New Issue
Block a user