chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

L3 3.1: redesign spill — 2-phase staging pipeline + proactive FIFO (fix evicted=0 starvation)

Browse Source 
The first 3.1 (d08ee9f8d) spilled the COLDEST resident objects and held the
protect_host eviction-pin through the entire slow disk write (released only at the
durable ack). Since the coldest objects ARE eviction's victims, a B300 cachebench
wire test wedged: evict-to-fit found everything pinned (evicted=0) ->
host_reservation_failed flood -> caching broke.

Redesign (docs_internal/cp_hicache_l3_phase3_impl_design.md §R3):
- cp_l3_store: split the single spill thread into a GATHER -> WRITE pipeline. Gather
  copies each owned page off the pinned slab into a slab-independent anonymous-mmap
  staging buffer (fast RAM memcpy) and acks GATHER; a separate write thread does the
  O_DIRECT write + fdatasync + durable LMDB put off the staged copy and acks DURABLE.
  The eviction-pin releases at the GATHER ack, so the object is evictable right after
  the RAM copy; the disk write completes off the staged copy even if L2 reuses the
  pages. Slab-full frees its partial slot allocation (no orphan) + acks ok=False.
- hiradix: continuous PROACTIVE spill — enqueue each just-committed object to a FIFO
  deque at the replicated commit frontier (_commit_pending_backup) AND at radix splits
  (the freshly-committed parent half), drained in commit order (not coldest). The deque
  is capped (rank-uniform) so a disk stall can't OOM it. Spilling HOT just-committed
  objects (disjoint from eviction's COLD victims) means the cold tail is already
  L3-durable when evicted -> eviction just drops it, never contends for the pin.
- 3-element CP-group MIN drain [gather, durable, reload]: gather-MIN -> release_host;
  durable-MIN -> l3_durable, gated by an ok-AND (a second MIN over per-op ok bits) so a
  write failure on any rank means NO rank marks it durable (rank-uniform durability,
  placement_digest stays green).

opus-reviewed (FIX-THEN-SHIP) + independently re-verified; review fixes folded:
split-enqueue (HIGH), deque cap for the disk-stall OOM (MED), fail-soft-on-write-failure
documented (HIGH; deliberately NOT re-enqueued — re-gather would churn under slab-full,
recompute is correctness-safe), rate-limited error logs (MED/LOW). 54 L3 unit tests
green (new: 2-phase gather-before-durable ordering, slab-full fail-soft + no slot leak).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
leavelet
2026-06-22 22:15:48 +00:00
co-authored by Claude Opus 4.8
parent f353cf4702
commit 815da6c4e5
3 changed files with 351 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/mem_cache/test_cp_l3_store.py
+61 -15
View File
@@ -93,15 +93,30 @@ class TestCpL3Store(unittest.TestCase):
self.store.close()
self._td.cleanup()
def _finish_spill(self):
"""Drive the 2-phase spill to completion (store-level; no cross-rank MIN). Returns durable acks
as [(op_id, ok), ...]. Draining the gather-ack is what releases the eviction pin in production."""
self.assertTrue(_wait_ack(self.store.ack_durable_qsize, 1))
self.store.drain_gather_acks(self.store.ack_gather_qsize())
return self.store.drain_durable_acks(self.store.ack_durable_qsize())
def test_two_phase_gather_precedes_durable(self):
# the core 3.1 fix: gather (pin release) acks BEFORE the slow durable write; inflight until durable.
op_id = self.store.submit_spill("obj-2p", {"target_kv": [(0, _h(0))]}, last_access=1)
self.assertTrue(_wait_ack(self.store.ack_durable_qsize, 1)) # durable implies gather already acked
self.assertGreaterEqual(self.store.ack_gather_qsize(), 1)
self.assertEqual(self.store.drain_gather_acks(self.store.ack_gather_qsize()), [op_id])
self.assertTrue(self.store.has_inflight()) # gather drained, but still inflight until durable
self.assertEqual(self.store.drain_durable_acks(self.store.ack_durable_qsize()), [(op_id, True)])
self.assertFalse(self.store.has_inflight())
def test_spill_exists_reload_roundtrip(self):
# spill pages 2 and 5
pages = {"target_kv": [(2, _h(2)), (5, _h(5))]}
orig2 = self.acc.gather(2)
orig5 = self.acc.gather(5)
self.store.submit_spill("obj-A", pages, last_access=10)
self.assertTrue(_wait_ack(self.store.ack_spill_qsize, 1))
acks = self.store.drain_spill_acks(self.store.ack_spill_qsize())
self.assertEqual(acks, [(1, True)])
op_id = self.store.submit_spill("obj-A", pages, last_access=10)
self.assertEqual(self._finish_spill(), [(op_id, True)])
self.assertFalse(self.store.has_inflight())
# exists_prefix: consecutive present count
@@ -125,38 +140,69 @@ class TestCpL3Store(unittest.TestCase):
self.assertEqual(self.acc.gather(5), orig5)
def test_zero_owned_spill_acks_in_lockstep(self):
# a rank that owns no pages of the object still must ack (empty op)
# a rank that owns no pages of the object still must ack (empty op) -- both phases
op_id = self.store.submit_spill("obj-empty", {"target_kv": []}, last_access=1)
self.assertTrue(_wait_ack(self.store.ack_spill_qsize, 1))
self.assertEqual(self.store.drain_spill_acks(1), [(op_id, True)])
self.assertEqual(self._finish_spill(), [(op_id, True)])
self.assertFalse(self.store.has_inflight())
def test_dedup_skip_existing(self):
pages = {"target_kv": [(3, _h(3))]}
self.store.submit_spill("o1", pages, last_access=1)
self.assertTrue(_wait_ack(self.store.ack_spill_qsize, 1))
self.store.drain_spill_acks(1)
self._finish_spill()
free_after_first = self.store.pools["target_kv"].num_free
# spill the SAME hash again -> dedup, no new slot consumed
self.store.submit_spill("o1b", pages, last_access=2)
self.assertTrue(_wait_ack(self.store.ack_spill_qsize, 1))
self.store.drain_spill_acks(1)
self._finish_spill()
self.assertEqual(self.store.pools["target_kv"].num_free, free_after_first)
def test_free_object_releases_slots_and_index(self):
pages = {"target_kv": [(1, _h(1)), (4, _h(4))]}
before = self.store.pools["target_kv"].num_free
self.store.submit_spill("o", pages, last_access=1)
self.assertTrue(_wait_ack(self.store.ack_spill_qsize, 1))
self.store.drain_spill_acks(1)
self._finish_spill()
self.assertEqual(self.store.pools["target_kv"].num_free, before - 2)
self.store.free_object(pages)
self.assertEqual(self.store.pools["target_kv"].num_free, before)
self.assertEqual(self.store.exists_prefix([_h(1)], ["target_kv"]), 0)
def test_slab_full_fail_soft_frees_partial_alloc(self):
# _write_object allocs ALL of an object's slots before writing; a slab-full mid-object must free the
# slots it already took (no orphan) and ack durable ok=False (fail-soft, not a crash). Tiny budget.
td = tempfile.TemporaryDirectory()
self.addCleanup(td.cleanup)
cfg = cfg_mod.CpL3Config.from_dict({
"backend": "posix", "require_plp": False, "index_map_gb": 0.05,
"disks": [{"path": os.path.join(td.name, "d"), "budget_gb": 0.00002}],
})
store = store_mod.CpL3Store.from_config(
cfg, cp_rank=0, cp_size=1, accessors={"target_kv": self.acc})
store.connect(cfg)
self.addCleanup(store.close)
pool = store.pools["target_kv"]
n = pool.num_slots
self.assertGreaterEqual(n, 2)
op = 0
for i in range(n - 1): # fill all but one slot (single-page objects, distinct hashes)
op += 1
store.submit_spill(f"o{i}", {"target_kv": [(i % PAGE_NUM, _h(2000 + i))]}, last_access=i)
self.assertTrue(_wait_ack(store.ack_durable_qsize, 1))
store.drain_gather_acks(store.ack_gather_qsize())
self.assertEqual(store.drain_durable_acks(store.ack_durable_qsize()), [(op, True)])
self.assertEqual(pool.num_free, 1)
# 2-distinct-page object with only 1 free slot: page-1 allocs, page-2 fails -> whole op ok=False,
# and the page-1 slot is freed (num_free back to 1, no leak).
op += 1
store.submit_spill(
"overflow", {"target_kv": [(0, _h(8001)), (1, _h(8002))]}, last_access=op)
self.assertTrue(_wait_ack(store.ack_durable_qsize, 1))
store.drain_gather_acks(store.ack_gather_qsize())
self.assertEqual(store.drain_durable_acks(store.ack_durable_qsize()), [(op, False)])
self.assertEqual(pool.num_free, 1) # no orphan: the partial alloc was rolled back
self.assertFalse(store.has_inflight())
def test_clear_resets(self):
self.store.submit_spill("o", {"target_kv": [(0, _h(0))]}, last_access=1)
self.assertTrue(_wait_ack(self.store.ack_spill_qsize, 1))
self.store.drain_spill_acks(1)
self._finish_spill()
self.store.clear()
self.assertEqual(self.store.exists_prefix([_h(0)], ["target_kv"]), 0)
self.assertEqual(self.store.pools["target_kv"].num_free, self.store.pools["target_kv"].num_slots)