L3 3.3: continuous-background, communication-free, page-level LRU GC (+ wire the reload touch)

Replaces §9.3's object-level rank-uniform eviction with per-rank, page-level LRU GC running ON THE
WRITE THREAD (the sole slot-pool owner -> no lock, no second owner). Genuinely communication-free:
each (hash,payload) is written/read/deleted by exactly its one owner (owner = i%cp_size, and the
prefix-chained content hash makes one-hash = one-position = one-owner), so the shared LMDB's own
consistency substitutes for collectives within L3 (design §9.5/§9.6). GC touches only L3, adds zero
collectives; the spill/reload MINs that remain are all the L2 bridge.

cp_l3_store.py:
- Per-payload lazy-deletion min-heap `_gc_heap` of (last_access, seq, hash) + authoritative
  `_gc_current{hash->(last_access,slot)}`. Add on write, touch via `_touch_q`, reclaim coldest.
- The write loop, each iteration: drain touches -> _gc_collect (reclaim coldest of any pool over the
  0.90 start watermark down to 0.85, bounded budget) BEFORE the next write, so spill never hits a full
  pool (no evict burst on the critical path). Continuous + proactive, symmetric with the spill.
- Reclaim deletes the index entry (durable) BEFORE freeing the slot, so a racing reload reads None or a
  CRC-mismatched reused slot -> miss -> recompute (fail-soft); no lock / in-flight-exclusion needed.
- submit_touch (scheduler->write thread); clear() resets the GC structures.

hiradix_cache.py: wire the reload `touch` (the 3.2 gap -- hit_count/last_access never updated): at
_cp_l3_admit_reload, bump the L3 last_access of THIS rank's owned reloaded pages (ALL pages, not just
the tail, so a prefix ages uniformly). _cp_l3_insert_reloaded_node now returns the node (its
last_access seeds the touch).

Sharing handled optimally for free (page = one entry, last_access = max over touchers; prefix pages
age together; shared-hot survives) -- no object-record table, no refcount, no content-keying. 56 L3
unit tests green (new: GC reclaims coldest-to-watermark + no slot leak; touch keeps a page warm).
Cold-rebuild (scan slab headers + LMDB) is designed (§9.6) but deferred to 3.4 (crash-recovery).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

test/registered/unit/mem_cache/test_cp_l3_store.py

@@ -93,12 +93,13 @@ class TestCpL3Store(unittest.TestCase):
         self.store.close()
         self._td.cleanup()
 
-    def _finish_spill(self):
+    def _finish_spill(self, store=None):
         """Drive the 2-phase spill to completion (store-level; no cross-rank MIN). Returns durable acks
         as [(op_id, ok), ...]. Draining the gather-ack is what releases the eviction pin in production."""
-        self.assertTrue(_wait_ack(self.store.ack_durable_qsize, 1))
-        self.store.drain_gather_acks(self.store.ack_gather_qsize())
-        return self.store.drain_durable_acks(self.store.ack_durable_qsize())
+        st = store if store is not None else self.store
+        self.assertTrue(_wait_ack(st.ack_durable_qsize, 1))
+        st.drain_gather_acks(st.ack_gather_qsize())
+        return st.drain_durable_acks(st.ack_durable_qsize())
 
     def test_two_phase_gather_precedes_durable(self):
         # the core 3.1 fix: gather (pin release) acks BEFORE the slow durable write; inflight until durable.
@@ -200,6 +201,77 @@ class TestCpL3Store(unittest.TestCase):
         self.assertEqual(pool.num_free, 1)  # no orphan: the partial alloc was rolled back
         self.assertFalse(store.has_inflight())
 
+    def test_gc_reclaims_coldest_to_stop_watermark(self):
+        # 3.3 continuous-background GC: filling a pool past the start watermark triggers the write-thread GC to
+        # reclaim the COLDEST (lowest last_access) owned pages down to the stop watermark, deleting their index
+        # entries, with no slot leak. Small pool so the watermark trips deterministically.
+        td = tempfile.TemporaryDirectory()
+        self.addCleanup(td.cleanup)
+        cfg = cfg_mod.CpL3Config.from_dict({
+            "backend": "posix", "require_plp": False, "index_map_gb": 0.05,
+            "disks": [{"path": os.path.join(td.name, "d"), "budget_gb": 0.0001}],
+        })
+        store = store_mod.CpL3Store.from_config(
+            cfg, cp_rank=0, cp_size=1, accessors={"target_kv": self.acc})
+        store.connect(cfg)
+        self.addCleanup(store.close)
+        pool = store.pools["target_kv"]
+        n = pool.num_slots
+        self.assertGreaterEqual(n, 8)
+        start = int(store._gc_start_frac * n)
+        stop = int(store._gc_stop_frac * n)
+        # spill n distinct pages, page i with last_access=i (so smaller i == colder)
+        for i in range(n):
+            store.submit_spill(f"o{i}", {"target_kv": [(i % PAGE_NUM, _h(3000 + i))]}, last_access=i)
+            self._finish_spill(store)
+        # the write-thread GC keeps usage within the watermark band [stop, start] (reclaim-to-stop on trigger)
+        t0 = time.time()
+        while pool.num_allocated > start and time.time() - t0 < 5.0:
+            time.sleep(0.01)
+        allocated = pool.num_allocated
+        self.assertLessEqual(allocated, start)
+        self.assertGreaterEqual(allocated, stop)
+        self.assertEqual(pool.num_free + allocated, n)  # no slot leak
+        n_evicted = n - allocated
+        self.assertGreater(n_evicted, 0)
+        for i in range(n_evicted):  # the coldest n_evicted pages were reclaimed (index entries deleted)
+            self.assertEqual(store.exists_prefix([_h(3000 + i)], ["target_kv"]), 0)
+        for i in range(n_evicted, n):  # the warmest survive
+            self.assertEqual(store.exists_prefix([_h(3000 + i)], ["target_kv"]), 1)
+
+    def test_gc_touch_keeps_a_page_warm(self):
+        # submit_touch bumps a page's last_access so it survives GC while colder untouched pages are reclaimed.
+        td = tempfile.TemporaryDirectory()
+        self.addCleanup(td.cleanup)
+        cfg = cfg_mod.CpL3Config.from_dict({
+            "backend": "posix", "require_plp": False, "index_map_gb": 0.05,
+            "disks": [{"path": os.path.join(td.name, "d"), "budget_gb": 0.0001}],
+        })
+        store = store_mod.CpL3Store.from_config(
+            cfg, cp_rank=0, cp_size=1, accessors={"target_kv": self.acc})
+        store.connect(cfg)
+        self.addCleanup(store.close)
+        pool = store.pools["target_kv"]
+        n = pool.num_slots
+        start = int(store._gc_start_frac * n)
+        stop = int(store._gc_stop_frac * n)
+        self.assertGreaterEqual(start, 4)
+        # fill below the start watermark (no GC yet), page i last_access=i
+        for i in range(start):
+            store.submit_spill(f"o{i}", {"target_kv": [(i % PAGE_NUM, _h(4000 + i))]}, last_access=i)
+            self._finish_spill(store)
+        # touch the coldest page (i=0) to the warmest -> it must survive the upcoming GC
+        store.submit_touch([(_h(4000 + 0), "target_kv", 10_000)])
+        # now push over the start watermark -> GC reclaims coldest down to stop
+        for i in range(start, n + 1):
+            store.submit_spill(f"o{i}", {"target_kv": [(i % PAGE_NUM, _h(4000 + i))]}, last_access=i)
+            self._finish_spill(store)
+        t0 = time.time()
+        while pool.num_allocated > start and time.time() - t0 < 5.0:
+            time.sleep(0.01)
+        self.assertEqual(store.exists_prefix([_h(4000 + 0)], ["target_kv"]), 1)  # touched -> warm -> survived
+        self.assertEqual(store.exists_prefix([_h(4000 + 1)], ["target_kv"]), 0)  # next-coldest -> reclaimed
+
     def test_clear_resets(self):
         self.store.submit_spill("o", {"target_kv": [(0, _h(0))]}, last_access=1)
         self._finish_spill()