Enforce one in-flight CP write ack per node at the producer

dfa168abe9 made duplicate ack completion idempotent at the consumer
(writing_check), which is correct and lock-balance-safe but masks the
producer invariant breach: two acks can only coexist for one radix
registration when an ack is orphaned in ack_write_queue after a
rollback cleared ongoing_write_through/pending_host_backups,
re-opening the _node_host_write_pending guard for a fresh
registration. _rollback_pending_backup (write_backup's exception
path) was the one rollback that did not scrub the node's acks — and
it also left a half-submitted layer-write state alive, which later
forwards would keep driving, writing D2H into host slots the rollback
had already evicted.

Close the invariant structurally:
- reserve_write_cp refuses (HiCacheWriteFailure, the existing
  skip-this-round path both callers already handle) when the node
  still has an undrained final ack, computed by scanning the small
  ack queue — no new state to keep in sync.
- _rollback_pending_backup now cancels the pending layer-write state
  (write-stream sync so in-flight per-layer copies finish before the
  host slots are evicted) and scrubs the node's queued acks via the
  scrub factored out of _rollback_prepared_cp_backup.
- The consumer-side duplicate guard from dfa168abe9 is kept as
  defense in depth.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

python/sglang/srt/managers/cache_controller.py

@@ -964,6 +964,37 @@ class HiCacheController:
                 device_indices, self.page_size, "physical_device_indices"
             )
 
+    def node_has_undrained_write_ack(self, node_id: int) -> bool:
+        """Whether a final write ack for this node is still in ack_write_queue.
+
+        The radix side drains acks via writing_check; until then the node's
+        previous backup owns radix/host state. Creating a second layer-write
+        state for the same node while its ack is undrained would put two acks
+        for one radix registration into the queue (one inc_lock_ref, two
+        completions) — the invariant is at most one in-flight ack per node.
+        """
+        ack_write_queue = getattr(self, "ack_write_queue", None)
+        if not ack_write_queue:
+            return False
+        return any(node_id in ack.node_ids for ack in ack_write_queue)
+
+    def cancel_layer_write_state(self, node_id: int) -> bool:
+        """Drop a pending per-layer write state during backup rollback.
+
+        Without this, an exception between reservation and final ack leaves a
+        half-submitted state in pending_layer_writes: the next forward's layer
+        hooks keep driving it, eventually writing D2H into host slots the
+        rollback already evicted and appending an ack for a node the radix
+        side no longer has registered. Synchronize the write stream so any
+        already-issued per-layer copies finish before the caller frees the
+        host slots.
+        """
+        state = self.pending_layer_writes.pop(node_id, None)
+        if state is None:
+            return False
+        self.write_stream.synchronize()
+        return True
+
     def reserve_write_cp(
         self,
         device_indices: torch.Tensor,
@@ -972,6 +1003,17 @@ class HiCacheController:
     ) -> HiCacheWriteReservation | HiCacheWriteFailure:
         from sglang.srt.mem_cache.hiradix_cache import CpHiCacheNodeMetadata
 
+        if node_id >= 0 and self.node_has_undrained_write_ack(node_id):
+            # Refusing here (instead of crashing later in writing_check) keeps
+            # the failure on the existing HiCacheWriteFailure path: the caller
+            # skips this backup round and may retry after the ack drains.
+            logger.warning(
+                "[CacheCtrl-write] reserve_write_cp refused: node_id=%d still "
+                "has an undrained write ack; skipping duplicate backup",
+                node_id,
+            )
+            return HiCacheWriteFailure(required_host_slots=0)
+
         page_size = self.page_size
         layout = self.cp_shared_kv_layout
         logical_len = len(device_indices)

python/sglang/srt/mem_cache/hiradix_cache.py

@@ -2090,9 +2090,39 @@ class HiRadixCache(RadixCache):
             self.dec_node_lock_ref(node)
         return node
 
+    def _remove_undrained_write_acks(self, node_id: int) -> bool:
+        """Scrub this node's final write acks out of the controller queue.
+
+        Every rollback that clears the radix-side registration
+        (ongoing_write_through / pending_host_backups) MUST also scrub the
+        node's acks: an orphaned ack re-opens the _node_host_write_pending
+        guard for a fresh registration while the stale ack is still queued,
+        producing two acks for one registration (one inc_lock_ref, two
+        completions) and crashing writing_check on the second.
+        """
+        retained_acks = []
+        removed = False
+        for ack in self.cache_controller.ack_write_queue:
+            if node_id not in ack.node_ids:
+                retained_acks.append(ack)
+                continue
+            ack.finish_event.synchronize()
+            remaining_node_ids = [nid for nid in ack.node_ids if nid != node_id]
+            if remaining_node_ids:
+                retained_acks.append(ack._replace(node_ids=remaining_node_ids))
+            removed = True
+        if removed:
+            self.cache_controller.ack_write_queue = retained_acks
+        return removed
+
     def _rollback_pending_backup(self, node_id: int) -> TreeNode:
         pending = self.pending_host_backups.pop(node_id)
         node = pending.node
+        # Cancel the half-submitted per-layer state (so later layer hooks
+        # cannot write into the host slots evicted below) and scrub any
+        # already-appended final ack before releasing the registration.
+        self.cache_controller.cancel_layer_write_state(node_id)
+        self._remove_undrained_write_acks(node_id)
         self.cache_controller.evict_cp_host(pending.metadata)
         if node.cp_hicache is pending.metadata:
             node.cp_hicache = None
@@ -2271,21 +2301,7 @@ class HiRadixCache(RadixCache):
                 f"while per-layer writes are still pending; reason={reason}"
             )
 
-        retained_acks = []
-        removed_ack = False
-        for ack in self.cache_controller.ack_write_queue:
-            if prepared.node_id not in ack.node_ids:
-                retained_acks.append(ack)
-                continue
-            ack.finish_event.synchronize()
-            remaining_node_ids = [
-                node_id for node_id in ack.node_ids if node_id != prepared.node_id
-            ]
-            if remaining_node_ids:
-                retained_acks.append(ack._replace(node_ids=remaining_node_ids))
-            removed_ack = True
-        if removed_ack:
-            self.cache_controller.ack_write_queue = retained_acks
+        self._remove_undrained_write_acks(prepared.node_id)
         logger.debug(
             "[HiCache-write] rollback unattached prepared CP backup: node_id=%d logical_len=%d reason=%s",
             prepared.node_id,