Revert "Fix CP HiCache writing_check deadlock at the source: rank-replicated symmetric attach"

This reverts commit 85585fcf4b.
This commit is contained in:
2026-06-22 21:35:27 +00:00
parent d08ee9f8d9
commit f353cf4702
3 changed files with 11 additions and 238 deletions

View File

@@ -2967,19 +2967,6 @@ class HiRadixCache(RadixCache):
self, "pending_host_backups", {}
)
def _node_backup_outstanding_replicated(self, node: TreeNode) -> bool:
"""Rank-replicated 'backup outstanding' signal for the CP radix probe path.
True from the rank-replicated prepared-backup attach until the MIN-committed
commit/rollback (see TreeNode.cp_backup_pending). Unlike
_node_host_write_pending (which reads the drain-timed ongoing_write_through/
pending_host_backups dicts and so carries per-rank ack-drain timing), this is
identical across CP ranks at every sub-tick instant, so split/defer/prune
decisions stay rank-uniform. The radix probe path must read THIS; only
writing_check / eviction may read the drain-timed _node_host_write_pending.
"""
return bool(getattr(node, "cp_backup_pending", False))
def _node_host_write_ready(self, node: TreeNode) -> bool:
return not self._node_host_write_pending(node)
@@ -3006,10 +2993,6 @@ class HiRadixCache(RadixCache):
node.cp_hicache = pending.metadata
node.host_value = None
self._cp_account_enter_committed(pending.metadata)
# Clear the replicated pending marker at the MIN-committed edge, BEFORE
# releasing the backup lock (keeps marker and lock_ref consistent). After
# commit, node.host_len > 0 (_node_backuped) is the authoritative signal.
node.cp_backup_pending = False
if pending.locked:
self.dec_node_lock_ref(node)
# B1: this commit is gated by the writing_check ReduceOp.MIN frontier, which
@@ -3071,7 +3054,6 @@ class HiRadixCache(RadixCache):
if node.cp_hicache is pending.metadata:
node.cp_hicache = None
node.host_len = 0
node.cp_backup_pending = False
if pending.locked:
self.dec_node_lock_ref(node)
return node
@@ -3360,13 +3342,7 @@ class HiRadixCache(RadixCache):
f"node_len={len(node.value)} prepared_len={prepared.logical_len}"
)
node.id = prepared.node_id
if getattr(node, "cp_backup_pending", False):
raise RuntimeError(
f"CP HiCache double-attach for node_id={node.id}: "
"cp_backup_pending already set"
)
self.ongoing_write_through[node.id] = node
node.cp_backup_pending = True
self.inc_node_lock_ref(node)
self.pending_host_backups[node.id] = PendingHiCacheBackup(
node=node,
@@ -3673,15 +3649,10 @@ class HiRadixCache(RadixCache):
stack = [node]
while stack:
current = stack.pop()
# Read only rank-replicated state: the backup-pending marker (covers the
# backup lock -- the sole rank-local lock_ref skew) and host_ref_counter
# (set by replicated pin/protect_host). Do NOT read lock_ref here: its only
# cross-rank skew is the backup lock, already covered by the marker, and a
# request-held lock is rank-uniform but irrelevant to splittability (the
# page-floor logic governs that). Reading it would re-leak per-rank timing.
if (
self._node_backup_outstanding_replicated(current)
current.lock_ref > 0
or current.host_ref_counter > 0
or self._node_host_write_pending(current)
):
return True
stack.extend(current.children.values())
@@ -3705,19 +3676,17 @@ class HiRadixCache(RadixCache):
return
def _cp_node_split_still_pending(self, node: TreeNode) -> bool:
# Rank-replicated marker only (see _node_backup_outstanding_replicated): the
# drain-timed _node_host_write_pending here is what let CP ranks diverge on the
# attach/defer fork. Do NOT call _cp_drain_write_acks_before_pending_split (it is
# a deliberate no-op; draining acks from the radix path desyncs collectives).
if not self._uses_cp_hicache:
if not self._uses_cp_hicache or not self._node_host_write_pending(node):
return False
return self._node_backup_outstanding_replicated(node)
self._cp_drain_write_acks_before_pending_split()
return self._node_host_write_pending(node)
def _cp_stale_tail_prune_still_blocked(self, stale_tail: TreeNode) -> bool:
# _cp_subtree_has_unprunable_state now reads only rank-replicated state, so a
# single read is rank-uniform; no ack drain (the helper is a deliberate no-op).
if not self._uses_cp_hicache:
if not self._uses_cp_hicache or not self._cp_subtree_has_unprunable_state(
stale_tail
):
return False
self._cp_drain_write_acks_before_pending_split()
return self._cp_subtree_has_unprunable_state(stale_tail)
def _cp_defer_insert_for_pending_backup_split(
@@ -4069,7 +4038,6 @@ class HiRadixCache(RadixCache):
return 0
self.ongoing_write_through[node.id] = node
node.cp_backup_pending = True
if not write_back:
self.inc_node_lock_ref(node)
self.pending_host_backups[node.id] = PendingHiCacheBackup(
@@ -4160,7 +4128,7 @@ class HiRadixCache(RadixCache):
return
node.hit_count += 1
if self._uses_cp_hicache and self._node_backup_outstanding_replicated(node):
if self._uses_cp_hicache and self._node_host_write_pending(node):
return
if not self._node_backuped(node):
@@ -5547,7 +5515,7 @@ class HiRadixCache(RadixCache):
def _split_node(self, key: RadixKey, child: TreeNode, split_len: int):
if (
self._uses_cp_hicache
and self._node_backup_outstanding_replicated(child)
and self._node_host_write_pending(child)
):
raise HiCachePendingBackupSplit(child)
# child node split into new_node -> child
@@ -5562,10 +5530,6 @@ class HiRadixCache(RadixCache):
new_node.host_ref_counter += 1
new_node.key = child.key[:split_len].compacted()
new_node.hit_count = child.hit_count
# Propagate the replicated backup-pending marker. The guard above raises
# HiCachePendingBackupSplit before splitting a pending node, so child is
# non-pending here and both halves carry False -- copied for fail-loud symmetry.
new_node.cp_backup_pending = child.cp_backup_pending
# split value and host value if exists
if child.evicted:

View File

@@ -229,12 +229,6 @@ class TreeNode:
self.host_value: Optional[torch.Tensor] = None
self.host_len: int = 0
self.cp_hicache = None
# Rank-replicated marker: True from the (rank-replicated) prepared-backup
# attach until the (MIN-committed) commit/rollback. The CP radix probe path
# reads THIS instead of the drain-timed ongoing_write_through/pending_host_backups
# dicts, so split/defer/prune decisions stay identical across CP ranks.
# See docs_internal/cp_hicache_symmetric_attach_design.md.
self.cp_backup_pending: bool = False
# store hash values of each pages
self.hash_value: Optional[List[str]] = None
# priority for priority-aware eviction