Revert "Fix CP HiCache writing_check deadlock at the source: rank-replicated symmetric attach"
This reverts commit 85585fcf4b.
This commit is contained in:
@@ -2967,19 +2967,6 @@ class HiRadixCache(RadixCache):
|
||||
self, "pending_host_backups", {}
|
||||
)
|
||||
|
||||
def _node_backup_outstanding_replicated(self, node: TreeNode) -> bool:
|
||||
"""Rank-replicated 'backup outstanding' signal for the CP radix probe path.
|
||||
|
||||
True from the rank-replicated prepared-backup attach until the MIN-committed
|
||||
commit/rollback (see TreeNode.cp_backup_pending). Unlike
|
||||
_node_host_write_pending (which reads the drain-timed ongoing_write_through/
|
||||
pending_host_backups dicts and so carries per-rank ack-drain timing), this is
|
||||
identical across CP ranks at every sub-tick instant, so split/defer/prune
|
||||
decisions stay rank-uniform. The radix probe path must read THIS; only
|
||||
writing_check / eviction may read the drain-timed _node_host_write_pending.
|
||||
"""
|
||||
return bool(getattr(node, "cp_backup_pending", False))
|
||||
|
||||
def _node_host_write_ready(self, node: TreeNode) -> bool:
|
||||
return not self._node_host_write_pending(node)
|
||||
|
||||
@@ -3006,10 +2993,6 @@ class HiRadixCache(RadixCache):
|
||||
node.cp_hicache = pending.metadata
|
||||
node.host_value = None
|
||||
self._cp_account_enter_committed(pending.metadata)
|
||||
# Clear the replicated pending marker at the MIN-committed edge, BEFORE
|
||||
# releasing the backup lock (keeps marker and lock_ref consistent). After
|
||||
# commit, node.host_len > 0 (_node_backuped) is the authoritative signal.
|
||||
node.cp_backup_pending = False
|
||||
if pending.locked:
|
||||
self.dec_node_lock_ref(node)
|
||||
# B1: this commit is gated by the writing_check ReduceOp.MIN frontier, which
|
||||
@@ -3071,7 +3054,6 @@ class HiRadixCache(RadixCache):
|
||||
if node.cp_hicache is pending.metadata:
|
||||
node.cp_hicache = None
|
||||
node.host_len = 0
|
||||
node.cp_backup_pending = False
|
||||
if pending.locked:
|
||||
self.dec_node_lock_ref(node)
|
||||
return node
|
||||
@@ -3360,13 +3342,7 @@ class HiRadixCache(RadixCache):
|
||||
f"node_len={len(node.value)} prepared_len={prepared.logical_len}"
|
||||
)
|
||||
node.id = prepared.node_id
|
||||
if getattr(node, "cp_backup_pending", False):
|
||||
raise RuntimeError(
|
||||
f"CP HiCache double-attach for node_id={node.id}: "
|
||||
"cp_backup_pending already set"
|
||||
)
|
||||
self.ongoing_write_through[node.id] = node
|
||||
node.cp_backup_pending = True
|
||||
self.inc_node_lock_ref(node)
|
||||
self.pending_host_backups[node.id] = PendingHiCacheBackup(
|
||||
node=node,
|
||||
@@ -3673,15 +3649,10 @@ class HiRadixCache(RadixCache):
|
||||
stack = [node]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
# Read only rank-replicated state: the backup-pending marker (covers the
|
||||
# backup lock -- the sole rank-local lock_ref skew) and host_ref_counter
|
||||
# (set by replicated pin/protect_host). Do NOT read lock_ref here: its only
|
||||
# cross-rank skew is the backup lock, already covered by the marker, and a
|
||||
# request-held lock is rank-uniform but irrelevant to splittability (the
|
||||
# page-floor logic governs that). Reading it would re-leak per-rank timing.
|
||||
if (
|
||||
self._node_backup_outstanding_replicated(current)
|
||||
current.lock_ref > 0
|
||||
or current.host_ref_counter > 0
|
||||
or self._node_host_write_pending(current)
|
||||
):
|
||||
return True
|
||||
stack.extend(current.children.values())
|
||||
@@ -3705,19 +3676,17 @@ class HiRadixCache(RadixCache):
|
||||
return
|
||||
|
||||
def _cp_node_split_still_pending(self, node: TreeNode) -> bool:
|
||||
# Rank-replicated marker only (see _node_backup_outstanding_replicated): the
|
||||
# drain-timed _node_host_write_pending here is what let CP ranks diverge on the
|
||||
# attach/defer fork. Do NOT call _cp_drain_write_acks_before_pending_split (it is
|
||||
# a deliberate no-op; draining acks from the radix path desyncs collectives).
|
||||
if not self._uses_cp_hicache:
|
||||
if not self._uses_cp_hicache or not self._node_host_write_pending(node):
|
||||
return False
|
||||
return self._node_backup_outstanding_replicated(node)
|
||||
self._cp_drain_write_acks_before_pending_split()
|
||||
return self._node_host_write_pending(node)
|
||||
|
||||
def _cp_stale_tail_prune_still_blocked(self, stale_tail: TreeNode) -> bool:
|
||||
# _cp_subtree_has_unprunable_state now reads only rank-replicated state, so a
|
||||
# single read is rank-uniform; no ack drain (the helper is a deliberate no-op).
|
||||
if not self._uses_cp_hicache:
|
||||
if not self._uses_cp_hicache or not self._cp_subtree_has_unprunable_state(
|
||||
stale_tail
|
||||
):
|
||||
return False
|
||||
self._cp_drain_write_acks_before_pending_split()
|
||||
return self._cp_subtree_has_unprunable_state(stale_tail)
|
||||
|
||||
def _cp_defer_insert_for_pending_backup_split(
|
||||
@@ -4069,7 +4038,6 @@ class HiRadixCache(RadixCache):
|
||||
return 0
|
||||
|
||||
self.ongoing_write_through[node.id] = node
|
||||
node.cp_backup_pending = True
|
||||
if not write_back:
|
||||
self.inc_node_lock_ref(node)
|
||||
self.pending_host_backups[node.id] = PendingHiCacheBackup(
|
||||
@@ -4160,7 +4128,7 @@ class HiRadixCache(RadixCache):
|
||||
return
|
||||
node.hit_count += 1
|
||||
|
||||
if self._uses_cp_hicache and self._node_backup_outstanding_replicated(node):
|
||||
if self._uses_cp_hicache and self._node_host_write_pending(node):
|
||||
return
|
||||
|
||||
if not self._node_backuped(node):
|
||||
@@ -5547,7 +5515,7 @@ class HiRadixCache(RadixCache):
|
||||
def _split_node(self, key: RadixKey, child: TreeNode, split_len: int):
|
||||
if (
|
||||
self._uses_cp_hicache
|
||||
and self._node_backup_outstanding_replicated(child)
|
||||
and self._node_host_write_pending(child)
|
||||
):
|
||||
raise HiCachePendingBackupSplit(child)
|
||||
# child node split into new_node -> child
|
||||
@@ -5562,10 +5530,6 @@ class HiRadixCache(RadixCache):
|
||||
new_node.host_ref_counter += 1
|
||||
new_node.key = child.key[:split_len].compacted()
|
||||
new_node.hit_count = child.hit_count
|
||||
# Propagate the replicated backup-pending marker. The guard above raises
|
||||
# HiCachePendingBackupSplit before splitting a pending node, so child is
|
||||
# non-pending here and both halves carry False -- copied for fail-loud symmetry.
|
||||
new_node.cp_backup_pending = child.cp_backup_pending
|
||||
|
||||
# split value and host value if exists
|
||||
if child.evicted:
|
||||
|
||||
@@ -229,12 +229,6 @@ class TreeNode:
|
||||
self.host_value: Optional[torch.Tensor] = None
|
||||
self.host_len: int = 0
|
||||
self.cp_hicache = None
|
||||
# Rank-replicated marker: True from the (rank-replicated) prepared-backup
|
||||
# attach until the (MIN-committed) commit/rollback. The CP radix probe path
|
||||
# reads THIS instead of the drain-timed ongoing_write_through/pending_host_backups
|
||||
# dicts, so split/defer/prune decisions stay identical across CP ranks.
|
||||
# See docs_internal/cp_hicache_symmetric_attach_design.md.
|
||||
self.cp_backup_pending: bool = False
|
||||
# store hash values of each pages
|
||||
self.hash_value: Optional[List[str]] = None
|
||||
# priority for priority-aware eviction
|
||||
|
||||
Reference in New Issue
Block a user