Prevent stale CP HiCache tails from overlapping new page owners

CP HiCache owns KV at page granularity, but exact valid-tail extension and backed partial-tail split could leave an old sub-page tail child beside a new suffix that reuses the same physical page. That makes radix residency ambiguous across device, host, and draft mirrors. The insert/match split paths now prune stale floored tails when safe, and defer/fail through the existing pending-split path when the subtree is protected or has in-flight backup state.\n\nThis also keeps a temporary scheduler boundary warning for externally observed zero-output responses so future ETE runs can classify whether zero visible output reaches SGLang's output processor.\n\nConstraint: CP shared KV and HiCache manage physical KV by page, while radix keys retain valid-token lengths.\nRejected: Keep overlapping old tail nodes after page-floor split | leaves two independent cache states for one physical tail page.\nRejected: Force-prune protected or in-flight backup tails | can mutate cache state still used by active transfer or inference.\nConfidence: medium\nScope-risk: moderate\nDirective: Do not remove the stale-tail prune without replacing it with another page-granular ownership rule for CP HiCache radix splits.\nTested: Remote py_compile for hiradix_cache.py and scheduler_output_processor_mixin.py in g0034 container.\nTested: Remote PYTHONPATH=python python -m pytest -q test/registered/unit/mem_cache/test_cp_hicache_metadata.py -> 97 passed, 5 warnings.\nNot-tested: Full ETE recovery of EAGLE accept length; latest ETE still shows accept collapse, documented in C55.\nNot-tested: Router/client-side output_len=0 correlation when server-side OUTPUT_ZERO_DEBUG does not fire.

python/sglang/srt/managers/scheduler_output_processor_mixin.py

@@ -1140,6 +1140,28 @@ class SchedulerOutputProcessorMixin:
 
                 # Exclude the tokens after stop condition
                 output_ids_ = req.output_ids_through_stop
+                if req.finished() and len(output_ids_) == 0:
+                    logger.warning(
+                        "[OUTPUT_ZERO_DEBUG] finished generation with zero visible "
+                        "tokens: rid=%s finish_reason=%s max_new_tokens=%s "
+                        "origin_len=%s raw_output_len=%s finished_len=%s "
+                        "send_token_offset=%s cached_tokens=%s disaggregation_mode=%s "
+                        "bootstrap_room=%s",
+                        getattr(req, "rid", None),
+                        (
+                            req.finished_reason.to_json()
+                            if req.finished_reason is not None
+                            else None
+                        ),
+                        getattr(req.sampling_params, "max_new_tokens", None),
+                        len(getattr(req, "origin_input_ids", []) or []),
+                        len(getattr(req, "output_ids", []) or []),
+                        getattr(req, "finished_len", None),
+                        send_token_offset,
+                        getattr(req, "cached_tokens", None),
+                        getattr(self, "disaggregation_mode", None),
+                        getattr(req, "bootstrap_room", None),
+                    )
 
                 req.send_decode_id_offset = len(decode_ids)
                 read_offsets.append(read_offset)

python/sglang/srt/mem_cache/hiradix_cache.py

@@ -2157,6 +2157,98 @@ class HiRadixCache(RadixCache):
             return prefix_len
         return floor_to_page_len(prefix_len, self.page_size)
 
+    def _cp_subtree_has_unprunable_state(self, node: TreeNode) -> bool:
+        stack = [node]
+        while stack:
+            current = stack.pop()
+            if (
+                current.lock_ref > 0
+                or current.host_ref_counter > 0
+                or self._node_host_write_pending(current)
+            ):
+                return True
+            stack.extend(current.children.values())
+        return False
+
+    def _cp_prune_stale_tail_after_page_floor(
+        self, parent: TreeNode, stale_tail: TreeNode
+    ) -> None:
+        """Drop a sub-page CP tail after flooring an extending request to a page.
+
+        CP HiCache owns KV at page granularity.  If a cached key ends inside a
+        page and a later request extends past that valid tail, the old sub-page
+        child must not remain as an independent radix child beside the new
+        page-boundary suffix.  Keeping both creates overlapping children for the
+        same physical page.  We sacrifice the old tail page and let the new
+        request re-own that page from the boundary.
+        """
+
+        if not self._uses_cp_hicache:
+            return
+        if stale_tail.parent is not parent:
+            raise RuntimeError(
+                "CP HiCache stale-tail prune saw a detached split child: "
+                f"parent_id={getattr(parent, 'id', None)} "
+                f"tail_id={getattr(stale_tail, 'id', None)}"
+            )
+        if self._cp_subtree_has_unprunable_state(stale_tail):
+            raise HiCachePendingBackupSplit(stale_tail)
+        self._cp_delete_stale_tail_subtree(stale_tail)
+
+    def _cp_delete_stale_tail_subtree(self, node: TreeNode) -> None:
+        for child in list(node.children.values()):
+            self._cp_delete_stale_tail_subtree(child)
+
+        parent = node.parent
+        if parent is None:
+            raise RuntimeError(
+                "CP HiCache stale-tail prune attempted to delete a root node"
+            )
+
+        device_resident_len = self._node_device_resident_len(node)
+        if hasattr(self, "evictable_leaves") and node in self.evictable_leaves:
+            self.evictable_leaves.remove(node)
+            if hasattr(self, "evictable_size_"):
+                self.evictable_size_ -= device_resident_len
+        if hasattr(self, "evictable_host_leaves") and node in self.evictable_host_leaves:
+            self.evictable_host_leaves.remove(node)
+
+        if node.value is not None:
+            self._record_remove_event(node)
+            cache_controller = getattr(self, "cache_controller", None)
+            if cache_controller is not None and hasattr(cache_controller, "evict_device"):
+                cache_controller.evict_device(node.value)
+            elif cache_controller is not None and hasattr(
+                cache_controller, "mem_pool_device_allocator"
+            ):
+                cache_controller.mem_pool_device_allocator.free(node.value)
+            node.value = None
+
+        if node.cp_hicache is not None:
+            cache_controller = getattr(self, "cache_controller", None)
+            if cache_controller is not None and hasattr(cache_controller, "evict_cp_host"):
+                cache_controller.evict_cp_host(node.cp_hicache)
+            node.cp_hicache = None
+            node.host_len = 0
+        elif node.host_value is not None:
+            cache_controller = getattr(self, "cache_controller", None)
+            if cache_controller is not None and hasattr(cache_controller, "evict_host"):
+                cache_controller.evict_host(node.host_value)
+            node.host_value = None
+            node.host_len = 0
+
+        key = self.get_child_key_fn(node.key)
+        removed = parent.children.pop(key, None)
+        if removed is not node:
+            raise RuntimeError(
+                "CP HiCache stale-tail prune removed unexpected child: "
+                f"key={key} node_id={getattr(node, 'id', None)}"
+            )
+        if hasattr(self, "evictable_leaves"):
+            self._update_leaf_status(parent)
+        if hasattr(self, "evictable_host_leaves"):
+            self._update_host_leaf_status(parent)
+
     def prepare_write_backup_for_req(self, req) -> None:
         if self.disable or not self._uses_cp_hicache:
             return
@@ -3552,10 +3644,12 @@ class HiRadixCache(RadixCache):
             # Refresh pin TTL on cache hit
             if self._is_pinned(child):
                 child.pin_expiry = time.monotonic() + child.pin_ttl
-            prefix_len = self.key_match_fn(child.key, key)
+            raw_prefix_len = self.key_match_fn(child.key, key)
             prefix_len = self._cp_floor_exact_valid_tail_extension_len(
-                child, prefix_len, len(key)
+                child, raw_prefix_len, len(key)
             )
+            stop_after_page_floor = prefix_len != raw_prefix_len
+            prune_stale_tail_after_split = stop_after_page_floor
             if prefix_len <= 0:
                 break
             if prefix_len < len(child.key):
@@ -3569,12 +3663,19 @@ class HiRadixCache(RadixCache):
                     and self.page_size > 1
                     and prefix_len % self.page_size != 0
                 ):
+                    unfloored_prefix_len = prefix_len
                     prefix_len = self._cp_floor_backed_partial_split_len(
                         child, prefix_len
                     )
+                    prune_stale_tail_after_split = (
+                        prune_stale_tail_after_split
+                        or prefix_len != unfloored_prefix_len
+                    )
                     if prefix_len == 0:
                         break
                 new_node = self._split_node(child.key, child, prefix_len)
+                if prune_stale_tail_after_split:
+                    self._cp_prune_stale_tail_after_page_floor(new_node, child)
                 if not new_node.evicted:
                     value.append(new_node.value)
                 node = new_node
@@ -3688,12 +3789,20 @@ class HiRadixCache(RadixCache):
                     total_prefix_length += prefix_len
             else:
                 # partial match, split the node
+                prune_stale_tail_after_split = stop_after_page_floor
+                unfloored_prefix_len = prefix_len
                 prefix_len = self._cp_floor_backed_partial_split_len(
                     node, prefix_len
                 )
+                prune_stale_tail_after_split = (
+                    prune_stale_tail_after_split
+                    or prefix_len != unfloored_prefix_len
+                )
                 if prefix_len <= 0:
                     break
                 new_node = self._split_node(node.key, node, prefix_len)
+                if prune_stale_tail_after_split:
+                    self._cp_prune_stale_tail_after_page_floor(new_node, node)
                 # shared-prefix node should also reflect max priority
                 new_node.priority = max(new_node.priority, priority)
                 if new_node.evicted: