fix(disagg/mooncake): notify prefill on decode-side abort

Port the decode->prefill half of upstream sgl-project/sglang #27372 (the worker
skip-Failed guard half landed in cb1a03f0a). In multi-prefill/multi-decode PD, a
decode-initiated abort frees the decode's KV pages back to the allocator, but the
prefill never learns of it (request_status is per-process), so the prefill keeps
RDMA-writing the remaining chunks into pages the decode may have reallocated to a
different live request -> KV corruption. The already-ported worker guard is a no-op
here because nothing sets the prefill room to Failed.

Now the decode receiver sends a 4-field b"ABORT" notification to every prefill peer
(on abort() and on the poll() WaitingForInput timeout, at most once); the prefill
marks the room Failed (so the worker guard fires) and replies b"ABORT_ACK". The
ABORT branch is handled before the unconditional waiting_req_bytes[3] decode in
bootstrap_thread (a 4-field ABORT would otherwise crash the else branch), and
ABORT_ACK before the 3-tuple unpack in the decode thread.

De-entangled from the staging/tracing infra this branch lacks. Component-tested
(test_mooncake_abort_protocol.py: format, send-once guard, abort wiring, ZMQ
round-trip). Cross-process corruption-window closure to be verified by the
multi-P/D PD harness.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

python/sglang/srt/disaggregation/mooncake/conn.py

@@ -1224,6 +1224,34 @@ class MooncakeKVManager(CommonKVManager):
             while True:
                 waiting_req_bytes = self.server_socket.recv_multipart()
                 room = waiting_req_bytes[0].decode("ascii")
+                # Decode-side abort notification: mark the room Failed and ACK, so the
+                # transfer worker (which skips Failed rooms) stops RDMA-writing into KV
+                # pages the decode has freed/reused. Port of upstream sgl-project/sglang
+                # #27372 (decode->prefill half). MUST be handled here, before the
+                # UNCONDITIONAL waiting_req_bytes[3] decode below: a 4-field ABORT
+                # message [ABORT, room, ip, port] would otherwise fall through into the
+                # else branch and crash on waiting_req_bytes[7].
+                if room == "ABORT":
+                    room_to_be_aborted = int(waiting_req_bytes[1].decode("ascii"))
+                    decode_ip = waiting_req_bytes[2].decode("ascii")
+                    decode_port = int(waiting_req_bytes[3].decode("ascii"))
+                    if (
+                        room_to_be_aborted in self.request_status
+                        and self.check_status(room_to_be_aborted) != KVPoll.Success
+                    ):
+                        self.update_status(room_to_be_aborted, KVPoll.Failed)
+                    try:
+                        na = NetworkAddress(decode_ip, decode_port)
+                        self._connect(
+                            na.to_tcp(), is_ipv6=na.is_ipv6
+                        ).send_multipart(
+                            [b"ABORT_ACK", str(room_to_be_aborted).encode("ascii")]
+                        )
+                    except Exception as e:
+                        logger.debug(
+                            f"Failed to send ABORT_ACK for room {room_to_be_aborted}: {e}"
+                        )
+                    continue
                 mooncake_session_id = waiting_req_bytes[3].decode("ascii")
                 if room == "None":
                     self.decode_kv_args_table[mooncake_session_id] = (
@@ -1261,6 +1289,12 @@ class MooncakeKVManager(CommonKVManager):
                     self._handle_aux_data(msg)
                     continue
 
+                if msg[0] == b"ABORT_ACK":
+                    # Prefill acknowledged our abort; the room is already Failed locally,
+                    # so nothing more to do. Port of upstream #27372. Must precede the
+                    # 3-tuple unpack below (a 2-field ACK would otherwise raise).
+                    continue
+
                 bootstrap_room, status, prefill_rank = msg
                 status = int(status.decode("ascii"))
                 bootstrap_room = int(bootstrap_room.decode("ascii"))
@@ -1617,6 +1651,7 @@ class MooncakeKVReceiver(CommonKVReceiver):
         self.session_id = mgr.get_session_id()
         self.conclude_state = None
         self.init_time = None
+        self.abort_notified = False
         super().__init__(mgr, bootstrap_addr, bootstrap_room, prefill_dp_rank)
 
         self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room)
@@ -1746,6 +1781,8 @@ class MooncakeKVReceiver(CommonKVReceiver):
                             self.bootstrap_room,
                             f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s in KVPoll.WaitingForInput",
                         )
+                        # Tell prefill peers to stop transferring into our pages.
+                        self._send_abort_notification()
                         self.conclude_state = KVPoll.Failed
                         return KVPoll.Failed
 
@@ -1777,6 +1814,35 @@ class MooncakeKVReceiver(CommonKVReceiver):
             )
         raise KVTransferError(self.bootstrap_room, failure_reason)
 
+    def _send_abort_notification(self):
+        # Notify every prefill peer that this room is aborting, so the prefill stops
+        # RDMA-writing into KV pages this decode is about to free/reuse. Without this,
+        # the prefill never learns of a decode-initiated abort (its request_status is
+        # per-process) and the worker's skip-Failed guard never fires. Port of upstream
+        # sgl-project/sglang #27372 (decode->prefill half). Sent at most once.
+        if self.abort_notified:
+            return
+        bootstrap_infos = getattr(self, "bootstrap_infos", None)
+        if not bootstrap_infos:
+            return
+        self.abort_notified = True
+        for bootstrap_info in bootstrap_infos:
+            try:
+                sock, lock = self._connect_to_bootstrap_server(bootstrap_info)
+                with lock:
+                    sock.send_multipart(
+                        [
+                            b"ABORT",
+                            str(self.bootstrap_room).encode("ascii"),
+                            self.kv_mgr.local_ip.encode("ascii"),
+                            str(self.kv_mgr.rank_port).encode("ascii"),
+                        ]
+                    )
+            except Exception as e:
+                logger.debug(
+                    f"Failed to send ABORT for room {self.bootstrap_room}: {e}"
+                )
+
     def abort(self):
         self.kv_mgr.record_failure(
             self.bootstrap_room,
@@ -1786,6 +1852,8 @@ class MooncakeKVReceiver(CommonKVReceiver):
         # transfer worker and other pollers observe the abort. Port of upstream
         # sgl-project/sglang #24522.
         self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed)
+        # Tell the prefill peers to stop transferring into our (soon-freed) pages.
+        self._send_abort_notification()
         # Explicitly set the status to failure since this request has been aborted
         self.conclude_state = KVPoll.Failed