Special logic for healthcheck (#17734)

Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>

python/sglang/srt/managers/detokenizer_manager.py

@@ -18,7 +18,7 @@ import logging
 import os
 import signal
 from collections import OrderedDict, defaultdict
-from typing import Dict, List, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import psutil
 import pybase64
@@ -94,6 +94,10 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
         # Init dispatcher
         self.init_request_dispatcher()
 
+    @staticmethod
+    def is_health_check_request(rid: Optional[str]) -> bool:
+        return isinstance(rid, str) and rid.startswith("HEALTH_CHECK")
+
     def init_ipc_channels(self, port_args: PortArgs):
         context = zmq.Context(2)
         self.recv_from_scheduler = get_zmq_socket(
@@ -232,7 +236,9 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
                     surr_offset=0,
                     read_offset=recv_obj.read_offsets[i],
                 )
-                self.decode_status[rid] = s
+                if not self.is_health_check_request(rid):
+                    # for health check requests, we do not store the decode status
+                    self.decode_status[rid] = s
             else:
                 s = self.decode_status[rid]
                 s.decode_ids.extend(recv_obj.decode_ids[i])
@@ -290,17 +296,26 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
         # Incremental decoding
         output_strs = []
         for i in range(bs):
-            try:
-                s = self.decode_status[recv_obj.rids[i]]
-            except KeyError:
-                raise RuntimeError(
-                    f"Decode status not found for request {recv_obj.rids[i]}. "
-                    "It may be due to the request being evicted from the decode status due to memory pressure. "
-                    "Please increase the maximum number of requests by setting "
-                    "the SGLANG_DETOKENIZER_MAX_STATES environment variable to a bigger value than the default value. "
-                    f"The current value is {DETOKENIZER_MAX_STATES}. "
-                    "For more details, see: https://github.com/sgl-project/sglang/issues/2812"
+            rid = recv_obj.rids[i]
+            if self.is_health_check_request(rid):
+                s = DecodeStatus(
+                    decoded_text=recv_obj.decoded_texts[i],
+                    decode_ids=recv_obj.decode_ids[i],
+                    surr_offset=0,
+                    read_offset=recv_obj.read_offsets[i],
                 )
+            else:
+                try:
+                    s = self.decode_status[rid]
+                except KeyError:
+                    raise RuntimeError(
+                        f"Decode status not found for request {rid}. "
+                        "It may be due to the request being evicted from the decode status due to memory pressure. "
+                        "Please increase the maximum number of requests by setting "
+                        "the SGLANG_DETOKENIZER_MAX_STATES environment variable to a bigger value than the default value. "
+                        f"The current value is {DETOKENIZER_MAX_STATES}. "
+                        "For more details, see: https://github.com/sgl-project/sglang/issues/2812"
+                    )
             new_text = read_texts[i][len(surr_texts[i]) :]
             if recv_obj.finished_reasons[i] is None:
                 # Streaming chunk: update the decode status
@@ -312,7 +327,8 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
                 else:
                     new_text = find_printable_text(new_text)
             else:
-                del self.decode_status[recv_obj.rids[i]]
+                if rid in self.decode_status:
+                    del self.decode_status[rid]
 
             output_str = self.trim_matched_stop(
                 s.decoded_text + new_text,