Support soft watchdog for tokenizer/detokenizer/dp-controller processes (#15607)

python/sglang/srt/managers/data_parallel_controller.py

@@ -59,6 +59,7 @@ from sglang.srt.utils.common import (
     maybe_reindex_device_id,
 )
 from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
+from sglang.srt.utils.watchdog import Watchdog
 from sglang.utils import TypeBasedDispatcher, get_exception_traceback
 
 logger = logging.getLogger(__name__)
@@ -173,6 +174,12 @@ class DataParallelController:
 
         self.init_dispatcher()
 
+        self.watchdog = Watchdog.create(
+            debug_name="DataParallelController",
+            watchdog_timeout=server_args.soft_watchdog_timeout,
+            soft=True,
+        )
+
     def send_to_all_workers(self, obj):
         for worker in self.workers:
             worker.send_pyobj(obj)
@@ -519,6 +526,7 @@ class DataParallelController:
     def event_loop(self):
         while True:
             while True:
+                self.watchdog.feed()
                 try:
                     recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                 except zmq.ZMQError: