Add soft watchdogs to debug soft hangs (#15023)

python/sglang/srt/managers/scheduler.py

@@ -527,6 +527,8 @@ class Scheduler(
         self.watchdog = SchedulerWatchdog(
             self, watchdog_timeout=server_args.watchdog_timeout
         )
+        if (x := server_args.soft_watchdog_timeout) is not None:
+            self.soft_watchdog = SchedulerWatchdog(self, watchdog_timeout=x, soft=True)
 
         # Init memory saver, profiler and metric stats
         self.memory_saver_adapter = TorchMemorySaverAdapter.create(

python/sglang/srt/managers/scheduler_runtime_checker_mixin.py

@@ -307,10 +307,13 @@ class SchedulerRuntimeCheckerMixin:
 
 
 class SchedulerWatchdog:
-    """A watch dog that will try to kill the server itself if one forward batch takes too long."""
+    """A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
 
-    def __init__(self, scheduler: Scheduler, watchdog_timeout: float):
+    def __init__(
+        self, scheduler: Scheduler, watchdog_timeout: float, soft: bool = False
+    ):
         self.scheduler = scheduler
+        self.soft = soft
 
         self.watchdog_timeout = watchdog_timeout
         t = threading.Thread(target=self._watchdog_thread, daemon=True)
@@ -318,6 +321,10 @@ class SchedulerWatchdog:
         self.parent_process = psutil.Process().parent()
 
     def _watchdog_thread(self):
+        while True:
+            self._watchdog_once()
+
+    def _watchdog_once(self):
         watchdog_last_forward_ct = 0
         watchdog_last_time = time.perf_counter()
 
@@ -350,10 +357,11 @@ class SchedulerWatchdog:
             )
 
         pyspy_dump_schedulers()
-        logger.error(f"Watchdog timeout ({self.watchdog_timeout=})")
+        logger.error(f"Watchdog timeout ({self.watchdog_timeout=}, {self.soft=})")
         print(file=sys.stderr, flush=True)
         print(file=sys.stdout, flush=True)
 
-        # Wait for some time so that the parent process can print the error.
-        time.sleep(5)
-        self.parent_process.send_signal(signal.SIGQUIT)
+        if not self.soft:
+            # Wait for some time so that the parent process can print the error.
+            time.sleep(5)
+            self.parent_process.send_signal(signal.SIGQUIT)

python/sglang/srt/server_args.py

@@ -314,6 +314,7 @@ class ServerArgs:
     constrained_json_whitespace_pattern: Optional[str] = None
     constrained_json_disable_any_whitespace: bool = False
     watchdog_timeout: float = 300
+    soft_watchdog_timeout: Optional[float] = None
     dist_timeout: Optional[int] = None  # timeout for torch.distributed
     download_dir: Optional[str] = None
     base_gpu_id: int = 0
@@ -2650,6 +2651,12 @@ class ServerArgs:
             default=ServerArgs.watchdog_timeout,
             help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.",
         )
+        parser.add_argument(
+            "--soft-watchdog-timeout",
+            type=float,
+            default=ServerArgs.soft_watchdog_timeout,
+            help="Set soft watchdog timeout in seconds. If a forward batch takes longer than this, the server will dump information for debugging.",
+        )
         parser.add_argument(
             "--dist-timeout",
             type=int,