Support soft watchdog for tokenizer/detokenizer/dp-controller processes (#15607)
This commit is contained in:
@@ -1,15 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import psutil
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -18,9 +13,9 @@ from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
|
||||
from sglang.srt.utils.common import (
|
||||
ceil_align,
|
||||
disable_request_logging,
|
||||
pyspy_dump_schedulers,
|
||||
raise_error_or_warn,
|
||||
)
|
||||
from sglang.srt.utils.watchdog import WatchdogRaw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
@@ -341,62 +336,31 @@ class SchedulerRuntimeCheckerMixin:
|
||||
self.maybe_sleep_on_idle()
|
||||
|
||||
|
||||
class SchedulerWatchdog:
|
||||
"""A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
|
||||
def create_scheduler_watchdog(
|
||||
scheduler: Scheduler, watchdog_timeout: float, soft: bool = False
|
||||
) -> WatchdogRaw:
|
||||
def dump_info() -> str:
|
||||
if disable_request_logging():
|
||||
return ""
|
||||
if scheduler.is_hybrid_swa:
|
||||
_, info_msg = scheduler._check_hybrid_memory()
|
||||
elif scheduler.is_hybrid_ssm and isinstance(
|
||||
scheduler.tree_cache, MambaRadixCache
|
||||
):
|
||||
_, info_msg = scheduler._check_mamba_memory()
|
||||
else:
|
||||
_, info_msg = scheduler._check_radix_cache_memory()
|
||||
return (
|
||||
f"{scheduler.cur_batch.batch_size()=}\n"
|
||||
f"{scheduler.cur_batch.reqs=}\n"
|
||||
f"{info_msg}"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self, scheduler: Scheduler, watchdog_timeout: float, soft: bool = False
|
||||
):
|
||||
self.scheduler = scheduler
|
||||
self.soft = soft
|
||||
|
||||
self.watchdog_timeout = watchdog_timeout
|
||||
t = threading.Thread(target=self._watchdog_thread, daemon=True)
|
||||
t.start()
|
||||
self.parent_process = psutil.Process().parent()
|
||||
|
||||
def _watchdog_thread(self):
|
||||
while True:
|
||||
self._watchdog_once()
|
||||
|
||||
def _watchdog_once(self):
|
||||
watchdog_last_forward_ct = 0
|
||||
watchdog_last_time = time.perf_counter()
|
||||
|
||||
while True:
|
||||
current = time.perf_counter()
|
||||
if self.scheduler.cur_batch is not None:
|
||||
if watchdog_last_forward_ct == self.scheduler.forward_ct:
|
||||
if current > watchdog_last_time + self.watchdog_timeout:
|
||||
break
|
||||
else:
|
||||
watchdog_last_forward_ct = self.scheduler.forward_ct
|
||||
watchdog_last_time = current
|
||||
time.sleep(self.watchdog_timeout // 2)
|
||||
|
||||
if not disable_request_logging():
|
||||
# TODO extract this duplicated logic w/ another place
|
||||
# Print batch size and memory pool info to check whether there are de-sync issues.
|
||||
if self.scheduler.is_hybrid_swa:
|
||||
_, info_msg = self.scheduler._check_hybrid_memory()
|
||||
elif self.scheduler.is_hybrid_ssm and isinstance(
|
||||
self.scheduler.tree_cache, MambaRadixCache
|
||||
):
|
||||
_, info_msg = self.scheduler._check_mamba_memory()
|
||||
else:
|
||||
_, info_msg = self.scheduler._check_radix_cache_memory()
|
||||
logger.error(
|
||||
f"{self.scheduler.cur_batch.batch_size()=}\n"
|
||||
f"{self.scheduler.cur_batch.reqs=}\n"
|
||||
f"{info_msg}"
|
||||
)
|
||||
|
||||
pyspy_dump_schedulers()
|
||||
logger.error(f"Watchdog timeout ({self.watchdog_timeout=}, {self.soft=})")
|
||||
print(file=sys.stderr, flush=True)
|
||||
print(file=sys.stdout, flush=True)
|
||||
|
||||
if not self.soft:
|
||||
# Wait for some time so that the parent process can print the error.
|
||||
time.sleep(5)
|
||||
self.parent_process.send_signal(signal.SIGQUIT)
|
||||
return WatchdogRaw(
|
||||
debug_name="Scheduler",
|
||||
get_counter=lambda: scheduler.forward_ct,
|
||||
is_active=lambda: scheduler.cur_batch is not None,
|
||||
watchdog_timeout=watchdog_timeout,
|
||||
soft=soft,
|
||||
dump_info=dump_info,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user