chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Rename request timeout env vars for waiting/running stages (#18766)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-02-12 22:58:40 -08:00
committed by GitHub
parent 5700b19cbf
commit e6f7a372ef
4 changed files with 34 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
python/sglang/srt/environ.py
View File

@@ -248,9 +248,9 @@ class Envs:
SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(30)
SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None)
SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1)
SGLANG_QUEUED_TIMEOUT_MS = EnvInt(-1)
SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
SGLANG_FORWARD_TIMEOUT_MS = EnvInt(-1)
SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds
# Test: pd-disaggregation
SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake")
@@ -509,6 +509,18 @@ def _convert_SGL_to_SGLANG():
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK",
"SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK",
)
_deprecated_ms_to_s = {
"SGLANG_QUEUED_TIMEOUT_MS": "SGLANG_REQ_WAITING_TIMEOUT",
"SGLANG_FORWARD_TIMEOUT_MS": "SGLANG_REQ_RUNNING_TIMEOUT",
}
for old_name, new_name in _deprecated_ms_to_s.items():
if old_name in os.environ:
ms_val = os.environ[old_name]
warnings.warn(
f"Environment variable {old_name} (in ms) is deprecated, "
f"please use {new_name} (in seconds) instead"
)
os.environ[new_name] = str(float(ms_val) / 1000.0)
for key, value in os.environ.items():
if key.startswith("SGL_"):

22
python/sglang/srt/managers/scheduler.py
View File

@@ -1062,20 +1062,20 @@ class Scheduler(
]
)
def _check_forward_timeout_for_running_batch(self):
def _abort_on_running_timeout(self):
# NOTE: this should be called before a batch is launched,
# as current spec-v1 still filters batch inside verify stage.
timeout_ms = envs.SGLANG_FORWARD_TIMEOUT_MS.get()
if timeout_ms <= 0:
timeout_s = envs.SGLANG_REQ_RUNNING_TIMEOUT.get()
if timeout_s <= 0:
return
if self.running_batch.is_empty():
return
deadline = time.perf_counter() - timeout_ms / 1000.0
deadline = time.perf_counter() - timeout_s
for req in self.running_batch.reqs:
if not req.finished() and 0 < req.time_stats.forward_entry_time < deadline:
req.to_finish = FINISH_ABORT(
"Forward timeout.", HTTPStatus.SERVICE_UNAVAILABLE
"Request running timeout reached.", HTTPStatus.SERVICE_UNAVAILABLE
)
@DynamicGradMode()
@@ -1736,12 +1736,12 @@ class Scheduler(
)
return req_to_abort.rid == recv_req.rid
def _abort_on_queued_timeout(self):
if (timeout_ms := envs.SGLANG_QUEUED_TIMEOUT_MS.get()) <= 0:
def _abort_on_waiting_timeout(self):
if (timeout_s := envs.SGLANG_REQ_WAITING_TIMEOUT.get()) <= 0:
return
deleted_reqs = set()
deadline = time.perf_counter() - (timeout_ms / 1000.0)
deadline = time.perf_counter() - timeout_s
for req in self.waiting_queue:
entry_time = req.time_stats.wait_queue_entry_time
if 0 < entry_time < deadline:
@@ -1753,7 +1753,7 @@ class Scheduler(
finished_reason={
"type": "abort",
"status_code": HTTPStatus.SERVICE_UNAVAILABLE,
"message": "Request queue timeout reached.",
"message": "Request waiting timeout reached.",
},
rid=req.rid,
),
@@ -1838,8 +1838,8 @@ class Scheduler(
self.tree_cache.cache_unfinished_req(req, chunked=True)
def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
self._abort_on_queued_timeout()
self._check_forward_timeout_for_running_batch()
self._abort_on_waiting_timeout()
self._abort_on_running_timeout()
if self.dllm_config is not None:
self.dllm_manager.filter_finished_reqs()