From e6f7a372ef071e5c3cee480e579e47d94d9bcc74 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 12 Feb 2026 22:58:40 -0800 Subject: [PATCH] Rename request timeout env vars for waiting/running stages (#18766) --- docs/references/environment_variables.md | 4 ++-- python/sglang/srt/environ.py | 16 ++++++++++++++-- python/sglang/srt/managers/scheduler.py | 22 +++++++++++----------- test/registered/scheduler/test_abort.py | 14 +++++++------- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md index 3b470b55d..9a42ae7b8 100644 --- a/docs/references/environment_variables.md +++ b/docs/references/environment_variables.md @@ -17,8 +17,8 @@ SGLang supports various environment variables that can be used to configure its | `SGLANG_HEALTH_CHECK_TIMEOUT` | Timeout for health check in seconds | `20` | | `SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL` | The interval of passes to collect the metric of selected count of physical experts on each layer and GPU rank. 0 means disabled. | `0` | | `SGLANG_FORWARD_UNKNOWN_TOOLS` | Forward unknown tool calls to clients instead of dropping them | `false` (drop unknown tools) | -| `SGLANG_QUEUED_TIMEOUT_MS` | Timeout (in ms) for requests in the waiting queue | `-1` | -| `SGLANG_FORWARD_TIMEOUT_MS` | Timeout (in ms) for requests in the forward batch | `-1` | +| `SGLANG_REQ_WAITING_TIMEOUT` | Timeout (in seconds) for requests waiting in the queue before being scheduled | `-1` | +| `SGLANG_REQ_RUNNING_TIMEOUT` | Timeout (in seconds) for requests running in the decode batch | `-1` | ## Performance Tuning diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 552c26911..69b695b87 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -248,9 +248,9 @@ class Envs: SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(30) SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None) SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1) - SGLANG_QUEUED_TIMEOUT_MS = EnvInt(-1) + SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) - SGLANG_FORWARD_TIMEOUT_MS = EnvInt(-1) + SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds # Test: pd-disaggregation SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake") @@ -509,6 +509,18 @@ def _convert_SGL_to_SGLANG(): "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK", "SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK", ) + _deprecated_ms_to_s = { + "SGLANG_QUEUED_TIMEOUT_MS": "SGLANG_REQ_WAITING_TIMEOUT", + "SGLANG_FORWARD_TIMEOUT_MS": "SGLANG_REQ_RUNNING_TIMEOUT", + } + for old_name, new_name in _deprecated_ms_to_s.items(): + if old_name in os.environ: + ms_val = os.environ[old_name] + warnings.warn( + f"Environment variable {old_name} (in ms) is deprecated, " + f"please use {new_name} (in seconds) instead" + ) + os.environ[new_name] = str(float(ms_val) / 1000.0) for key, value in os.environ.items(): if key.startswith("SGL_"): diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 8419c9d17..8bf5245d8 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1062,20 +1062,20 @@ class Scheduler( ] ) - def _check_forward_timeout_for_running_batch(self): + def _abort_on_running_timeout(self): # NOTE: this should be called before a batch is launched, # as current spec-v1 still filters batch inside verify stage. - timeout_ms = envs.SGLANG_FORWARD_TIMEOUT_MS.get() - if timeout_ms <= 0: + timeout_s = envs.SGLANG_REQ_RUNNING_TIMEOUT.get() + if timeout_s <= 0: return if self.running_batch.is_empty(): return - deadline = time.perf_counter() - timeout_ms / 1000.0 + deadline = time.perf_counter() - timeout_s for req in self.running_batch.reqs: if not req.finished() and 0 < req.time_stats.forward_entry_time < deadline: req.to_finish = FINISH_ABORT( - "Forward timeout.", HTTPStatus.SERVICE_UNAVAILABLE + "Request running timeout reached.", HTTPStatus.SERVICE_UNAVAILABLE ) @DynamicGradMode() @@ -1736,12 +1736,12 @@ class Scheduler( ) return req_to_abort.rid == recv_req.rid - def _abort_on_queued_timeout(self): - if (timeout_ms := envs.SGLANG_QUEUED_TIMEOUT_MS.get()) <= 0: + def _abort_on_waiting_timeout(self): + if (timeout_s := envs.SGLANG_REQ_WAITING_TIMEOUT.get()) <= 0: return deleted_reqs = set() - deadline = time.perf_counter() - (timeout_ms / 1000.0) + deadline = time.perf_counter() - timeout_s for req in self.waiting_queue: entry_time = req.time_stats.wait_queue_entry_time if 0 < entry_time < deadline: @@ -1753,7 +1753,7 @@ class Scheduler( finished_reason={ "type": "abort", "status_code": HTTPStatus.SERVICE_UNAVAILABLE, - "message": "Request queue timeout reached.", + "message": "Request waiting timeout reached.", }, rid=req.rid, ), @@ -1838,8 +1838,8 @@ class Scheduler( self.tree_cache.cache_unfinished_req(req, chunked=True) def get_next_batch_to_run(self) -> Optional[ScheduleBatch]: - self._abort_on_queued_timeout() - self._check_forward_timeout_for_running_batch() + self._abort_on_waiting_timeout() + self._abort_on_running_timeout() if self.dllm_config is not None: self.dllm_manager.filter_finished_reqs() diff --git a/test/registered/scheduler/test_abort.py b/test/registered/scheduler/test_abort.py index da4c70adb..21fcc75c0 100644 --- a/test/registered/scheduler/test_abort.py +++ b/test/registered/scheduler/test_abort.py @@ -236,12 +236,12 @@ class TestAbortAllWithRetraction(CustomTestCase): print("Finished test_abort_all_with_retraction") -class TestAbortWithQueueTimeout(CustomTestCase): +class TestAbortWithWaitingTimeout(CustomTestCase): @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_QUEUED_TIMEOUT_MS.override(1): + with envs.SGLANG_REQ_WAITING_TIMEOUT.override(0.001): cls.process = popen_launch_server( cls.model, cls.base_url, @@ -269,7 +269,7 @@ class TestAbortWithQueueTimeout(CustomTestCase): ) return response.json() - def test_queue_timeout(self): + def test_waiting_timeout(self): num_requests = 2 with ThreadPoolExecutor(num_requests) as executor: futures = [executor.submit(self._run_decode) for _ in range(num_requests)] @@ -283,13 +283,13 @@ class TestAbortWithQueueTimeout(CustomTestCase): self.assertEqual(error_count, 1) -class TestAbortWithForwardTimeout(CustomTestCase): +class TestAbortWithRunningTimeout(CustomTestCase): @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_FORWARD_TIMEOUT_MS.override( - 1 + with envs.SGLANG_REQ_RUNNING_TIMEOUT.override( + 0.001 ), envs.SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION.override(False): cls.process = popen_launch_server( cls.model, @@ -302,7 +302,7 @@ class TestAbortWithForwardTimeout(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_forward_timeout(self): + def test_running_timeout(self): response = requests.post( self.base_url + "/generate", json={