chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Rename request timeout env vars for waiting/running stages (#18766)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-02-12 22:58:40 -08:00
committed by GitHub
parent 5700b19cbf
commit e6f7a372ef
4 changed files with 34 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/references/environment_variables.md
View File

@@ -17,8 +17,8 @@ SGLang supports various environment variables that can be used to configure its
| `SGLANG_HEALTH_CHECK_TIMEOUT` | Timeout for health check in seconds | `20` |
| `SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL` | The interval of passes to collect the metric of selected count of physical experts on each layer and GPU rank. 0 means disabled. | `0` |
| `SGLANG_FORWARD_UNKNOWN_TOOLS` | Forward unknown tool calls to clients instead of dropping them | `false` (drop unknown tools) |
| `SGLANG_QUEUED_TIMEOUT_MS` | Timeout (in ms) for requests in the waiting queue | `-1` |
| `SGLANG_FORWARD_TIMEOUT_MS` | Timeout (in ms) for requests in the forward batch | `-1` |
| `SGLANG_REQ_WAITING_TIMEOUT` | Timeout (in seconds) for requests waiting in the queue before being scheduled | `-1` |
| `SGLANG_REQ_RUNNING_TIMEOUT` | Timeout (in seconds) for requests running in the decode batch | `-1` |
## Performance Tuning

16
python/sglang/srt/environ.py
View File

@@ -248,9 +248,9 @@ class Envs:
SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(30)
SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None)
SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1)
SGLANG_QUEUED_TIMEOUT_MS = EnvInt(-1)
SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
SGLANG_FORWARD_TIMEOUT_MS = EnvInt(-1)
SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds
# Test: pd-disaggregation
SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake")
@@ -509,6 +509,18 @@ def _convert_SGL_to_SGLANG():
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK",
"SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK",
)
_deprecated_ms_to_s = {
"SGLANG_QUEUED_TIMEOUT_MS": "SGLANG_REQ_WAITING_TIMEOUT",
"SGLANG_FORWARD_TIMEOUT_MS": "SGLANG_REQ_RUNNING_TIMEOUT",
}
for old_name, new_name in _deprecated_ms_to_s.items():
if old_name in os.environ:
ms_val = os.environ[old_name]
warnings.warn(
f"Environment variable {old_name} (in ms) is deprecated, "
f"please use {new_name} (in seconds) instead"
)
os.environ[new_name] = str(float(ms_val) / 1000.0)
for key, value in os.environ.items():
if key.startswith("SGL_"):

22
python/sglang/srt/managers/scheduler.py
View File

@@ -1062,20 +1062,20 @@ class Scheduler(
]
)
def _check_forward_timeout_for_running_batch(self):
def _abort_on_running_timeout(self):
# NOTE: this should be called before a batch is launched,
# as current spec-v1 still filters batch inside verify stage.
timeout_ms = envs.SGLANG_FORWARD_TIMEOUT_MS.get()
if timeout_ms <= 0:
timeout_s = envs.SGLANG_REQ_RUNNING_TIMEOUT.get()
if timeout_s <= 0:
return
if self.running_batch.is_empty():
return
deadline = time.perf_counter() - timeout_ms / 1000.0
deadline = time.perf_counter() - timeout_s
for req in self.running_batch.reqs:
if not req.finished() and 0 < req.time_stats.forward_entry_time < deadline:
req.to_finish = FINISH_ABORT(
"Forward timeout.", HTTPStatus.SERVICE_UNAVAILABLE
"Request running timeout reached.", HTTPStatus.SERVICE_UNAVAILABLE
)
@DynamicGradMode()
@@ -1736,12 +1736,12 @@ class Scheduler(
)
return req_to_abort.rid == recv_req.rid
def _abort_on_queued_timeout(self):
if (timeout_ms := envs.SGLANG_QUEUED_TIMEOUT_MS.get()) <= 0:
def _abort_on_waiting_timeout(self):
if (timeout_s := envs.SGLANG_REQ_WAITING_TIMEOUT.get()) <= 0:
return
deleted_reqs = set()
deadline = time.perf_counter() - (timeout_ms / 1000.0)
deadline = time.perf_counter() - timeout_s
for req in self.waiting_queue:
entry_time = req.time_stats.wait_queue_entry_time
if 0 < entry_time < deadline:
@@ -1753,7 +1753,7 @@ class Scheduler(
finished_reason={
"type": "abort",
"status_code": HTTPStatus.SERVICE_UNAVAILABLE,
"message": "Request queue timeout reached.",
"message": "Request waiting timeout reached.",
},
rid=req.rid,
),
@@ -1838,8 +1838,8 @@ class Scheduler(
self.tree_cache.cache_unfinished_req(req, chunked=True)
def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
self._abort_on_queued_timeout()
self._check_forward_timeout_for_running_batch()
self._abort_on_waiting_timeout()
self._abort_on_running_timeout()
if self.dllm_config is not None:
self.dllm_manager.filter_finished_reqs()

14
test/registered/scheduler/test_abort.py
View File

@@ -236,12 +236,12 @@ class TestAbortAllWithRetraction(CustomTestCase):
print("Finished test_abort_all_with_retraction")
class TestAbortWithQueueTimeout(CustomTestCase):
class TestAbortWithWaitingTimeout(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
with envs.SGLANG_QUEUED_TIMEOUT_MS.override(1):
with envs.SGLANG_REQ_WAITING_TIMEOUT.override(0.001):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
@@ -269,7 +269,7 @@ class TestAbortWithQueueTimeout(CustomTestCase):
)
return response.json()
def test_queue_timeout(self):
def test_waiting_timeout(self):
num_requests = 2
with ThreadPoolExecutor(num_requests) as executor:
futures = [executor.submit(self._run_decode) for _ in range(num_requests)]
@@ -283,13 +283,13 @@ class TestAbortWithQueueTimeout(CustomTestCase):
self.assertEqual(error_count, 1)
class TestAbortWithForwardTimeout(CustomTestCase):
class TestAbortWithRunningTimeout(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
with envs.SGLANG_FORWARD_TIMEOUT_MS.override(
1
with envs.SGLANG_REQ_RUNNING_TIMEOUT.override(
0.001
), envs.SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION.override(False):
cls.process = popen_launch_server(
cls.model,
@@ -302,7 +302,7 @@ class TestAbortWithForwardTimeout(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_forward_timeout(self):
def test_running_timeout(self):
response = requests.post(
self.base_url + "/generate",
json={