From 9d6029fb9292bd5897712bf7dec3ca006c2cb4cd Mon Sep 17 00:00:00 2001 From: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com> Date: Mon, 5 Jan 2026 16:28:25 +0800 Subject: [PATCH] Fix TokenizerManager bottleneck for offline generation (#16456) --- python/sglang/srt/environ.py | 3 +++ python/sglang/srt/managers/tokenizer_manager.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index de90328ef..6d3728ede 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -419,6 +419,9 @@ class Envs: # Tokenizer SGLANG_PATCH_TOKENIZER = EnvBool(False) # TODO enable by default + # TokenizerManager + SGLANG_REQUEST_STATE_WAIT_TIMEOUT = EnvInt(4) + # fmt: on diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index fc65ce962..c9fedff28 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -116,6 +116,8 @@ from sglang.utils import TypeBasedDispatcher, get_exception_traceback asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) +_REQUEST_STATE_WAIT_TIMEOUT = envs.SGLANG_REQUEST_STATE_WAIT_TIMEOUT.get() + logger = logging.getLogger(__name__) @@ -1081,7 +1083,9 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi is_stream = getattr(obj, "stream", False) while True: try: - await asyncio.wait_for(state.event.wait(), timeout=4) + await asyncio.wait_for( + state.event.wait(), timeout=_REQUEST_STATE_WAIT_TIMEOUT + ) except asyncio.TimeoutError: if ( request is not None