diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index c8b5fcbeb..18ee1f6e0 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -72,6 +72,7 @@ from sglang.srt.entrypoints.openai.serving_tokenize import ( OpenAIServingDetokenize, OpenAIServingTokenize, ) +from sglang.srt.environ import envs from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.managers.io_struct import ( AbortReq, @@ -1528,12 +1529,14 @@ def _execute_server_warmup( json_data["sampling_params"]["max_new_tokens"] = 0 try: + warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get() if server_args.disaggregation_mode == "null": + logger.info(f"Start of co-locate warmup ...") res = requests.post( url + request_name, json=json_data, headers=headers, - timeout=600, + timeout=warmup_timeout if warmup_timeout > 0 else 600, ) assert res.status_code == 200, f"{res.text}" _global_state.tokenizer_manager.server_status = ServerStatus.Up @@ -1559,7 +1562,9 @@ def _execute_server_warmup( url + request_name, json=json_data, headers=headers, - timeout=1800, # because of deep gemm precache is very long if not precache. + timeout=( + warmup_timeout if warmup_timeout > 0 else 1800 + ), # because of deep gemm precache is very long if not precache. ) if res.status_code == 200: logger.info( diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 8bec1c3c8..a7afa81d3 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -288,6 +288,9 @@ class Envs: # Ngram SGLANG_NGRAM_FORCE_GREEDY_VERIFY = EnvBool(False) + # Warmup + SGLANG_WARMUP_TIMEOUT = EnvFloat(-1) # in seconds. If a warmup forward batch takes longer than this, the server will crash to prevent hanging. Recommend to increase warmup timeout to 1800 to accommodate some kernel JIT precache e.g. deep gemm + # fmt: on