From 3299c4f9c1735c17305175ce1685d0ddbc2e7f91 Mon Sep 17 00:00:00 2001 From: shuwenn <47200617+alphabetc1@users.noreply.github.com> Date: Sat, 14 Feb 2026 08:46:09 +0800 Subject: [PATCH] [CI] feat: add early exit to wait_for_server when process dies (#18602) --- docs/advanced_features/lora.ipynb | 10 +- .../separate_reasoning.ipynb | 2 +- .../speculative_decoding.ipynb | 16 +-- .../structured_outputs.ipynb | 2 +- ...uctured_outputs_for_reasoning_models.ipynb | 2 +- docs/advanced_features/tool_parser.ipynb | 8 +- docs/basic_usage/native_api.ipynb | 14 +-- docs/basic_usage/openai_api_completions.ipynb | 2 +- docs/basic_usage/openai_api_embeddings.ipynb | 2 +- docs/basic_usage/openai_api_vision.ipynb | 2 +- docs/basic_usage/send_request.ipynb | 2 +- .../frontend/frontend_tutorial.ipynb | 4 +- .../hidden_states/hidden_states_server.py | 2 +- .../token_in_token_out_llm_server.py | 2 +- .../token_in_token_out_vlm_server.py | 2 +- .../server_fixtures/disaggregation_fixture.py | 24 ++--- python/sglang/utils.py | 97 ++++++++++++++----- scripts/playground/frontend_reasoning.ipynb | 2 +- .../hicache/test_disaggregation_hicache.py | 4 +- ...est_disaggregation_piecewise_cuda_graph.py | 4 +- test/manual/test_mori_transfer_engine_e2e.py | 24 +++-- .../test_disaggregation_basic.py | 12 +-- .../disaggregation/test_disaggregation_pp.py | 12 +-- .../test_disaggregation_basic.py | 16 +-- .../test_disaggregation_aarch64.py | 4 +- .../test_disaggregation_different_tp.py | 16 +-- .../test_disaggregation_dp_attention.py | 4 +- .../test_disaggregation_hybrid_attention.py | 12 +-- .../distributed/test_disaggregation_pp.py | 12 +-- .../distributed/test_epd_disaggregation.py | 14 +-- .../test_hicache_storage_file_backend.py | 21 ++-- ...t_hicache_storage_runtime_attach_detach.py | 25 +++-- test/srt/experiment_runner.py | 33 ++++--- 33 files changed, 229 insertions(+), 179 deletions(-) diff --git a/docs/advanced_features/lora.ipynb b/docs/advanced_features/lora.ipynb index a8245f1b2..7be400b89 100644 --- a/docs/advanced_features/lora.ipynb +++ b/docs/advanced_features/lora.ipynb @@ -102,7 +102,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -162,7 +162,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -240,7 +240,7 @@ ")\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url)" + "wait_for_server(url, process=server_process)" ] }, { @@ -453,7 +453,7 @@ "\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url)" + "wait_for_server(url, process=server_process)" ] }, { @@ -610,7 +610,7 @@ ")\n", "\n", "url = f\"http://127.0.0.1:{port}\"\n", - "wait_for_server(url)" + "wait_for_server(url, process=server_process)" ] }, { diff --git a/docs/advanced_features/separate_reasoning.ipynb b/docs/advanced_features/separate_reasoning.ipynb index fde97d8a6..6277dd8bd 100644 --- a/docs/advanced_features/separate_reasoning.ipynb +++ b/docs/advanced_features/separate_reasoning.ipynb @@ -70,7 +70,7 @@ " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { diff --git a/docs/advanced_features/speculative_decoding.ipynb b/docs/advanced_features/speculative_decoding.ipynb index 31d59c4d5..ba253c79d 100644 --- a/docs/advanced_features/speculative_decoding.ipynb +++ b/docs/advanced_features/speculative_decoding.ipynb @@ -111,7 +111,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -167,7 +167,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -227,7 +227,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -283,7 +283,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -338,7 +338,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -402,7 +402,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -466,7 +466,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -538,7 +538,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { diff --git a/docs/advanced_features/structured_outputs.ipynb b/docs/advanced_features/structured_outputs.ipynb index b0ec5e6c7..6106d40d6 100644 --- a/docs/advanced_features/structured_outputs.ipynb +++ b/docs/advanced_features/structured_outputs.ipynb @@ -54,7 +54,7 @@ " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" ] }, diff --git a/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb b/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb index 2b05a5837..b12dbdfea 100644 --- a/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb +++ b/docs/advanced_features/structured_outputs_for_reasoning_models.ipynb @@ -50,7 +50,7 @@ " \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" ] }, diff --git a/docs/advanced_features/tool_parser.ipynb b/docs/advanced_features/tool_parser.ipynb index df1bc4bc7..9afc9663e 100644 --- a/docs/advanced_features/tool_parser.ipynb +++ b/docs/advanced_features/tool_parser.ipynb @@ -60,7 +60,7 @@ "server_process, port = launch_server_cmd(\n", " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\" # qwen25\n", ")\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -550,7 +550,9 @@ "server_process_tool_choice, port_tool_choice = launch_server_cmd(\n", " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"\n", ")\n", - "wait_for_server(f\"http://localhost:{port_tool_choice}\")\n", + "wait_for_server(\n", + " f\"http://localhost:{port_tool_choice}\", process=server_process_tool_choice\n", + ")\n", "\n", "# Initialize client for tool choice examples\n", "client_tool_choice = OpenAI(\n", @@ -695,7 +697,7 @@ "server_process, port = launch_server_cmd(\n", " \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1 --log-level warning\" # llama-3.2-1b-instruct\n", ")\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "\n", "tools = [\n", " {\n", diff --git a/docs/basic_usage/native_api.ipynb b/docs/basic_usage/native_api.ipynb index 52e4386af..7a914fc3e 100644 --- a/docs/basic_usage/native_api.ipynb +++ b/docs/basic_usage/native_api.ipynb @@ -49,7 +49,7 @@ " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { @@ -282,7 +282,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" ] }, { @@ -331,7 +331,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=reranker_process)" ] }, { @@ -399,7 +399,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=score_process)" ] }, { @@ -462,7 +462,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=reward_process)" ] }, { @@ -526,7 +526,7 @@ " \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=expert_record_server_process)" ] }, { @@ -581,7 +581,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)" ] }, { diff --git a/docs/basic_usage/openai_api_completions.ipynb b/docs/basic_usage/openai_api_completions.ipynb index e89dfd57f..ffa576ae5 100644 --- a/docs/basic_usage/openai_api_completions.ipynb +++ b/docs/basic_usage/openai_api_completions.ipynb @@ -39,7 +39,7 @@ " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, diff --git a/docs/basic_usage/openai_api_embeddings.ipynb b/docs/basic_usage/openai_api_embeddings.ipynb index 836fe8681..693931824 100644 --- a/docs/basic_usage/openai_api_embeddings.ipynb +++ b/docs/basic_usage/openai_api_embeddings.ipynb @@ -37,7 +37,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)" ] }, { diff --git a/docs/basic_usage/openai_api_vision.ipynb b/docs/basic_usage/openai_api_vision.ipynb index cfb3d5d21..a6bb88f0c 100644 --- a/docs/basic_usage/openai_api_vision.ipynb +++ b/docs/basic_usage/openai_api_vision.ipynb @@ -39,7 +39,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=vision_process)" ] }, { diff --git a/docs/basic_usage/send_request.ipynb b/docs/basic_usage/send_request.ipynb index aa4f745d2..fd5f87a27 100644 --- a/docs/basic_usage/send_request.ipynb +++ b/docs/basic_usage/send_request.ipynb @@ -38,7 +38,7 @@ "\"\"\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")" + "wait_for_server(f\"http://localhost:{port}\", process=server_process)" ] }, { diff --git a/docs/references/frontend/frontend_tutorial.ipynb b/docs/references/frontend/frontend_tutorial.ipynb index 906d28518..fc458fd18 100644 --- a/docs/references/frontend/frontend_tutorial.ipynb +++ b/docs/references/frontend/frontend_tutorial.ipynb @@ -42,7 +42,7 @@ " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, @@ -398,7 +398,7 @@ " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, diff --git a/examples/runtime/hidden_states/hidden_states_server.py b/examples/runtime/hidden_states/hidden_states_server.py index b04f74372..c05646841 100644 --- a/examples/runtime/hidden_states/hidden_states_server.py +++ b/examples/runtime/hidden_states/hidden_states_server.py @@ -25,7 +25,7 @@ def main(): server_process, port = launch_server_cmd( "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct --enable-return-hidden-states --host 0.0.0.0" ) - wait_for_server(f"http://localhost:{port}") + wait_for_server(f"http://localhost:{port}", process=server_process) prompts = [ "Hello, my name is", diff --git a/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py b/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py index 7e498f513..3f2c98636 100644 --- a/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py +++ b/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py @@ -25,7 +25,7 @@ def main(): server_process, port = launch_server_cmd( f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" ) - wait_for_server(f"http://localhost:{port}") + wait_for_server(f"http://localhost:{port}", process=server_process) # Sample prompts. prompts = [ diff --git a/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py b/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py index 392e1bf0e..8ce79adad 100644 --- a/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py +++ b/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py @@ -47,7 +47,7 @@ def main(): server_process, port = launch_server_cmd( f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" ) - wait_for_server(f"http://localhost:{port}") + wait_for_server(f"http://localhost:{port}", process=server_process) input_ids, image_data = get_input_ids() diff --git a/python/sglang/test/server_fixtures/disaggregation_fixture.py b/python/sglang/test/server_fixtures/disaggregation_fixture.py index 4e2017633..9e2e04aff 100644 --- a/python/sglang/test/server_fixtures/disaggregation_fixture.py +++ b/python/sglang/test/server_fixtures/disaggregation_fixture.py @@ -5,8 +5,6 @@ import time import warnings from urllib.parse import urlparse -import requests - from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( @@ -16,6 +14,7 @@ from sglang.test.test_utils import ( is_in_ci, popen_with_error_check, ) +from sglang.utils import wait_for_http_ready logger = logging.getLogger(__name__) @@ -72,23 +71,14 @@ class PDDisaggregationServerBase(CustomTestCase): ] print("Starting load balancer:", shlex.join(lb_command)) cls.process_lb = popen_with_error_check(lb_command) - cls.wait_server_ready(cls.lb_url + "/health") + cls.wait_server_ready(cls.lb_url + "/health", process=cls.process_lb) @classmethod - def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH): - start_time = time.perf_counter() - while True: - try: - response = requests.get(url) - if response.status_code == 200: - print(f"Server {url} is ready") - return - except Exception: - pass - - if time.perf_counter() - start_time > timeout: - raise RuntimeError(f"Server {url} failed to start in {timeout}s") - time.sleep(1) + def wait_server_ready( + cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, process=None + ): + wait_for_http_ready(url=url, timeout=timeout, process=process) + print(f"Server {url} is ready") @classmethod def tearDownClass(cls): diff --git a/python/sglang/utils.py b/python/sglang/utils.py index 760021048..cefa4bbe7 100644 --- a/python/sglang/utils.py +++ b/python/sglang/utils.py @@ -464,37 +464,82 @@ def terminate_process(process): release_port(lock_socket) -def wait_for_server(base_url: str, timeout: int = None) -> None: +def _raise_if_process_exited(process: Optional[Any]) -> None: + if process is None: + return + + if hasattr(process, "poll"): + return_code = process.poll() + if return_code is not None: + raise RuntimeError(f"Server process exited with code {return_code}") + return + + if hasattr(process, "is_alive") and not process.is_alive(): + return_code = getattr(process, "exitcode", None) + if return_code is None: + raise RuntimeError("Server process exited") + raise RuntimeError(f"Server process exited with code {return_code}") + + +def _is_wait_timeout(start_time: float, timeout: Optional[int]) -> bool: + if timeout is None: + return False + return time.perf_counter() - start_time > timeout + + +def wait_for_http_ready( + url: str, + timeout: Optional[int] = None, + process: Optional[Any] = None, + headers: Optional[dict] = None, + request_timeout: int = 5, +) -> None: + """Wait for an HTTP endpoint to return status 200.""" + start_time = time.perf_counter() + while True: + _raise_if_process_exited(process) + try: + response = requests.get(url, headers=headers, timeout=request_timeout) + if response.status_code == 200: + return + except requests.exceptions.RequestException: + _raise_if_process_exited(process) + + if _is_wait_timeout(start_time, timeout): + raise TimeoutError( + f"Endpoint {url} did not become ready within timeout period" + ) + time.sleep(1) + + +def wait_for_server( + base_url: str, + timeout: int = None, + process: Optional[subprocess.Popen] = None, +) -> None: """Wait for the server to be ready by polling the /v1/models endpoint. Args: - base_url: The base URL of the server + base_url: The base URL of the server. timeout: Maximum time to wait in seconds. None means wait forever. + process: Optional server process used for early-exit checks. """ - start_time = time.perf_counter() - while True: - try: - response = requests.get( - f"{base_url}/v1/models", - headers={"Authorization": "Bearer None"}, - ) - if response.status_code == 200: - time.sleep(5) - print_highlight( - """\n - NOTE: Typically, the server runs in a separate terminal. - In this notebook, we run the server and notebook code together, so their outputs are combined. - To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue. - To reduce the log length, we set the log level to warning for the server, the default log level is info. - We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance. - """ - ) - break - - if timeout and time.perf_counter() - start_time > timeout: - raise TimeoutError("Server did not become ready within timeout period") - except requests.exceptions.RequestException: - time.sleep(1) + wait_for_http_ready( + url=f"{base_url}/v1/models", + timeout=timeout, + process=process, + headers={"Authorization": "Bearer None"}, + ) + time.sleep(5) + print_highlight( + """\n + NOTE: Typically, the server runs in a separate terminal. + In this notebook, we run the server and notebook code together, so their outputs are combined. + To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue. + To reduce the log length, we set the log level to warning for the server, the default log level is info. + We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance. + """ + ) class TypeBasedDispatcher: diff --git a/scripts/playground/frontend_reasoning.ipynb b/scripts/playground/frontend_reasoning.ipynb index fcdce25ab..c2412f638 100644 --- a/scripts/playground/frontend_reasoning.ipynb +++ b/scripts/playground/frontend_reasoning.ipynb @@ -34,7 +34,7 @@ " \"python3 -m sglang.launch_server --model-path Qwen/Qwen3-4B --reasoning-parser qwen3 --host 0.0.0.0\"\n", ")\n", "\n", - "wait_for_server(f\"http://localhost:{port}\")\n", + "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n", "print(f\"Server started on http://localhost:{port}\")" ] }, diff --git a/test/manual/hicache/test_disaggregation_hicache.py b/test/manual/hicache/test_disaggregation_hicache.py index 651dfd7eb..5640d5d64 100644 --- a/test/manual/hicache/test_disaggregation_hicache.py +++ b/test/manual/hicache/test_disaggregation_hicache.py @@ -33,8 +33,8 @@ class DisaggregationHiCacheBase(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py index a54a97846..90e6f5547 100644 --- a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py +++ b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py @@ -25,8 +25,8 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase): cls.start_decode() # Wait for both to be ready - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/manual/test_mori_transfer_engine_e2e.py b/test/manual/test_mori_transfer_engine_e2e.py index 249d14420..869739b23 100644 --- a/test/manual/test_mori_transfer_engine_e2e.py +++ b/test/manual/test_mori_transfer_engine_e2e.py @@ -52,10 +52,14 @@ class TestMoriTransferEngineE2E(PDDisaggregationServerBase): cls.start_decode() cls.wait_server_ready( - cls.prefill_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + cls.prefill_url + "/health", + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + process=cls.process_prefill, ) cls.wait_server_ready( - cls.decode_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + cls.decode_url + "/health", + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + process=cls.process_decode, ) cls.launch_lb() @@ -85,7 +89,9 @@ class TestMoriTransferEngineE2E(PDDisaggregationServerBase): print("Starting load balancer:", " ".join(lb_command)) cls.process_lb = subprocess.Popen(lb_command, stdout=None, stderr=None) cls.wait_server_ready( - cls.lb_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + cls.lb_url + "/health", + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + process=cls.process_lb, ) @classmethod @@ -190,10 +196,14 @@ class TestMoriTransferEngineTPMismatchE2E(PDDisaggregationServerBase): cls.start_decode() cls.wait_server_ready( - cls.prefill_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + cls.prefill_url + "/health", + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + process=cls.process_prefill, ) cls.wait_server_ready( - cls.decode_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + cls.decode_url + "/health", + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + process=cls.process_decode, ) cls.launch_lb() @@ -222,7 +232,9 @@ class TestMoriTransferEngineTPMismatchE2E(PDDisaggregationServerBase): print("Starting load balancer:", " ".join(lb_command)) cls.process_lb = subprocess.Popen(lb_command, stdout=None, stderr=None) cls.wait_server_ready( - cls.lb_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + cls.lb_url + "/health", + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + process=cls.process_lb, ) @classmethod diff --git a/test/registered/amd/disaggregation/test_disaggregation_basic.py b/test/registered/amd/disaggregation/test_disaggregation_basic.py index be2a95c58..86b3051e2 100644 --- a/test/registered/amd/disaggregation/test_disaggregation_basic.py +++ b/test/registered/amd/disaggregation/test_disaggregation_basic.py @@ -44,8 +44,8 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -238,8 +238,8 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -346,8 +346,8 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/amd/disaggregation/test_disaggregation_pp.py b/test/registered/amd/disaggregation/test_disaggregation_pp.py index 9c1dfa815..5603f463e 100644 --- a/test/registered/amd/disaggregation/test_disaggregation_pp.py +++ b/test/registered/amd/disaggregation/test_disaggregation_pp.py @@ -40,8 +40,8 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -129,8 +129,8 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -219,8 +219,8 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/disaggregation/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py index bc2e77f1d..568196aec 100644 --- a/test/registered/disaggregation/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -34,8 +34,8 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -204,8 +204,8 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -306,8 +306,8 @@ class TestDisaggregationMooncakeSpec(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -375,8 +375,8 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/distributed/test_disaggregation_aarch64.py b/test/registered/distributed/test_disaggregation_aarch64.py index ddd6e96f7..e07f7fede 100644 --- a/test/registered/distributed/test_disaggregation_aarch64.py +++ b/test/registered/distributed/test_disaggregation_aarch64.py @@ -29,8 +29,8 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/distributed/test_disaggregation_different_tp.py b/test/registered/distributed/test_disaggregation_different_tp.py index 00199ec25..1d944dec3 100644 --- a/test/registered/distributed/test_disaggregation_different_tp.py +++ b/test/registered/distributed/test_disaggregation_different_tp.py @@ -32,8 +32,8 @@ class TestDisaggregationMooncakePrefillLargerTP(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -103,8 +103,8 @@ class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -174,8 +174,8 @@ class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -245,8 +245,8 @@ class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/distributed/test_disaggregation_dp_attention.py b/test/registered/distributed/test_disaggregation_dp_attention.py index 90b5b4bab..3fbb620b6 100644 --- a/test/registered/distributed/test_disaggregation_dp_attention.py +++ b/test/registered/distributed/test_disaggregation_dp_attention.py @@ -34,8 +34,8 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/distributed/test_disaggregation_hybrid_attention.py b/test/registered/distributed/test_disaggregation_hybrid_attention.py index 226a22a8f..a926894ef 100644 --- a/test/registered/distributed/test_disaggregation_hybrid_attention.py +++ b/test/registered/distributed/test_disaggregation_hybrid_attention.py @@ -29,8 +29,8 @@ class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -97,8 +97,8 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -175,8 +175,8 @@ class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase) cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/distributed/test_disaggregation_pp.py b/test/registered/distributed/test_disaggregation_pp.py index b6396e55b..63cbea175 100644 --- a/test/registered/distributed/test_disaggregation_pp.py +++ b/test/registered/distributed/test_disaggregation_pp.py @@ -28,8 +28,8 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -101,8 +101,8 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -175,8 +175,8 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase): cls.start_decode() # Block until both - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/distributed/test_epd_disaggregation.py b/test/registered/distributed/test_epd_disaggregation.py index 96e13de3b..298655cfa 100644 --- a/test/registered/distributed/test_epd_disaggregation.py +++ b/test/registered/distributed/test_epd_disaggregation.py @@ -44,9 +44,9 @@ class TestEPDDisaggregationOneEncoder(PDDisaggregationServerBase): decode_thread.join() # Wait for all servers to be ready - cls.wait_server_ready(cls.encode_url + "/health") - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.encode_url + "/health", process=cls.process_encode) + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() @@ -249,10 +249,10 @@ class TestEPDDisaggregationMultiEncoders(PDDisaggregationServerBase): prefill_thread.join() decode_thread.join() - cls.wait_server_ready(cls.encode_url1 + "/health") - cls.wait_server_ready(cls.encode_url2 + "/health") - cls.wait_server_ready(cls.prefill_url + "/health") - cls.wait_server_ready(cls.decode_url + "/health") + cls.wait_server_ready(cls.encode_url1 + "/health", process=cls.process_encode1) + cls.wait_server_ready(cls.encode_url2 + "/health", process=cls.process_encode2) + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) cls.launch_lb() diff --git a/test/registered/hicache/test_hicache_storage_file_backend.py b/test/registered/hicache/test_hicache_storage_file_backend.py index a3ce03564..4dfdf028a 100644 --- a/test/registered/hicache/test_hicache_storage_file_backend.py +++ b/test/registered/hicache/test_hicache_storage_file_backend.py @@ -29,6 +29,7 @@ from sglang.test.test_utils import ( is_in_ci, popen_launch_server, ) +from sglang.utils import wait_for_http_ready register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu") register_amd_ci(est_time=526, suite="stage-b-test-large-2-gpu-amd") @@ -53,7 +54,7 @@ class HiCacheStorageBaseMixin: # Launch server with HiCache enabled and cache report cls.process = cls._launch_server_with_hicache() - cls._wait_for_server_ready() + cls._wait_for_server_ready(process=cls.process) print(f"Test server launched successfully at {cls.base_url}") print(f"Cache directory: {cls.temp_dir}") @@ -128,18 +129,14 @@ class HiCacheStorageBaseMixin: ) @classmethod - def _wait_for_server_ready(cls, timeout: int = 60) -> bool: + def _wait_for_server_ready(cls, timeout: int = 60, process=None) -> bool: """Wait for server to be ready""" - start_time = time.time() - while time.time() - start_time < timeout: - try: - response = requests.get(f"{cls.base_url}/health", timeout=5) - if response.status_code == 200: - return True - except requests.RequestException: - pass - time.sleep(2) - raise TimeoutError("Server failed to start within timeout") + wait_for_http_ready( + url=f"{cls.base_url}/health", + timeout=timeout, + process=process, + ) + return True def send_request( self, prompt: str, max_tokens: int = 100, temperature: float = 0.0 diff --git a/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py b/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py index 65e52041d..28094e850 100644 --- a/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py +++ b/test/registered/hicache/test_hicache_storage_runtime_attach_detach.py @@ -26,6 +26,7 @@ from sglang.test.test_utils import ( find_available_port, popen_launch_server, ) +from sglang.utils import wait_for_http_ready register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu") @@ -68,17 +69,15 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase): shutil.rmtree(cls.temp_dir, ignore_errors=True) @classmethod - def _wait_for_server_ready(cls, base_url: str, timeout: int = 60) -> bool: - start_time = time.time() - while time.time() - start_time < timeout: - try: - code, _body = cls._http_get(f"{base_url}/health", timeout=5) - if code == 200: - return True - except Exception: - pass - time.sleep(2) - raise TimeoutError("Server failed to start within timeout") + def _wait_for_server_ready( + cls, base_url: str, timeout: int = 60, process=None + ) -> bool: + wait_for_http_ready( + url=f"{base_url}/health", + timeout=timeout, + process=process, + ) + return True @staticmethod def _http_get(url: str, timeout: int = 10, headers: dict | None = None): @@ -204,7 +203,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase): env=self.env, ) try: - self._wait_for_server_ready(self.base_url) + self._wait_for_server_ready(self.base_url, process=process1) code_info, _body_info = self._http_get( f"{self.base_url}/hicache/storage-backend", timeout=10 @@ -234,7 +233,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase): env=self.env, ) try: - self._wait_for_server_ready(base_url2) + self._wait_for_server_ready(base_url2, process=process2) # 1) Initially disabled (but unauthorized without admin key) code_info2_unauth, _ = self._http_get( diff --git a/test/srt/experiment_runner.py b/test/srt/experiment_runner.py index f6af05623..04368e252 100644 --- a/test/srt/experiment_runner.py +++ b/test/srt/experiment_runner.py @@ -11,9 +11,10 @@ from datetime import datetime from typing import List, Optional, Tuple import psutil -import requests import yaml +from sglang.utils import wait_for_http_ready + @dataclass class ServerConfig: @@ -182,18 +183,20 @@ class ExperimentRunner: self.process_manager = ProcessManager() self.logger = logging.getLogger(__name__) - def wait_for_server(self, port: int, timeout: int = 300) -> bool: - start_time = time.perf_counter() - - while time.perf_counter() - start_time < timeout: - try: - response = requests.get(f"http://localhost:{port}/health") - if response.status_code == 200: - self.logger.debug(f"Server ready on port {port}") - return True - except requests.RequestException: - time.sleep(2) - return False + def wait_for_server( + self, port: int, timeout: int = 300, process: Optional[subprocess.Popen] = None + ) -> bool: + try: + wait_for_http_ready( + url=f"http://localhost:{port}/health", + timeout=timeout, + process=process, + ) + self.logger.debug(f"Server ready on port {port}") + return True + except (RuntimeError, TimeoutError) as e: + self.logger.error("Server failed to become ready: %s", e) + return False def run_task(self, config: TaskConfig) -> TaskResult: start_time = time.perf_counter() @@ -216,7 +219,9 @@ class ExperimentRunner: self.process_manager.start_process(config.server_cmd, "SERVER") ) - if not self.wait_for_server(port): + if not self.wait_for_server( + port, process=self.process_manager.server_process + ): raise TimeoutError("Server startup timeout") time.sleep(10)