From 267170bf1dc4780a6f3b69751e71eaf1dfa94bef Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Fri, 12 Dec 2025 18:46:07 -0800 Subject: [PATCH] Clean up server args and engine startup processes (#15015) --- .github/workflows/pr-gate.yml | 2 +- python/pyproject.toml | 21 ++++--- python/sglang/srt/entrypoints/engine.py | 43 ++++++------- python/sglang/srt/server_args.py | 82 ++++++++++++++----------- python/sglang/srt/utils/common.py | 12 ---- scripts/code_sync/guideline.md | 24 ++++++++ test/srt/quant/test_awq.py | 2 +- 7 files changed, 105 insertions(+), 81 deletions(-) diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index a291b27db..9e65c8566 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -55,7 +55,7 @@ jobs: run: | labels='${{ steps.pr.outputs.labels }}' if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci') }}" == "false" ]]; then - echo "Missing required label 'run-ci'." + echo "Missing required label 'run-ci'. See https://docs.sglang.io/developer_guide/contribution_guide.html#how-to-trigger-ci-tests for more details." exit 1 fi diff --git a/python/pyproject.toml b/python/pyproject.toml index 5e6104578..198792559 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -98,9 +98,12 @@ diffusion = [ "cache-dit==1.1.8" ] -[tool.uv.extra-build-dependencies] -st-attn = ["torch", "setuptools"] -vsa = ["torch", "setuptools"] +tracing = [ + "opentelemetry-api", + "opentelemetry-exporter-otlp", + "opentelemetry-exporter-otlp-proto-grpc", + "opentelemetry-sdk", +] test = [ "accelerate", @@ -109,18 +112,18 @@ test = [ "jsonlines", "matplotlib", "pandas", + "parameterized", "peft", "pytest", "sentence_transformers", "tabulate", ] + dev = ["sglang[test]"] -tracing = [ - "opentelemetry-api", - "opentelemetry-exporter-otlp", - "opentelemetry-exporter-otlp-proto-grpc", - "opentelemetry-sdk", -] + +[tool.uv.extra-build-dependencies] +st-attn = ["torch", "setuptools"] +vsa = ["torch", "setuptools"] [project.urls] "Homepage" = "https://github.com/sgl-project/sglang" diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 921c31dc2..9879ae932 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -76,7 +76,6 @@ from sglang.srt.utils import ( launch_dummy_health_check_server, maybe_reindex_device_id, numa_utils, - prepare_model_and_tokenizer, set_prometheus_multiproc_dir, set_ulimit, ) @@ -105,11 +104,6 @@ def _launch_subprocesses( port_args = PortArgs.init_new(server_args) logger.info(f"{server_args=}") - # If using model from www.modelscope.cn, first download the model - server_args.model_path, server_args.tokenizer_path = prepare_model_and_tokenizer( - server_args.model_path, server_args.tokenizer_path - ) - # Launch scheduler processes scheduler_procs, scheduler_pipe_readers = _launch_scheduler_processes( server_args=server_args, @@ -826,22 +820,23 @@ def _set_envs_and_config(server_args: ServerArgs): set_ulimit() # Check flashinfer version - if server_args.attention_backend == "flashinfer": - assert_pkg_version( - "flashinfer_python", - "0.5.3", - "Please uninstall the old version and " - "reinstall the latest version by following the instructions " - "at https://docs.flashinfer.ai/installation.html.", - ) - if _is_cuda and not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"): - assert_pkg_version( - "sgl-kernel", - "0.3.19", - "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`", - ) + if not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"): + if server_args.attention_backend == "flashinfer": + assert_pkg_version( + "flashinfer_python", + "0.5.3", + "Please uninstall the old version and " + "reinstall the latest version by following the instructions " + "at https://docs.flashinfer.ai/installation.html.", + ) + if _is_cuda: + assert_pkg_version( + "sgl-kernel", + "0.3.19", + "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`", + ) - if True: # Keep this check for internal code compatibility + if server_args.custom_sigquit_handler is None: # Register the signal handler. # The child processes will send SIGQUIT to this process when any error happens # This process then clean up the whole process tree @@ -854,6 +849,12 @@ def _set_envs_and_config(server_args: ServerArgs): kill_process_tree(os.getpid()) signal.signal(signal.SIGQUIT, launch_phase_sigquit_handler) + else: + # Allow users to register a custom SIGQUIT handler for things like crash dump + logger.error( + f"Using custom SIGQUIT handler: {server_args.custom_sigquit_handler}" + ) + signal.signal(signal.SIGQUIT, server_args.custom_sigquit_handler) # Set mp start method mp.set_start_method("spawn", force=True) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 320e2db19..37bd21da5 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -23,7 +23,7 @@ import logging import os import random import tempfile -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Union import orjson @@ -252,7 +252,6 @@ class ServerArgs: skip_tokenizer_init: bool = False load_format: str = "auto" model_loader_extra_config: str = "{}" - rl_quant_profile: Optional[str] = None # For flash_rl load format trust_remote_code: bool = False context_length: Optional[int] = None is_embedding: bool = False @@ -281,6 +280,7 @@ class ServerArgs: modelopt_checkpoint_save_path: Optional[str] = None modelopt_export_path: Optional[str] = None quantize_and_serve: bool = False + rl_quant_profile: Optional[str] = None # For flash_rl load format # Memory and scheduling mem_fraction_static: Optional[float] = None @@ -320,7 +320,7 @@ class ServerArgs: base_gpu_id: int = 0 gpu_id_step: int = 1 sleep_on_idle: bool = False - mm_process_config: Optional[Dict[str, Any]] = None + custom_sigquit_handler: Optional[Callable] = None # Logging log_level: str = "info" @@ -606,14 +606,13 @@ class ServerArgs: mm_max_concurrent_calls: int = 32 mm_per_request_timeout: float = 10.0 enable_broadcast_mm_inputs_process: bool = False + mm_enable_dp_encoder: bool = False + mm_process_config: Optional[Dict[str, Any]] = None # For checkpoint decryption decrypted_config_file: Optional[str] = None decrypted_draft_config_file: Optional[str] = None - # For encoder dp - mm_enable_dp_encoder: bool = False - # For forward hooks forward_hooks: Optional[List[dict[str, Any]]] = None @@ -643,9 +642,6 @@ class ServerArgs: self._handle_cpu_backends() self._handle_npu_backends() - # Handle compilation config - self._handle_compilation_cfg() - # Apply model-specific adjustments. self._handle_model_specific_adjustments() @@ -709,7 +705,7 @@ class ServerArgs: self._handle_elastic_ep() def _handle_deprecated_args(self): - # handle deprecated tool call parsers + # Handle deprecated tool call parsers deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"} if self.tool_call_parser in deprecated_tool_call_parsers: logger.warning( @@ -729,6 +725,16 @@ class ServerArgs: if self.mm_process_config is None: self.mm_process_config = {} + # Handle ModelScope model downloads + if get_bool_env_var("SGLANG_USE_MODELSCOPE"): + if not os.path.exists(self.model_path): + from modelscope import snapshot_download + + self.model_path = snapshot_download(self.model_path) + self.tokenizer_path = snapshot_download( + self.tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"] + ) + def _handle_gpu_memory_settings(self, gpu_mem): """ Configure GPU memory-dependent settings including @@ -908,7 +914,7 @@ class ServerArgs: if self.disable_cuda_graph_padding: capture_bs = list(range(1, self.cuda_graph_max_bs + 1)) elif self.speculative_algorithm is None: - # Normal case: [1, 2, 4, 8, 12] + list(range(16, 257, 8)) + list(range(272, 512, 16)) + list(range(512, cuda_graph_max_bs + 1)) + # Normal case: capture_bs = ( [1, 2, 4, 8, 12] + list(range(16, 257, 8)) @@ -916,7 +922,7 @@ class ServerArgs: + list(range(512, self.cuda_graph_max_bs + 1, 32)) ) else: - # Spec decoding case: list(range(1, 9, 1)) + list(range(10, 33, 2)) + list(range(40, 64, 4)) + list(range(72, 257, 8)) + # Spec decoding case: less padding for smaller batch sizes capture_bs = ( list(range(1, 9, 1)) + list(range(10, 33, 2)) @@ -959,21 +965,19 @@ class ServerArgs: self.attention_backend = "intel_amx" self.sampling_backend = "pytorch" - def _handle_compilation_cfg(self): - # NPU platform - if is_npu() and self.piecewise_cuda_graph_compiler != "eager": - logger.warning( - "At this moment Ascend platform only support prefill graph compilation with " - "piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'." - ) - self.piecewise_cuda_graph_compiler = "eager" - def _handle_npu_backends(self): if self.device == "npu": from sglang.srt.hardware_backend.npu.utils import set_default_server_args set_default_server_args(self) + if self.piecewise_cuda_graph_compiler != "eager": + logger.warning( + "At this moment Ascend platform only support prefill graph compilation with " + "piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'." + ) + self.piecewise_cuda_graph_compiler = "eager" + def _handle_model_specific_adjustments(self): from sglang.srt.configs.model_config import is_deepseek_nsa @@ -2283,12 +2287,6 @@ class ServerArgs: "This will be passed to the model loader corresponding to the chosen load_format.", default=ServerArgs.model_loader_extra_config, ) - parser.add_argument( - "--rl-quant-profile", - type=str, - default=ServerArgs.rl_quant_profile, - help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.", - ) parser.add_argument( "--trust-remote-code", action="store_true", @@ -2464,6 +2462,12 @@ class ServerArgs: "This is useful for development and prototyping. For production, it's recommended " "to use separate quantization and deployment steps.", ) + parser.add_argument( + "--rl-quant-profile", + type=str, + default=ServerArgs.rl_quant_profile, + help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.", + ) # Memory and scheduling parser.add_argument( @@ -2687,10 +2691,8 @@ class ServerArgs: help="Reduce CPU usage when sglang is idle.", ) parser.add_argument( - "--mm-process-config", - type=json.loads, - default=ServerArgs.mm_process_config, - help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`", + "--custom-sigquit-handler", + help="Register a custom sigquit handler so you can do additional cleanup after the server is shutdown. This is only available for Engine, not for CLI.", ) # Logging @@ -4110,6 +4112,18 @@ class ServerArgs: default=ServerArgs.enable_broadcast_mm_inputs_process, help="Enable broadcast mm-inputs process in scheduler.", ) + parser.add_argument( + "--mm-process-config", + type=json.loads, + default=ServerArgs.mm_process_config, + help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`", + ) + parser.add_argument( + "--mm-enable-dp-encoder", + action="store_true", + default=ServerArgs.mm_enable_dp_encoder, + help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.", + ) # For checkpoint decryption parser.add_argument( @@ -4124,12 +4138,6 @@ class ServerArgs: default=ServerArgs.decrypted_draft_config_file, help="The path of the decrypted draft config file.", ) - parser.add_argument( - "--mm-enable-dp-encoder", - action="store_true", - default=ServerArgs.mm_enable_dp_encoder, - help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.", - ) # For registering hooks parser.add_argument( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 6b39a1402..375064c56 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1145,18 +1145,6 @@ def add_api_key_middleware(app, api_key: str): return await call_next(request) -def prepare_model_and_tokenizer(model_path: str, tokenizer_path: str): - if get_bool_env_var("SGLANG_USE_MODELSCOPE"): - if not os.path.exists(model_path): - from modelscope import snapshot_download - - model_path = snapshot_download(model_path) - tokenizer_path = snapshot_download( - tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"] - ) - return model_path, tokenizer_path - - def configure_logger(server_args, prefix: str = ""): if SGLANG_LOGGING_CONFIG_PATH := os.getenv("SGLANG_LOGGING_CONFIG_PATH"): if not os.path.exists(SGLANG_LOGGING_CONFIG_PATH): diff --git a/scripts/code_sync/guideline.md b/scripts/code_sync/guideline.md index 52f08eb4b..e9e94bb1d 100644 --- a/scripts/code_sync/guideline.md +++ b/scripts/code_sync/guideline.md @@ -25,3 +25,27 @@ It learns from [Copybara](https://github.com/google/copybara), a tool used at Go - For example, you can have a PR that changes both `python/sglang/srt` and `python/sglang/private/srt`. Once you merge the PR into the private repo, `python/sglang/srt` becomes desynced between the two repos. You need to run this action on your merge commit immediately to open a PR to send your diff to the OSS repo. Then, we need to merge the OSS PR as soon as possible. Once your OSS PR is merged, we can run action A again. - Action A copies files directly, but Action B applies diff. This is because OSS is the source of truth; action A can just copy files. Action B cannot copy, so it uses diff instead. - This action currently needs a manual trigger in order to prevent incidental code leaks. One can also consider making it automatic. + +## Examples +- If you want to have some private server arguments, you can create a new file `python/sglang/private/server_args.py`. It defines a class that inherits the oss ServerArgs. + ```python + from sglang.srt.server_args import ServerArgs as ServerArgsOSS + + @dataclasses.dataclass + class ServerArgs(ServerArgsOSS): + private_flag: str = "foo" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + # Get all public args + ServerArgsOSS.add_cli_args(parser) + + # Add your private flags + parser.add_argument( + "--private-flag", + type=str, + default=ServerArgs.private_flag, + ) + ``` +- Similarly, you can inherit `Engine` and override `launch_subprocesses_func`, `server_args_class`. +- You can pass your own subprocesses launch functions to `launch_server.py::launch_server` diff --git a/test/srt/quant/test_awq.py b/test/srt/quant/test_awq.py index 7eb936284..87e126adb 100644 --- a/test/srt/quant/test_awq.py +++ b/test/srt/quant/test_awq.py @@ -71,7 +71,7 @@ class TestAWQMarlinBfloat16(CustomTestCase): ) metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.85) + self.assertGreater(metrics["score"], 0.83) class TestAWQMarlinFloat16(CustomTestCase):