chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Clean up server args and engine startup processes (#15015)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-12-12 18:46:07 -08:00
committed by GitHub
parent 313f59ad80
commit 267170bf1d
7 changed files with 105 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
python/pyproject.toml
View File

@@ -98,9 +98,12 @@ diffusion = [
"cache-dit==1.1.8"
]
[tool.uv.extra-build-dependencies]
st-attn = ["torch", "setuptools"]
vsa = ["torch", "setuptools"]
tracing = [
"opentelemetry-api",
"opentelemetry-exporter-otlp",
"opentelemetry-exporter-otlp-proto-grpc",
"opentelemetry-sdk",
]
test = [
"accelerate",
@@ -109,18 +112,18 @@ test = [
"jsonlines",
"matplotlib",
"pandas",
"parameterized",
"peft",
"pytest",
"sentence_transformers",
"tabulate",
]
dev = ["sglang[test]"]
tracing = [
"opentelemetry-api",
"opentelemetry-exporter-otlp",
"opentelemetry-exporter-otlp-proto-grpc",
"opentelemetry-sdk",
]
[tool.uv.extra-build-dependencies]
st-attn = ["torch", "setuptools"]
vsa = ["torch", "setuptools"]
[project.urls]
"Homepage" = "https://github.com/sgl-project/sglang"

43
python/sglang/srt/entrypoints/engine.py
View File

@@ -76,7 +76,6 @@ from sglang.srt.utils import (
launch_dummy_health_check_server,
maybe_reindex_device_id,
numa_utils,
prepare_model_and_tokenizer,
set_prometheus_multiproc_dir,
set_ulimit,
)
@@ -105,11 +104,6 @@ def _launch_subprocesses(
port_args = PortArgs.init_new(server_args)
logger.info(f"{server_args=}")
# If using model from www.modelscope.cn, first download the model
server_args.model_path, server_args.tokenizer_path = prepare_model_and_tokenizer(
server_args.model_path, server_args.tokenizer_path
)
# Launch scheduler processes
scheduler_procs, scheduler_pipe_readers = _launch_scheduler_processes(
server_args=server_args,
@@ -826,22 +820,23 @@ def _set_envs_and_config(server_args: ServerArgs):
set_ulimit()
# Check flashinfer version
if server_args.attention_backend == "flashinfer":
assert_pkg_version(
"flashinfer_python",
"0.5.3",
"Please uninstall the old version and "
"reinstall the latest version by following the instructions "
"at https://docs.flashinfer.ai/installation.html.",
)
if _is_cuda and not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"):
assert_pkg_version(
"sgl-kernel",
"0.3.19",
"Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`",
)
if not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"):
if server_args.attention_backend == "flashinfer":
assert_pkg_version(
"flashinfer_python",
"0.5.3",
"Please uninstall the old version and "
"reinstall the latest version by following the instructions "
"at https://docs.flashinfer.ai/installation.html.",
)
if _is_cuda:
assert_pkg_version(
"sgl-kernel",
"0.3.19",
"Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`",
)
if True: # Keep this check for internal code compatibility
if server_args.custom_sigquit_handler is None:
# Register the signal handler.
# The child processes will send SIGQUIT to this process when any error happens
# This process then clean up the whole process tree
@@ -854,6 +849,12 @@ def _set_envs_and_config(server_args: ServerArgs):
kill_process_tree(os.getpid())
signal.signal(signal.SIGQUIT, launch_phase_sigquit_handler)
else:
# Allow users to register a custom SIGQUIT handler for things like crash dump
logger.error(
f"Using custom SIGQUIT handler: {server_args.custom_sigquit_handler}"
)
signal.signal(signal.SIGQUIT, server_args.custom_sigquit_handler)
# Set mp start method
mp.set_start_method("spawn", force=True)

82
python/sglang/srt/server_args.py
View File

@@ -23,7 +23,7 @@ import logging
import os
import random
import tempfile
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Callable, Dict, List, Literal, Optional, Union
import orjson
@@ -252,7 +252,6 @@ class ServerArgs:
skip_tokenizer_init: bool = False
load_format: str = "auto"
model_loader_extra_config: str = "{}"
rl_quant_profile: Optional[str] = None # For flash_rl load format
trust_remote_code: bool = False
context_length: Optional[int] = None
is_embedding: bool = False
@@ -281,6 +280,7 @@ class ServerArgs:
modelopt_checkpoint_save_path: Optional[str] = None
modelopt_export_path: Optional[str] = None
quantize_and_serve: bool = False
rl_quant_profile: Optional[str] = None # For flash_rl load format
# Memory and scheduling
mem_fraction_static: Optional[float] = None
@@ -320,7 +320,7 @@ class ServerArgs:
base_gpu_id: int = 0
gpu_id_step: int = 1
sleep_on_idle: bool = False
mm_process_config: Optional[Dict[str, Any]] = None
custom_sigquit_handler: Optional[Callable] = None
# Logging
log_level: str = "info"
@@ -606,14 +606,13 @@ class ServerArgs:
mm_max_concurrent_calls: int = 32
mm_per_request_timeout: float = 10.0
enable_broadcast_mm_inputs_process: bool = False
mm_enable_dp_encoder: bool = False
mm_process_config: Optional[Dict[str, Any]] = None
# For checkpoint decryption
decrypted_config_file: Optional[str] = None
decrypted_draft_config_file: Optional[str] = None
# For encoder dp
mm_enable_dp_encoder: bool = False
# For forward hooks
forward_hooks: Optional[List[dict[str, Any]]] = None
@@ -643,9 +642,6 @@ class ServerArgs:
self._handle_cpu_backends()
self._handle_npu_backends()
# Handle compilation config
self._handle_compilation_cfg()
# Apply model-specific adjustments.
self._handle_model_specific_adjustments()
@@ -709,7 +705,7 @@ class ServerArgs:
self._handle_elastic_ep()
def _handle_deprecated_args(self):
# handle deprecated tool call parsers
# Handle deprecated tool call parsers
deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"}
if self.tool_call_parser in deprecated_tool_call_parsers:
logger.warning(
@@ -729,6 +725,16 @@ class ServerArgs:
if self.mm_process_config is None:
self.mm_process_config = {}
# Handle ModelScope model downloads
if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
if not os.path.exists(self.model_path):
from modelscope import snapshot_download
self.model_path = snapshot_download(self.model_path)
self.tokenizer_path = snapshot_download(
self.tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"]
)
def _handle_gpu_memory_settings(self, gpu_mem):
"""
Configure GPU memory-dependent settings including
@@ -908,7 +914,7 @@ class ServerArgs:
if self.disable_cuda_graph_padding:
capture_bs = list(range(1, self.cuda_graph_max_bs + 1))
elif self.speculative_algorithm is None:
# Normal case: [1, 2, 4, 8, 12] + list(range(16, 257, 8)) + list(range(272, 512, 16)) + list(range(512, cuda_graph_max_bs + 1))
# Normal case:
capture_bs = (
[1, 2, 4, 8, 12]
+ list(range(16, 257, 8))
@@ -916,7 +922,7 @@ class ServerArgs:
+ list(range(512, self.cuda_graph_max_bs + 1, 32))
)
else:
# Spec decoding case: list(range(1, 9, 1)) + list(range(10, 33, 2)) + list(range(40, 64, 4)) + list(range(72, 257, 8))
# Spec decoding case: less padding for smaller batch sizes
capture_bs = (
list(range(1, 9, 1))
+ list(range(10, 33, 2))
@@ -959,21 +965,19 @@ class ServerArgs:
self.attention_backend = "intel_amx"
self.sampling_backend = "pytorch"
def _handle_compilation_cfg(self):
# NPU platform
if is_npu() and self.piecewise_cuda_graph_compiler != "eager":
logger.warning(
"At this moment Ascend platform only support prefill graph compilation with "
"piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'."
)
self.piecewise_cuda_graph_compiler = "eager"
def _handle_npu_backends(self):
if self.device == "npu":
from sglang.srt.hardware_backend.npu.utils import set_default_server_args
set_default_server_args(self)
if self.piecewise_cuda_graph_compiler != "eager":
logger.warning(
"At this moment Ascend platform only support prefill graph compilation with "
"piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'."
)
self.piecewise_cuda_graph_compiler = "eager"
def _handle_model_specific_adjustments(self):
from sglang.srt.configs.model_config import is_deepseek_nsa
@@ -2283,12 +2287,6 @@ class ServerArgs:
"This will be passed to the model loader corresponding to the chosen load_format.",
default=ServerArgs.model_loader_extra_config,
)
parser.add_argument(
"--rl-quant-profile",
type=str,
default=ServerArgs.rl_quant_profile,
help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.",
)
parser.add_argument(
"--trust-remote-code",
action="store_true",
@@ -2464,6 +2462,12 @@ class ServerArgs:
"This is useful for development and prototyping. For production, it's recommended "
"to use separate quantization and deployment steps.",
)
parser.add_argument(
"--rl-quant-profile",
type=str,
default=ServerArgs.rl_quant_profile,
help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.",
)
# Memory and scheduling
parser.add_argument(
@@ -2687,10 +2691,8 @@ class ServerArgs:
help="Reduce CPU usage when sglang is idle.",
)
parser.add_argument(
"--mm-process-config",
type=json.loads,
default=ServerArgs.mm_process_config,
help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
"--custom-sigquit-handler",
help="Register a custom sigquit handler so you can do additional cleanup after the server is shutdown. This is only available for Engine, not for CLI.",
)
# Logging
@@ -4110,6 +4112,18 @@ class ServerArgs:
default=ServerArgs.enable_broadcast_mm_inputs_process,
help="Enable broadcast mm-inputs process in scheduler.",
)
parser.add_argument(
"--mm-process-config",
type=json.loads,
default=ServerArgs.mm_process_config,
help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
)
parser.add_argument(
"--mm-enable-dp-encoder",
action="store_true",
default=ServerArgs.mm_enable_dp_encoder,
help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
)
# For checkpoint decryption
parser.add_argument(
@@ -4124,12 +4138,6 @@ class ServerArgs:
default=ServerArgs.decrypted_draft_config_file,
help="The path of the decrypted draft config file.",
)
parser.add_argument(
"--mm-enable-dp-encoder",
action="store_true",
default=ServerArgs.mm_enable_dp_encoder,
help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
)
# For registering hooks
parser.add_argument(

12
python/sglang/srt/utils/common.py
View File

@@ -1145,18 +1145,6 @@ def add_api_key_middleware(app, api_key: str):
return await call_next(request)
def prepare_model_and_tokenizer(model_path: str, tokenizer_path: str):
if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
if not os.path.exists(model_path):
from modelscope import snapshot_download
model_path = snapshot_download(model_path)
tokenizer_path = snapshot_download(
tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"]
)
return model_path, tokenizer_path
def configure_logger(server_args, prefix: str = ""):
if SGLANG_LOGGING_CONFIG_PATH := os.getenv("SGLANG_LOGGING_CONFIG_PATH"):
if not os.path.exists(SGLANG_LOGGING_CONFIG_PATH):