chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Clean up logging (#15919)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-12-27 15:27:12 -08:00
committed by GitHub
parent 2af955e16c
commit 183b65190a
9 changed files with 57 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
python/sglang/srt/distributed/device_communicators/custom_all_reduce.py
View File

@@ -433,6 +433,25 @@ def dispatch_custom_allreduce():
On AMD with 1-stage AR enabled, use sglang's CustomAllreduce (has deterministic_all_reduce method).
Otherwise use AiterCustomAllreduce if available.
"""
if _is_cuda:
return CustomAllreduce
assert _is_hip
if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set():
if envs.SGLANG_USE_1STAGE_ALLREDUCE.get():
logger.debug(
"[AR] All-reduce: 1-stage kernel (SGLANG_USE_1STAGE_ALLREDUCE=1)"
)
else:
logger.debug("[AR] All-reduce: default (SGLANG_USE_1STAGE_ALLREDUCE=0)")
elif envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get():
logger.debug(
"[AR] All-reduce: 1-stage kernel (deterministic inference enabled)"
)
else:
logger.debug("[AR] All-reduce: default")
# Check if 1-stage AR should be used
if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set():
use_1stage = envs.SGLANG_USE_1STAGE_ALLREDUCE.get()
@@ -441,11 +460,10 @@ def dispatch_custom_allreduce():
# On AMD with 1-stage AR, use sglang's CustomAllreduce
# (AiterCustomAllreduce doesn't have deterministic_all_reduce method)
if is_hip() and use_1stage:
logger.info("[AR] Using sglang CustomAllreduce (1-stage kernel)")
if use_1stage:
return CustomAllreduce
if is_hip() and get_bool_env_var("SGLANG_USE_AITER_AR", default="true"):
if get_bool_env_var("SGLANG_USE_AITER_AR", default="true"):
try:
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as AiterCustomAllreduce,
@@ -460,5 +478,5 @@ def dispatch_custom_allreduce():
e,
)
return CustomAllreduce
logger.info("[AR] Using sglang CustomAllreduce")
return CustomAllreduce

18
python/sglang/srt/distributed/parallel_state.py
View File

@@ -376,28 +376,12 @@ class GroupCoordinator:
group=self.cpu_group,
device=self.device,
)
# Log which all-reduce mode will be used
if is_hip():
if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set():
if envs.SGLANG_USE_1STAGE_ALLREDUCE.get():
logger.info(
"[AR] All-reduce: 1-stage kernel (SGLANG_USE_1STAGE_ALLREDUCE=1)"
)
else:
logger.info(
"[AR] All-reduce: default (SGLANG_USE_1STAGE_ALLREDUCE=0)"
)
elif envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get():
logger.info(
"[AR] All-reduce: 1-stage kernel (deterministic inference enabled)"
)
else:
logger.info("[AR] All-reduce: default")
except Exception as e:
logger.warning(
f"Setup Custom allreduce failed with {e}. To silence this "
"warning, specify --disable-custom-all-reduce explicitly."
)
if is_hip():
try:
# Initialize a custom quick all-reduce implementation for AMD

43
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -185,27 +185,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
self.preferred_sampling_params = server_args.preferred_sampling_params
self.crash_dump_folder = server_args.crash_dump_folder
self.enable_trace = server_args.enable_trace
# Read model args
self.model_path = server_args.model_path
self.served_model_name = server_args.served_model_name
self.model_config = ModelConfig.from_server_args(server_args)
self.is_generation = self.model_config.is_generation
self.is_image_gen = self.model_config.is_image_gen
self.context_len = self.model_config.context_len
self.image_token_id = self.model_config.image_token_id
self.max_req_input_len = None # Will be set later in engine.py
speculative_algorithm = SpeculativeAlgorithm.from_string(
server_args.speculative_algorithm
)
self.reserve_input_token_num = (
0
if speculative_algorithm.is_none()
else server_args.speculative_num_draft_tokens
)
set_global_server_args_for_tokenizer(server_args)
# Init model config
self.init_model_config()
# Initialize tokenizer and multimodalprocessor
self.init_tokenizer_and_processor()
@@ -233,6 +217,27 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
# Init request dispatcher
self.init_request_dispatcher()
def init_model_config(self):
server_args = self.server_args
# Read model args
self.model_path = server_args.model_path
self.served_model_name = server_args.served_model_name
self.model_config = ModelConfig.from_server_args(server_args)
self.is_generation = self.model_config.is_generation
self.is_image_gen = self.model_config.is_image_gen
self.context_len = self.model_config.context_len
self.image_token_id = self.model_config.image_token_id
self.max_req_input_len = None # Will be set later in engine.py
speculative_algorithm = SpeculativeAlgorithm.from_string(
server_args.speculative_algorithm
)
self.reserve_input_token_num = (
0
if speculative_algorithm.is_none()
else server_args.speculative_num_draft_tokens
)
def init_tokenizer_and_processor(self):
server_args = self.server_args

1
python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py
View File

@@ -140,7 +140,6 @@ class ModelRunnerKVCacheMixin:
if self.mambaish_config is not None:
rest_memory = self.handle_max_mamba_cache(rest_memory)
logger.info(f"The available memory for KV cache is {rest_memory:.2f} GB.")
return int(rest_memory * (1 << 30)) // cell_size
def handle_max_mamba_cache(self: ModelRunner, total_rest_memory):

9
python/sglang/srt/server_args.py
View File

@@ -1540,7 +1540,7 @@ class ServerArgs:
else:
self.attention_backend = "triton"
logger.warning(
logger.info(
f"Attention backend not specified. Use {self.attention_backend} backend by default."
)
@@ -1981,19 +1981,20 @@ class ServerArgs:
):
self.disable_overlap_schedule = False
logger.warning(
"Beta spec is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
"Spec v2 is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
)
if (
self.speculative_eagle_topk is not None
and self.speculative_eagle_topk > 1
):
raise ValueError(
"Beta spec currently only supports topk = 1 for speculative decoding."
"Spec v2 currently only supports topk = 1 for speculative decoding."
)
else:
self.disable_overlap_schedule = True
logger.warning(
"Overlap scheduler is disabled when beta spec is off or using unsupported speculative algorithm."
"Overlap scheduler is disabled when spec v2 is off or using unsupported speculative algorithm. "
"You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler. "
)
if self.enable_mixed_chunk:

2
python/sglang/srt/utils/hf_transformers_utils.py
View File

@@ -125,7 +125,7 @@ def get_hf_text_config(config: PretrainedConfig):
# read the wrong values from the unused default text_config.
# NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
# `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
setattr(config, "torch_dtype", torch.float16)
setattr(config, "dtype", torch.float16)
return config
if hasattr(config, "text_config"):

1
python/sglang/test/test_utils.py
View File

@@ -683,6 +683,7 @@ def popen_launch_server(
response = session.get(
f"{base_url}/health_generate",
headers=headers,
timeout=5,
)
if response.status_code == 200:
return process