From 183b65190a8525afda50300b837cada7da756556 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sat, 27 Dec 2025 15:27:12 -0800 Subject: [PATCH] Clean up logging (#15919) --- .../device_communicators/custom_all_reduce.py | 26 +++++++++-- .../sglang/srt/distributed/parallel_state.py | 18 +------- .../sglang/srt/managers/tokenizer_manager.py | 43 +++++++++++-------- .../model_runner_kv_cache_mixin.py | 1 - python/sglang/srt/server_args.py | 9 ++-- .../sglang/srt/utils/hf_transformers_utils.py | 2 +- python/sglang/test/test_utils.py | 1 + .../models/test_nvidia_nemotron_nano_v2_vl.py | 2 +- test/srt/test_vision_chunked_prefill.py | 2 + 9 files changed, 57 insertions(+), 47 deletions(-) diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py index d4e71f535..0d5f612ce 100644 --- a/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py @@ -433,6 +433,25 @@ def dispatch_custom_allreduce(): On AMD with 1-stage AR enabled, use sglang's CustomAllreduce (has deterministic_all_reduce method). Otherwise use AiterCustomAllreduce if available. """ + if _is_cuda: + return CustomAllreduce + + assert _is_hip + + if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set(): + if envs.SGLANG_USE_1STAGE_ALLREDUCE.get(): + logger.debug( + "[AR] All-reduce: 1-stage kernel (SGLANG_USE_1STAGE_ALLREDUCE=1)" + ) + else: + logger.debug("[AR] All-reduce: default (SGLANG_USE_1STAGE_ALLREDUCE=0)") + elif envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get(): + logger.debug( + "[AR] All-reduce: 1-stage kernel (deterministic inference enabled)" + ) + else: + logger.debug("[AR] All-reduce: default") + # Check if 1-stage AR should be used if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set(): use_1stage = envs.SGLANG_USE_1STAGE_ALLREDUCE.get() @@ -441,11 +460,10 @@ def dispatch_custom_allreduce(): # On AMD with 1-stage AR, use sglang's CustomAllreduce # (AiterCustomAllreduce doesn't have deterministic_all_reduce method) - if is_hip() and use_1stage: - logger.info("[AR] Using sglang CustomAllreduce (1-stage kernel)") + if use_1stage: return CustomAllreduce - if is_hip() and get_bool_env_var("SGLANG_USE_AITER_AR", default="true"): + if get_bool_env_var("SGLANG_USE_AITER_AR", default="true"): try: from aiter.dist.device_communicators.custom_all_reduce import ( CustomAllreduce as AiterCustomAllreduce, @@ -460,5 +478,5 @@ def dispatch_custom_allreduce(): e, ) return CustomAllreduce - logger.info("[AR] Using sglang CustomAllreduce") + return CustomAllreduce diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 448b4d30d..8f0cda98d 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -376,28 +376,12 @@ class GroupCoordinator: group=self.cpu_group, device=self.device, ) - # Log which all-reduce mode will be used - if is_hip(): - if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set(): - if envs.SGLANG_USE_1STAGE_ALLREDUCE.get(): - logger.info( - "[AR] All-reduce: 1-stage kernel (SGLANG_USE_1STAGE_ALLREDUCE=1)" - ) - else: - logger.info( - "[AR] All-reduce: default (SGLANG_USE_1STAGE_ALLREDUCE=0)" - ) - elif envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get(): - logger.info( - "[AR] All-reduce: 1-stage kernel (deterministic inference enabled)" - ) - else: - logger.info("[AR] All-reduce: default") except Exception as e: logger.warning( f"Setup Custom allreduce failed with {e}. To silence this " "warning, specify --disable-custom-all-reduce explicitly." ) + if is_hip(): try: # Initialize a custom quick all-reduce implementation for AMD diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 429157bc2..5ebd1c207 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -185,27 +185,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi self.preferred_sampling_params = server_args.preferred_sampling_params self.crash_dump_folder = server_args.crash_dump_folder self.enable_trace = server_args.enable_trace - - # Read model args - self.model_path = server_args.model_path - self.served_model_name = server_args.served_model_name - self.model_config = ModelConfig.from_server_args(server_args) - self.is_generation = self.model_config.is_generation - self.is_image_gen = self.model_config.is_image_gen - self.context_len = self.model_config.context_len - self.image_token_id = self.model_config.image_token_id - self.max_req_input_len = None # Will be set later in engine.py - speculative_algorithm = SpeculativeAlgorithm.from_string( - server_args.speculative_algorithm - ) - self.reserve_input_token_num = ( - 0 - if speculative_algorithm.is_none() - else server_args.speculative_num_draft_tokens - ) - set_global_server_args_for_tokenizer(server_args) + # Init model config + self.init_model_config() + # Initialize tokenizer and multimodalprocessor self.init_tokenizer_and_processor() @@ -233,6 +217,27 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi # Init request dispatcher self.init_request_dispatcher() + def init_model_config(self): + server_args = self.server_args + + # Read model args + self.model_path = server_args.model_path + self.served_model_name = server_args.served_model_name + self.model_config = ModelConfig.from_server_args(server_args) + self.is_generation = self.model_config.is_generation + self.is_image_gen = self.model_config.is_image_gen + self.context_len = self.model_config.context_len + self.image_token_id = self.model_config.image_token_id + self.max_req_input_len = None # Will be set later in engine.py + speculative_algorithm = SpeculativeAlgorithm.from_string( + server_args.speculative_algorithm + ) + self.reserve_input_token_num = ( + 0 + if speculative_algorithm.is_none() + else server_args.speculative_num_draft_tokens + ) + def init_tokenizer_and_processor(self): server_args = self.server_args diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 0d7d89a2d..fc753436a 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -140,7 +140,6 @@ class ModelRunnerKVCacheMixin: if self.mambaish_config is not None: rest_memory = self.handle_max_mamba_cache(rest_memory) - logger.info(f"The available memory for KV cache is {rest_memory:.2f} GB.") return int(rest_memory * (1 << 30)) // cell_size def handle_max_mamba_cache(self: ModelRunner, total_rest_memory): diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 22c99f11a..84fcfd7cb 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1540,7 +1540,7 @@ class ServerArgs: else: self.attention_backend = "triton" - logger.warning( + logger.info( f"Attention backend not specified. Use {self.attention_backend} backend by default." ) @@ -1981,19 +1981,20 @@ class ServerArgs: ): self.disable_overlap_schedule = False logger.warning( - "Beta spec is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on." + "Spec v2 is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on." ) if ( self.speculative_eagle_topk is not None and self.speculative_eagle_topk > 1 ): raise ValueError( - "Beta spec currently only supports topk = 1 for speculative decoding." + "Spec v2 currently only supports topk = 1 for speculative decoding." ) else: self.disable_overlap_schedule = True logger.warning( - "Overlap scheduler is disabled when beta spec is off or using unsupported speculative algorithm." + "Overlap scheduler is disabled when spec v2 is off or using unsupported speculative algorithm. " + "You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler. " ) if self.enable_mixed_chunk: diff --git a/python/sglang/srt/utils/hf_transformers_utils.py b/python/sglang/srt/utils/hf_transformers_utils.py index f0db619ee..04ea73141 100644 --- a/python/sglang/srt/utils/hf_transformers_utils.py +++ b/python/sglang/srt/utils/hf_transformers_utils.py @@ -125,7 +125,7 @@ def get_hf_text_config(config: PretrainedConfig): # read the wrong values from the unused default text_config. # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`. - setattr(config, "torch_dtype", torch.float16) + setattr(config, "dtype", torch.float16) return config if hasattr(config, "text_config"): diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 71e237bde..b724f5205 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -683,6 +683,7 @@ def popen_launch_server( response = session.get( f"{base_url}/health_generate", headers=headers, + timeout=5, ) if response.status_code == 200: return process diff --git a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py b/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py index 9caca5bdb..9ba105a15 100644 --- a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py +++ b/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py @@ -15,7 +15,7 @@ class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, DefaultServerBase): class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMTestBase): - accuracy = 0.454 + accuracy = 0.444 model = MODEL other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"] mmmu_args = ["--limit=0.1"] diff --git a/test/srt/test_vision_chunked_prefill.py b/test/srt/test_vision_chunked_prefill.py index bfcd90114..7236bb2eb 100644 --- a/test/srt/test_vision_chunked_prefill.py +++ b/test/srt/test_vision_chunked_prefill.py @@ -199,6 +199,7 @@ class TestVisionChunkedPrefill(CustomTestCase): logger.info(f"Killing chunked server pid={chunked_server_pid}") kill_process_tree(chunked_server_pid) logger.info("Chunked server killed") + time.sleep(4) # None-chunked logger.info("Phase 2: Testing with chunked_prefill_size=-1 (no chunking)") @@ -218,6 +219,7 @@ class TestVisionChunkedPrefill(CustomTestCase): logger.info(f"Killing non-chunked server pid={no_chunked_server_pid}") kill_process_tree(no_chunked_server_pid) logger.info("Non-chunked server killed") + time.sleep(4) for output_chunked, output_no_chunked in zip( outputs_chunked, outputs_no_chunked