diff --git a/.gitignore b/.gitignore index f25d4e85f..9f11f1a36 100644 --- a/.gitignore +++ b/.gitignore @@ -224,16 +224,11 @@ work_dirs/ *.exe *.out *.app - -compile_commands.json - *.iml # VSCode .vscode -1 - # Autoenv .env.leave @@ -243,6 +238,7 @@ Cargo.lock # Generated vision test fixtures (regenerate with: python scripts/generate_vision_golden.py) sgl-model-gateway/tests/fixtures/golden/ +# Other repos lmms-eval **/.serena/ @@ -262,7 +258,6 @@ inputs/ # setuptools-scm generated version file python/sglang/_version.py - # MUSA section # Generated source files by torchada sgl-kernel/csrc_musa/ diff --git a/python/sglang/cli/main.py b/python/sglang/cli/main.py index 36ed5ae5e..3cfef9cb7 100644 --- a/python/sglang/cli/main.py +++ b/python/sglang/cli/main.py @@ -14,13 +14,11 @@ def main(): # complex sub commands subparsers = parser.add_subparsers(dest="subcommand", required=True) - subparsers.add_parser( "serve", - help="Launch the SGLang server.", + help="Launch an SGLang server.", add_help=False, ) - subparsers.add_parser( "generate", help="Run inference on a multimodal model.", diff --git a/python/sglang/cli/serve.py b/python/sglang/cli/serve.py index 0388c8519..a9a1874cc 100644 --- a/python/sglang/cli/serve.py +++ b/python/sglang/cli/serve.py @@ -52,17 +52,13 @@ def serve(args, extra_argv): # we can't show the exact help. Instead, we show a general help message and then # the help for both possible server types. print( - "Usage: sglang serve --model-path [additional-arguments]\n" - ) - print( - "This command can launch either a standard language model server or a diffusion model server." - ) - print("The server type is determined by the model path.\n") - print( + "Usage: sglang serve --model-path [additional-arguments]\n\n" + "This command can launch either a standard language model server or a diffusion model server.\n" + "The server type is determined by the --model-path.\n" "Optional override: --model-type {auto,llm,diffusion} " - "(default: auto, fallback to LLM on detection failure).\n" + "(default: auto, fallback to LLM on detection failure)." ) - print("For specific arguments, please provide a model_path.") + print("\n--- Help for Standard Language Model Server ---") from sglang.srt.server_args import prepare_server_args @@ -72,13 +68,22 @@ def serve(args, extra_argv): pass # argparse --help calls sys.exit print("\n--- Help for Diffusion Model Server ---") - from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ( - add_multimodal_gen_serve_args, - ) + try: + from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ( + add_multimodal_gen_serve_args, + ) - parser = argparse.ArgumentParser(description="SGLang Diffusion Model Serving") - add_multimodal_gen_serve_args(parser) - parser.print_help() + parser = argparse.ArgumentParser( + prog="sglang serve", + description="SGLang Diffusion Model Serving", + ) + add_multimodal_gen_serve_args(parser) + parser.print_help() + except ImportError: + print( + "Diffusion model support is not available. " + 'Install with: pip install "sglang[diffusion]"' + ) return model_type, dispatch_argv = _extract_model_type_override(extra_argv) diff --git a/python/sglang/cli/utils.py b/python/sglang/cli/utils.py index e3e4d2032..1d867eccf 100644 --- a/python/sglang/cli/utils.py +++ b/python/sglang/cli/utils.py @@ -80,8 +80,7 @@ def get_model_path(extra_argv): raise Exception( "Usage: sglang serve --model-path [additional-arguments]\n\n" "This command can launch either a standard language model server or a diffusion model server.\n" - "The server type is determined by the model path.\n" - "For specific arguments, please provide a model_path." + "The server type is determined by the --model-path.\n" ) else: raise Exception( diff --git a/python/sglang/launch_server.py b/python/sglang/launch_server.py index fe06a9289..e1d05b1f8 100644 --- a/python/sglang/launch_server.py +++ b/python/sglang/launch_server.py @@ -3,6 +3,7 @@ import asyncio import os import sys +import warnings from sglang.srt.server_args import prepare_server_args from sglang.srt.utils import kill_process_tree @@ -14,6 +15,7 @@ suppress_noisy_warnings() def run_server(server_args): """Run the server based on server_args.grpc_mode and server_args.encoder_only.""" if server_args.encoder_only: + # For encoder disaggregation if server_args.grpc_mode: from sglang.srt.disaggregation.encode_grpc_server import ( serve_grpc_encoder, @@ -46,8 +48,6 @@ def run_server(server_args): if __name__ == "__main__": - import warnings - warnings.warn( "'python -m sglang.launch_server' is still supported, but " "'sglang serve' is the recommended entrypoint.\n" diff --git a/python/sglang/multimodal_gen/runtime/platforms/__init__.py b/python/sglang/multimodal_gen/runtime/platforms/__init__.py index 55aa8ad5d..fde775378 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/__init__.py +++ b/python/sglang/multimodal_gen/runtime/platforms/__init__.py @@ -51,7 +51,7 @@ def cuda_platform_plugin() -> str | None: if cuda_is_jetson(): is_cuda = True if is_cuda: - logger.info("CUDA is available") + logger.debug("CUDA is available") return ( "sglang.multimodal_gen.runtime.platforms.cuda.CudaPlatform" if is_cuda else None @@ -67,9 +67,9 @@ def mps_platform_plugin() -> str | None: if torch.backends.mps.is_available(): is_mps = True - logger.info("MPS (Metal Performance Shaders) is available") + logger.debug("MPS (Metal Performance Shaders) is available") except Exception as e: - logger.info("MPS detection failed: %s", e) + logger.debug("MPS detection failed: %s", e) return "sglang.multimodal_gen.runtime.platforms.mps.MpsPlatform" if is_mps else None @@ -90,11 +90,11 @@ def rocm_platform_plugin() -> str | None: try: if len(amdsmi.amdsmi_get_processor_handles()) > 0: is_rocm = True - logger.info("ROCm platform is available") + logger.debug("ROCm platform is available") finally: amdsmi.amdsmi_shut_down() except Exception as e: - logger.info("ROCm platform is unavailable: %s", e) + logger.debug("ROCm platform is unavailable: %s", e) return ( "sglang.multimodal_gen.runtime.platforms.rocm.RocmPlatform" if is_rocm else None @@ -109,9 +109,9 @@ def npu_platform_plugin() -> str | None: if torch.npu.is_available(): is_npu = True - logger.info("NPU is available") + logger.debug("NPU is available") except Exception as e: - logger.info("NPU detection failed: %s", e) + logger.debug("NPU detection failed: %s", e) return ( "sglang.multimodal_gen.runtime.platforms.npu.NPUPlatformBase" if is_npu @@ -131,7 +131,7 @@ def musa_platform_plugin() -> str | None: finally: pymtml.mtmlLibraryShutDown() except Exception as e: - logger.info("MUSA platform is unavailable: %s", e) + logger.debug("MUSA platform is unavailable: %s", e) return ( "sglang.multimodal_gen.runtime.platforms.musa.MusaPlatform" if is_musa else None diff --git a/python/sglang/srt/compilation/backend.py b/python/sglang/srt/compilation/backend.py index 8af025707..f9d376e95 100644 --- a/python/sglang/srt/compilation/backend.py +++ b/python/sglang/srt/compilation/backend.py @@ -21,7 +21,7 @@ from sglang.srt.compilation.compiler_interface import EagerAdapter, InductorAdap from sglang.srt.compilation.cuda_piecewise_backend import CUDAPiecewiseBackend from sglang.srt.compilation.npu_piecewise_backend import NPUPiecewiseBackend from sglang.srt.compilation.pass_manager import PostGradPassManager -from sglang.srt.utils.common import is_npu, rank0_log +from sglang.srt.utils.common import is_npu logger = logging.getLogger(__name__) @@ -375,7 +375,6 @@ class SGLangBackend: config: CompilationConfig, graph_pool: Any, ): - rank0_log(f"Initializing SGLangBackend") assert graph_pool is not None self.graph_pool = graph_pool @@ -394,7 +393,6 @@ class SGLangBackend: self.inductor_config["post_grad_custom_post_pass"] = self.post_grad_pass_manager def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: - rank0_log(f"SGLangBackend __call__") base_cache_dir = os.path.expanduser( os.getenv("SGLANG_CACHE_DIR", "~/.cache/sglang/") ) @@ -466,7 +464,5 @@ class SGLangBackend: with open(graph_path, "w") as f: f.write(src) - rank0_log(f"Computation graph saved to {graph_path}") - self._called = True return self.split_gm diff --git a/python/sglang/srt/compilation/compile.py b/python/sglang/srt/compilation/compile.py index 46a9240fb..448c1beb0 100644 --- a/python/sglang/srt/compilation/compile.py +++ b/python/sglang/srt/compilation/compile.py @@ -10,7 +10,6 @@ import torch from sglang.srt.compilation.compilation_config import CompilationConfig from sglang.srt.compilation.piecewise_context_manager import is_in_piecewise_cuda_graph -from sglang.srt.utils.common import rank0_log logger = logging.getLogger(__name__) @@ -118,7 +117,6 @@ def install_torch_compiled( fullgraph: bool = True, graph_pool: Any = None, ): - rank0_log(f"install_torch_compiled") unbound_fwd = module.__class__.forward if not callable(unbound_fwd): raise TypeError("module.__class__.forward must be callable") diff --git a/python/sglang/srt/layers/attention/mamba/ops/ssu_dispatch.py b/python/sglang/srt/layers/attention/mamba/ops/ssu_dispatch.py index 77d586eee..2854e5fa4 100644 --- a/python/sglang/srt/layers/attention/mamba/ops/ssu_dispatch.py +++ b/python/sglang/srt/layers/attention/mamba/ops/ssu_dispatch.py @@ -198,7 +198,7 @@ def initialize_mamba_selective_state_update_backend(server_args: ServerArgs) -> f"--mamba-backend value." ) - logger.info( + logger.debug( "Mamba selective_state_update backend initialized: %s", _mamba_ssu_backend.name, ) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 6bac100b1..ae87216e9 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -25,6 +25,10 @@ from dataclasses import dataclass from http import HTTPStatus from typing import Any, Deque, Dict, List, Optional, Tuple, Union +from sglang.srt.utils.common import suppress_noisy_warnings + +suppress_noisy_warnings() + import psutil import setproctitle import torch diff --git a/python/sglang/srt/observability/scheduler_metrics_mixin.py b/python/sglang/srt/observability/scheduler_metrics_mixin.py index eee09ff72..052c23645 100644 --- a/python/sglang/srt/observability/scheduler_metrics_mixin.py +++ b/python/sglang/srt/observability/scheduler_metrics_mixin.py @@ -258,9 +258,6 @@ class SchedulerMetricsMixin: if self.disaggregation_mode == DisaggregationMode.PREFILL: msg += f"#prealloc-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, " msg += f"#inflight-req: {len(self.disagg_prefill_inflight_queue)}, " - msg += f"input throughput (token/s): {self.last_input_throughput:.2f}, " - else: - msg += f"input throughput (token/s): {self.last_input_throughput:.2f}, " if ( self.server_args.language_only @@ -275,7 +272,8 @@ class SchedulerMetricsMixin: }, ) - msg += f"{graph_backend[self.device]}: {can_run_cuda_graph}" + msg += f"{graph_backend[self.device]}: {can_run_cuda_graph}, " + msg += f"input throughput (token/s): {self.last_input_throughput:.2f}" if self.is_stats_logging_rank: logger.info(msg) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 4f0f6c60d..e0421138a 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -6282,7 +6282,7 @@ def prepare_server_args(argv: List[str]) -> ServerArgs: Returns: The server arguments. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(prog="sglang serve") ServerArgs.add_cli_args(parser) # Check for config file and merge arguments if present diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index cc812c297..9a9dcf542 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -121,7 +121,7 @@ builtins.FP8_E4M3_MIN = FP8_E4M3_MIN # this makes it impossible to see the animation in the progress bar # but will avoid messing up with ray or multiprocessing, which wraps # each line of output with some prefix. -BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]\n" # noqa: E501 +BAR_FORMAT = "{desc}: {percentage:3.0f}% Completed | {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]" @lru_cache(maxsize=1)