diff --git a/README.md b/README.md index 3ae194934..0553ab853 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,10 @@ | [**Slides**](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#slides) | ## News +- [2025/12] SGLang provides day-0 support for latest open models ([MiNo-V2](https://lmsys.org/blog/2025-12-16-mimo-v2-flash/), [Nemotron 3 Nano](https://lmsys.org/blog/2025-12-15-run-nvidia-nemotron-3-nano/), [Mistral Large 3](https://github.com/sgl-project/sglang/pull/14213), [LLaDA 2.0 Diffusion LLM](https://x.com/lmsysorg/status/1999378073125552375?s=20), [MiniMax M2](https://lmsys.org/blog/2025-11-04-miminmax-m2/)). - [2025/11] 🔥 SGLang Diffusion accelerates video and image generation ([blog](https://lmsys.org/blog/2025-11-07-sglang-diffusion/)). - [2025/10] 🔥 SGLang now runs natively on TPU with the SGLang-Jax backend ([blog](https://lmsys.org/blog/2025-10-29-sglang-jax/)). -- [2025/10] PyTorch Conference 2025 SGLang Talk ([slide](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/sglang_pytorch_2025.pdf)). -- [2025/09] 🔥 Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part II): 3.8x Prefill, 4.8x Decode Throughput ([blog](https://lmsys.org/blog/2025-09-25-gb200-part-2/)). +- [2025/09] Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part II): 3.8x Prefill, 4.8x Decode Throughput ([blog](https://lmsys.org/blog/2025-09-25-gb200-part-2/)). - [2025/09] SGLang Day 0 Support for DeepSeek-V3.2 with Sparse Attention ([blog](https://lmsys.org/blog/2025-09-29-deepseek-V32/)). - [2025/08] SGLang x AMD SF Meetup on 8/22: Hands-on GPU workshop, tech talks by AMD/xAI/SGLang, and networking ([Roadmap](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_roadmap.pdf), [Large-scale EP](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_ep.pdf), [Highlights](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_highlights.pdf), [AITER/MoRI](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_aiter_mori.pdf), [Wave](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_wave.pdf)). - [2025/08] SGLang provides day-0 support for OpenAI gpt-oss model ([instructions](https://github.com/sgl-project/sglang/issues/8833)) @@ -32,6 +32,7 @@
More +- [2025/10] PyTorch Conference 2025 SGLang Talk ([slide](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/sglang_pytorch_2025.pdf)). - [2025/10] SGLang x Nvidia SF Meetup on 10/2 ([recap](https://x.com/lmsysorg/status/1975339501934510231)). - [2025/06] SGLang, the high-performance serving infrastructure powering trillions of tokens daily, has been awarded the third batch of the Open Source AI Grant by a16z ([a16z blog](https://a16z.com/advancing-open-source-ai-through-benchmarks-and-bold-experimentation/)). - [2025/06] Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part I): 2.7x Higher Decoding Throughput ([blog](https://lmsys.org/blog/2025-06-16-gb200-part-1/)). diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e491e41e4..a58219ce7 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -292,13 +292,8 @@ class Scheduler( ) ) - # Init model config - self.model_config = ModelConfig.from_server_args(server_args) - self.dllm_config = ( # For diffusion LLM - DllmConfig.from_server_args(server_args) - if server_args.dllm_algorithm is not None - else None - ) + # Init model configs + self.init_model_config() # Init metrics stats self.init_metrics(tp_rank, pp_rank, dp_rank) @@ -306,7 +301,7 @@ class Scheduler( # Init inter-process communication self.init_sockets(server_args, port_args) - # Init pdmux context + # Init PD-multiplexing context if self.enable_pdmux: self.init_pdmux() @@ -316,10 +311,10 @@ class Scheduler( # Init moe config and GEMM config (FP8 GEMM, etc.) self.init_moe_gemm_config() - # Launch a tensor parallel worker + # Launch a model worker and draft model worker if using speculative decoding self.init_model_worker() - # Init cache using the existing memory pool + # Init cache and memory pool self.init_cache_with_memory_pool() # Init running status @@ -340,10 +335,10 @@ class Scheduler( # Init profiler self.init_profiler() - # Init disaggregation + # Init prefill-decodedisaggregation self.init_disaggregation() - # Init overlap + # Init overlap schedule self.init_overlap() # Init prefill kv split size when deterministic inference is enabled with various attention backends @@ -352,6 +347,14 @@ class Scheduler( # Init request dispatcher self.init_request_dispatcher() + def init_model_config(self): + self.model_config = ModelConfig.from_server_args(self.server_args) + self.dllm_config = ( # For diffusion LLM + DllmConfig.from_server_args(self.server_args) + if self.server_args.dllm_algorithm is not None + else None + ) + def init_sockets(self, server_args: ServerArgs, port_args: PortArgs): context = zmq.Context(2) self.idle_sleeper = None