From db7299aa30de2a3f7c55860d6825fab8b5f54bac Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sat, 15 Nov 2025 17:48:17 -0800 Subject: [PATCH] [Fix] Register custom ops only if they exist (#13321) --- README.md | 8 +++--- python/sglang/srt/entrypoints/engine.py | 8 ++++-- python/sglang/srt/layers/quantization/awq.py | 5 ++-- python/sglang/srt/layers/quantization/gptq.py | 14 +++++----- .../srt/managers/detokenizer_manager.py | 18 +----------- python/sglang/srt/utils/patch_torch.py | 28 ++++++++++++++++++- 6 files changed, 47 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index d3f36aa7d..327336de0 100644 --- a/README.md +++ b/README.md @@ -20,23 +20,23 @@ | [**Slides**](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#slides) | ## News -- [2025/11] 🔥 SGLang diffusion is now available ([blog](https://lmsys.org/blog/2025-11-07-sglang-diffusion/)). +- [2025/11] 🔥 SGLang Diffusion accelerates video and image generation ([blog](https://lmsys.org/blog/2025-11-07-sglang-diffusion/)). - [2025/10] 🔥 SGLang now runs natively on TPU with the SGLang-Jax backend ([blog](https://lmsys.org/blog/2025-10-29-sglang-jax/)). -- [2025/10] AMD AI Dev Day 2025 SGLang ([slide](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/sglang_amd_ai_devday_2025.pdf)), PyTorch Conference 2025 SGLang ([slide](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/sglang_pytorch_2025.pdf)). -- [2025/10] SGLang x Nvidia SF Meetup on 10/2 ([recap](https://x.com/lmsysorg/status/1975339501934510231)). +- [2025/10] PyTorch Conference 2025 SGLang Talk ([slide](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/sglang_pytorch_2025.pdf)). - [2025/09] 🔥 Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part II): 3.8x Prefill, 4.8x Decode Throughput ([blog](https://lmsys.org/blog/2025-09-25-gb200-part-2/)). - [2025/09] SGLang Day 0 Support for DeepSeek-V3.2 with Sparse Attention ([blog](https://lmsys.org/blog/2025-09-29-deepseek-V32/)). - [2025/08] SGLang x AMD SF Meetup on 8/22: Hands-on GPU workshop, tech talks by AMD/xAI/SGLang, and networking ([Roadmap](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_roadmap.pdf), [Large-scale EP](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_ep.pdf), [Highlights](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_highlights.pdf), [AITER/MoRI](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_aiter_mori.pdf), [Wave](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_wave.pdf)). - [2025/08] SGLang provides day-0 support for OpenAI gpt-oss model ([instructions](https://github.com/sgl-project/sglang/issues/8833)) - [2025/05] Deploying DeepSeek with PD Disaggregation and Large-scale Expert Parallelism on 96 H100 GPUs ([blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/)). -- [2025/03] SGLang Joins PyTorch Ecosystem: Efficient LLM Serving Engine ([PyTorch blog](https://pytorch.org/blog/sglang-joins-pytorch/))
More +- [2025/10] SGLang x Nvidia SF Meetup on 10/2 ([recap](https://x.com/lmsysorg/status/1975339501934510231)). - [2025/06] SGLang, the high-performance serving infrastructure powering trillions of tokens daily, has been awarded the third batch of the Open Source AI Grant by a16z ([a16z blog](https://a16z.com/advancing-open-source-ai-through-benchmarks-and-bold-experimentation/)). - [2025/06] Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part I): 2.7x Higher Decoding Throughput ([blog](https://lmsys.org/blog/2025-06-16-gb200-part-1/)). - [2025/03] Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html)) +- [2025/03] SGLang Joins PyTorch Ecosystem: Efficient LLM Serving Engine ([PyTorch blog](https://pytorch.org/blog/sglang-joins-pytorch/)) - [2025/02] Unlock DeepSeek-R1 Inference Performance on AMD Instinctâ„¢ MI300X GPU ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1_Perf/README.html)) - [2025/01] SGLang provides day one support for DeepSeek V3/R1 models on NVIDIA and AMD GPUs with DeepSeek-specific optimizations. ([instructions](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3), [AMD blog](https://www.amd.com/en/developer/resources/technical-articles/amd-instinct-gpus-power-deepseek-v3-revolutionizing-ai-development-with-sglang.html), [10+ other companies](https://x.com/lmsysorg/status/1887262321636221412)) - [2024/12] v0.4 Release: Zero-Overhead Batch Scheduler, Cache-Aware Load Balancer, Faster Structured Outputs ([blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/)). diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index b2e057b62..6ab02f935 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -683,7 +683,6 @@ class Engine(EngineBase): def _set_envs_and_config(server_args: ServerArgs): # Set global environments - os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" if "NCCL_CUMEM_ENABLE" not in os.environ or server_args.enable_symm_mem: os.environ["NCCL_CUMEM_ENABLE"] = str(int(server_args.enable_symm_mem)) if ( @@ -756,10 +755,13 @@ def _set_envs_and_config(server_args: ServerArgs): def _init_tokenizer_manager( - server_args: ServerArgs, port_args: PortArgs + server_args: ServerArgs, + port_args: PortArgs, + TokenizerManagerClass: Optional[TokenizerManager] = None, ) -> TokenizerManager: # Launch tokenizer process - tokenizer_manager = TokenizerManager(server_args, port_args) + TokenizerManagerClass = TokenizerManagerClass or TokenizerManager + tokenizer_manager = TokenizerManagerClass(server_args, port_args) # Initialize templates template_manager = TemplateManager() diff --git a/python/sglang/srt/layers/quantization/awq.py b/python/sglang/srt/layers/quantization/awq.py index 9e85a4109..c0f14738b 100644 --- a/python/sglang/srt/layers/quantization/awq.py +++ b/python/sglang/srt/layers/quantization/awq.py @@ -32,6 +32,7 @@ from sglang.srt.layers.quantization.marlin_utils import ( from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod from sglang.srt.layers.quantization.utils import get_scalar_types, replace_parameter from sglang.srt.layers.quantization.w8a8_int8 import npu_fused_experts +from sglang.srt.utils.patch_torch import register_fake_if_exists if TYPE_CHECKING: from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig @@ -959,7 +960,7 @@ class AWQMoEAscendMethod(AWQMoEMethod): # Register fake implementations for torch.compile support if _is_cuda: - @torch.library.register_fake("sgl_kernel::awq_dequantize") + @register_fake_if_exists("sgl_kernel::awq_dequantize") def _( qweight, scales, @@ -971,7 +972,7 @@ if _is_cuda: out_shape = qweight.shape[:-1] + (qweight.shape[-1] * 32 // num_bits,) return qweight.new_empty(out_shape, dtype=scales.dtype) - @torch.library.register_fake("sgl_kernel::awq_marlin_repack") + @register_fake_if_exists("sgl_kernel::awq_marlin_repack") def _(b_q_weight, size_k, size_n, num_bits): return b_q_weight.new_empty( (size_k // 16, size_n * (num_bits // 2)), dtype=b_q_weight.dtype diff --git a/python/sglang/srt/layers/quantization/gptq.py b/python/sglang/srt/layers/quantization/gptq.py index c5f7acf25..ceafc6391 100644 --- a/python/sglang/srt/layers/quantization/gptq.py +++ b/python/sglang/srt/layers/quantization/gptq.py @@ -42,16 +42,16 @@ from sglang.srt.layers.quantization.utils import ( replace_parameter, unpack_cols, ) +from sglang.srt.utils import is_cuda +from sglang.srt.utils.patch_torch import register_fake_if_exists if TYPE_CHECKING: from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig from sglang.srt.layers.moe.token_dispatcher import ( - StandardDispatchOutput, CombineInput, + StandardDispatchOutput, ) -from sglang.srt.utils import is_cuda - _is_cuda = is_cuda() if _is_cuda: @@ -1099,21 +1099,21 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): # Register fake implementations for torch.compile support if _is_cuda: - @torch.library.register_fake("sgl_kernel::gptq_gemm") + @register_fake_if_exists("sgl_kernel::gptq_gemm") def _(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit): return a.new_empty((a.shape[0], b_q_weight.shape[-1]), dtype=a.dtype) - @torch.library.register_fake("sgl_kernel::gptq_marlin_repack") + @register_fake_if_exists("sgl_kernel::gptq_marlin_repack") def _(b_q_weight, perm, size_k, size_n, num_bits): return b_q_weight.new_empty( (size_k // 16, size_n * (num_bits // 2)), dtype=b_q_weight.dtype ) - @torch.library.register_fake("sgl_kernel::gptq_shuffle") + @register_fake_if_exists("sgl_kernel::gptq_shuffle") def _(q_weight, q_perm, bit): return - @torch.library.register_fake("sgl_kernel::moe_wna16_marlin_gemm") + @register_fake_if_exists("sgl_kernel::moe_wna16_marlin_gemm") def _( a, c, diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py index d09f17d2e..87922077e 100644 --- a/python/sglang/srt/managers/detokenizer_manager.py +++ b/python/sglang/srt/managers/detokenizer_manager.py @@ -27,7 +27,6 @@ import zmq from sglang.srt.managers.io_struct import ( BatchEmbeddingOutput, BatchMultimodalDecodeReq, - BatchMultimodalOutput, BatchStrOutput, BatchTokenIDOutput, FreezeGCReq, @@ -284,22 +283,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): ) def handle_multimodal_decode_req(self, recv_obj: BatchMultimodalDecodeReq): - outputs = self.tokenizer.detokenize(recv_obj) - return BatchMultimodalOutput( - rids=recv_obj.rids, - http_worker_ipcs=recv_obj.http_worker_ipcs, - finished_reasons=recv_obj.finished_reasons, - outputs=outputs, - prompt_tokens=recv_obj.prompt_tokens, - completion_tokens=recv_obj.completion_tokens, - cached_tokens=recv_obj.cached_tokens, - placeholder_tokens_idx=None, - placeholder_tokens_val=None, - queue_time=recv_obj.queue_time, - forward_entry_time=recv_obj.forward_entry_time, - prefill_launch_delay=recv_obj.prefill_launch_delay, - prefill_launch_latency=recv_obj.prefill_launch_latency, - ) + raise NotImplementedError() def handle_freeze_gc_req(self, recv_req: FreezeGCReq): freeze_gc("Detokenizer Manager") diff --git a/python/sglang/srt/utils/patch_torch.py b/python/sglang/srt/utils/patch_torch.py index 6dc329a9d..9b4e21154 100644 --- a/python/sglang/srt/utils/patch_torch.py +++ b/python/sglang/srt/utils/patch_torch.py @@ -17,7 +17,7 @@ import torch from packaging import version from torch.multiprocessing import reductions -from sglang.srt.utils import is_npu +from sglang.srt.utils.common import is_npu _is_npu = is_npu() @@ -88,3 +88,29 @@ def monkey_patch_torch_compile(): af.auto_functionalized_v2._cacheable = True af.auto_functionalized._cacheable = True + + +def register_fake_if_exists(op_name): + """ + Decorator factory to conditionally register a fake for a custom op if it exists. + Parses op_name (e.g., 'sgl_kernel::gptq_gemm'), checks if the op exists via hasattr + on the namespace attribute of torch.ops. Registers the fake if present; otherwise, + returns the function unchanged. + Args: + op_name (str): Full operator name (e.g., 'sgl_kernel::gptq_gemm'). + Returns: + callable: Decorator for the fake function. + Example: + @register_fake_if_exists('sgl_kernel::gptq_gemm') + def fake_gptq_gemm(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit): + return a.new_empty((a.shape[0], b_q_weight.shape[-1]), dtype=a.dtype) + """ + + def decorator(func): + namespace, bare_op = op_name.split("::") + ops_namespace = getattr(torch.ops, namespace, None) + if ops_namespace and hasattr(ops_namespace, bare_op): + torch.library.register_fake(op_name, func) + return func + + return decorator