From d4e68ead1dc841cf5799c12de5cdce6f4e8eb2e7 Mon Sep 17 00:00:00 2001 From: StonyPort <157573149+zhooooong@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:59:39 +0800 Subject: [PATCH] [quant] Ignore FP8 quantization layers (#20340) Co-authored-by: qiuxuan.lzw Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- docs/references/environment_variables.md | 3 ++- python/sglang/srt/environ.py | 1 + python/sglang/srt/layers/quantization/fp8.py | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md index a07a4fcc2..98c50c03d 100644 --- a/docs/references/environment_variables.md +++ b/docs/references/environment_variables.md @@ -126,7 +126,8 @@ SGLang supports various environment variables that can be used to configure its | `SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE` | Quantize moe of nextn layer from BF16 to FP8 when launching DeepSeek NVFP4 checkpoint | `false` | | `SGLANG_ENABLE_FLASHINFER_FP8_GEMM` (deprecated) | Use flashinfer kernels when running blockwise fp8 GEMM on Blackwell GPUs. **DEPRECATED**: Please use `--fp8-gemm-backend=flashinfer_trtllm` (SM100/SM103) or `--fp8-gemm-backend=flashinfer_cutlass` (SM120/SM121 and newer) instead. | `false` | | `SGLANG_SUPPORT_CUTLASS_BLOCK_FP8` (deprecated) | Use Cutlass kernels when running blockwise fp8 GEMM on Hopper or Blackwell GPUs. **DEPRECATED**: Please use `--fp8-gemm-backend=cutlass` instead. | `false` | -| `SGLANG_QUANT_ALLOW_DOWNCASTING` | Allow weights downcasting | `false` | +| `SGLANG_QUANT_ALLOW_DOWNCASTING` | Allow weight dtype downcasting during loading (e.g., fp32 → fp16). By default, SGLang rejects this kind of downcasting when using quantization. | `false` | +| `SGLANG_FP8_IGNORED_LAYERS` | A comma-separated list of layer names to ignore during FP8 quantization. For example: `model.layers.0,model.layers.1.,qkv_proj`. | `""` | ## Distributed Computing diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 48a9ea122..e9c6481ba 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -333,6 +333,7 @@ class Envs: SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 = EnvBool(False) SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE = EnvBool(False) SGLANG_QUANT_ALLOW_DOWNCASTING = EnvBool(False) + SGLANG_FP8_IGNORED_LAYERS = EnvStr("") # Flashinfer SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True) diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 57236c072..1e12baff1 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -131,6 +131,14 @@ class Fp8Config(QuantizationConfig): raise ValueError(f"Unsupported activation scheme {activation_scheme}") self.activation_scheme = activation_scheme self.ignored_layers = ignored_layers or [] + if ignored_layers_str := envs.SGLANG_FP8_IGNORED_LAYERS.get(): + self.ignored_layers.extend( + [ + layer.strip() + for layer in ignored_layers_str.split(",") + if layer.strip() + ] + ) self.packed_modules_mapping = packed_modules_mapping or {} self.use_mxfp8 = use_mxfp8 if weight_block_size is not None: