diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md index f2923765e..17f115f9a 100644 --- a/docs/references/environment_variables.md +++ b/docs/references/environment_variables.md @@ -122,6 +122,7 @@ SGLang supports various environment variables that can be used to configure its | `SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE` | Quantize moe of nextn layer from BF16 to FP8 when launching DeepSeek NVFP4 checkpoint | `false` | | `SGLANG_ENABLE_FLASHINFER_FP8_GEMM` (deprecated) | Use flashinfer kernels when running blockwise fp8 GEMM on Blackwell GPUs. **DEPRECATED**: Please use `--fp8-gemm-backend=flashinfer_trtllm` (SM100/SM103) or `--fp8-gemm-backend=flashinfer_cutlass` (SM120/SM121 and newer) instead. | `false` | | `SGLANG_SUPPORT_CUTLASS_BLOCK_FP8` (deprecated) | Use Cutlass kernels when running blockwise fp8 GEMM on Hopper or Blackwell GPUs. **DEPRECATED**: Please use `--fp8-gemm-backend=cutlass` instead. | `false` | +| `SGLANG_QUANT_ALLOW_DOWNCASTING` | Allow weights downcasting | `false` | ## Distributed Computing diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index cd10b1281..22160f6a0 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -326,6 +326,7 @@ class Envs: SGLANG_NVFP4_CKPT_FP8_GEMM_IN_ATTN = EnvBool(False) SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2 = EnvBool(False) SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE = EnvBool(False) + SGLANG_QUANT_ALLOW_DOWNCASTING = EnvBool(False) # Flashinfer SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True) diff --git a/python/sglang/srt/layers/parameter.py b/python/sglang/srt/layers/parameter.py index 565f5b9fd..ff0deb03e 100644 --- a/python/sglang/srt/layers/parameter.py +++ b/python/sglang/srt/layers/parameter.py @@ -7,6 +7,7 @@ from typing import Callable, Optional, Union import torch from torch.nn import Parameter +from sglang.srt.environ import envs from sglang.srt.layers.utils import pad_or_narrow_weight from sglang.srt.utils import is_cpu @@ -65,7 +66,7 @@ def copy_with_check(target: torch.Tensor, loaded_weight: torch.Tensor): raise ValueError( f"Unsupported copy between dtypes: {target.dtype=}, {loaded_weight.dtype=}" ) - if target_rank < loaded_rank: + if target_rank < loaded_rank and not envs.SGLANG_QUANT_ALLOW_DOWNCASTING.get(): raise ValueError( f"Downcasting not allowed: {target.dtype=}, {loaded_weight.dtype=}" )