From 8bfce9b08dbe645f7b36980b18544a71181195c1 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Sat, 22 Nov 2025 00:05:20 -0800 Subject: [PATCH] [Tiny] Renaming environ for NVFP4 dispatch (#13756) --- docs/references/environment_variables.md | 2 +- python/sglang/srt/environ.py | 1 + .../sglang/srt/layers/quantization/modelopt_quant.py | 10 ++++------ test/srt/test_deepseek_v3_cutedsl_4gpu.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md index fda66255c..1e618c9d1 100644 --- a/docs/references/environment_variables.md +++ b/docs/references/environment_variables.md @@ -49,7 +49,7 @@ SGLang supports various environment variables that can be used to configure its | Environment Variable | Description | Default Value | | --- | --- | --- | | `SGLANG_DEEPEP_BF16_DISPATCH` | Use Bfloat16 for dispatch | `"false"` | -| `SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH` | Use nvfp4 for dispatch | `"false"` | +| `SGLANG_MOE_NVFP4_DISPATCH` | Use nvfp4 for moe dispatch | `"false"` | ## Memory Management diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 63ce4b0a8..9c1321454 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -215,6 +215,7 @@ class Envs: SGLANG_CPU_QUANTIZATION = EnvBool(False) SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False) SGLANG_FORCE_FP8_MARLIN = EnvBool(False) + SGLANG_MOE_NVFP4_DISPATCH = EnvBool(False) # Flashinfer SGLANG_IS_FLASHINFER_AVAILABLE = EnvBool(True) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index e627a3151..fdba02037 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -139,9 +139,7 @@ CUTEDSL_MOE_SCALAR_INPUT_SCALE = get_bool_env_var( ) # TODO make it true by default when the DeepEP PR is merged -CUTEDSL_MOE_NVFP4_DISPATCH = get_bool_env_var( - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH", "false" -) +MOE_NVFP4_DISPATCH = envs.SGLANG_MOE_NVFP4_DISPATCH.get() FLASHINFER_FP4_GEMM_BACKEND = envs.SGLANG_FLASHINFER_FP4_GEMM_BACKEND.get() # Supported activation schemes for the current configuration ACTIVATION_SCHEMES = ["static"] @@ -1492,7 +1490,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): w13_input_scale = _slice_scale(w13_input_scale) w2_input_scale = _slice_scale(w2_input_scale) - if CUTEDSL_MOE_NVFP4_DISPATCH: + if MOE_NVFP4_DISPATCH: assert torch.all(w13_input_scale == w13_input_scale[0]) w13_input_scale = w13_input_scale[0] else: @@ -1518,7 +1516,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): layer.dispatcher.set_quant_config( { "input_global_scale": ( - layer.w13_input_scale_quant if CUTEDSL_MOE_NVFP4_DISPATCH else None + layer.w13_input_scale_quant if MOE_NVFP4_DISPATCH else None ) } ) @@ -1740,7 +1738,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): out = flashinfer_cutedsl_moe_masked( hidden_states=x, input_global_scale=( - None if CUTEDSL_MOE_NVFP4_DISPATCH else layer.w13_input_scale_quant + None if MOE_NVFP4_DISPATCH else layer.w13_input_scale_quant ), w1=layer.w13_weight, w1_blockscale=layer.w13_blockscale_swizzled, diff --git a/test/srt/test_deepseek_v3_cutedsl_4gpu.py b/test/srt/test_deepseek_v3_cutedsl_4gpu.py index 63b5fc20b..7d8f39e01 100644 --- a/test/srt/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/srt/test_deepseek_v3_cutedsl_4gpu.py @@ -60,7 +60,7 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase): **os.environ, "SGLANG_DEEPEP_BF16_DISPATCH": "1", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "0", + "SGLANG_MOE_NVFP4_DISPATCH": "0", }, ) @@ -136,7 +136,7 @@ class TestDummyWithSBO(CustomTestCase): **os.environ, "SGLANG_DEEPEP_BF16_DISPATCH": "1", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", - "SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH": "0", + "SGLANG_MOE_NVFP4_DISPATCH": "0", }, )