diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md index e45a36d6e..344f0746e 100644 --- a/docs/references/environment_variables.md +++ b/docs/references/environment_variables.md @@ -27,7 +27,7 @@ SGLang supports various environment variables that can be used to configure its | `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN` | Allows the scheduler to overwrite longer context length requests (often set to `1` in Docker builds) | `0` | | `SGLANG_IS_FLASHINFER_AVAILABLE` | Control FlashInfer availability check | `true` | | `SGLANG_SKIP_P2P_CHECK` | Skip P2P (peer-to-peer) access check | `false` | -| `SGL_CHUNKED_PREFIX_CACHE_THRESHOLD` | Sets the threshold for enabling chunked prefix caching | `8192` | +| `SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD` | Sets the threshold for enabling chunked prefix caching | `8192` | | `SGLANG_FUSED_MLA_ENABLE_ROPE_FUSION` | Enable RoPE fusion in Fused Multi-Layer Attention | `1` | | `SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP` | Disable overlap schedule for consecutive prefill batches | `false` | | `SGLANG_SCHEDULER_MAX_RECV_PER_POLL` | Set the maximum number of requests per poll, with a negative value indicating no limit | `-1` | @@ -45,7 +45,7 @@ SGLang supports various environment variables that can be used to configure its | `SGLANG_ENABLE_JIT_DEEPGEMM` | Enable Just-In-Time compilation of DeepGEMM kernels (enabled by default on NVIDIA Hopper (SM90) and Blackwell (SM100) GPUs when the DeepGEMM package is installed; set to `"0"` to disable) | `"true"` | | `SGLANG_JIT_DEEPGEMM_PRECOMPILE` | Enable precompilation of DeepGEMM kernels | `"true"` | | `SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS` | Number of workers for parallel DeepGEMM kernel compilation | `4` | -| `SGL_IN_DEEPGEMM_PRECOMPILE_STAGE` | Indicator flag used during the DeepGEMM precompile script | `"false"` | +| `SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE` | Indicator flag used during the DeepGEMM precompile script | `"false"` | | `SGLANG_DG_CACHE_DIR` | Directory for caching compiled DeepGEMM kernels | `~/.cache/deep_gemm` | | `SGL_DG_USE_NVRTC` | Use NVRTC (instead of Triton) for JIT compilation (Experimental) | `"0"` | | `SGL_USE_DEEPGEMM_BMM` | Use DeepGEMM for Batched Matrix Multiplication (BMM) operations | `"false"` | @@ -89,7 +89,7 @@ SGLang supports various environment variables that can be used to configure its | Environment Variable | Description | Default Value | | --- | --- | --- | | `SGLANG_BLOCK_NONZERO_RANK_CHILDREN` | Control blocking of non-zero rank children processes | `1` | -| `SGL_IS_FIRST_RANK_ON_NODE` | Indicates if the current process is the first rank on its node | `"true"` | +| `SGLANG_IS_FIRST_RANK_ON_NODE` | Indicates if the current process is the first rank on its node | `"true"` | | `SGLANG_PP_LAYER_PARTITION` | Pipeline parallel layer partition specification | Not set | | `SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS` | Set one visible device per process for distributed computing | `false` | diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml index 1c5b58704..11bbcd30a 100644 --- a/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml +++ b/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml @@ -226,7 +226,7 @@ spec: value: "8" - name: SGLANG_ENABLE_JIT_DEEPGEMM value: "1" - - name: SGL_CHUNKED_PREFIX_CACHE_THRESHOLD + - name: SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD value: "0" - name: NCCL_IB_QPS_PER_CONNECTION value: "8" diff --git a/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md b/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md index b35089683..368aee34b 100644 --- a/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md +++ b/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md @@ -259,7 +259,7 @@ spec: value: "8" - name: SGLANG_ENABLE_JIT_DEEPGEMM value: "1" - - name: SGL_CHUNKED_PREFIX_CACHE_THRESHOLD + - name: SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD value: "0" - name: NCCL_IB_QPS_PER_CONNECTION value: "8" diff --git a/python/sglang/compile_deep_gemm.py b/python/sglang/compile_deep_gemm.py index 2386a46c0..aa6f0703c 100644 --- a/python/sglang/compile_deep_gemm.py +++ b/python/sglang/compile_deep_gemm.py @@ -33,7 +33,7 @@ envs.SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE.set(True) # Force enable deep gemm envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(True) # Force enable mha chunked kv for DeepSeek V3 to avoid missing kv_b_proj DeepGEMM case -os.environ["SGL_CHUNKED_PREFIX_CACHE_THRESHOLD"] = "0" +os.environ["SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD"] = "0" @dataclasses.dataclass diff --git a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py index b13dbd23b..fee23cc49 100644 --- a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py +++ b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py @@ -25,8 +25,8 @@ if ENABLE_JIT_DEEPGEMM: _BUILTIN_M_LIST = list(range(1, 1024 * 16 + 1)) _ENABLE_JIT_DEEPGEMM_PRECOMPILE = envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.get() _DO_COMPILE_ALL = True -_IS_FIRST_RANK_ON_NODE = get_bool_env_var("SGL_IS_FIRST_RANK_ON_NODE", "true") -_IN_PRECOMPILE_STAGE = get_bool_env_var("SGL_IN_DEEPGEMM_PRECOMPILE_STAGE", "false") +_IS_FIRST_RANK_ON_NODE = get_bool_env_var("SGLANG_IS_FIRST_RANK_ON_NODE", "true") +_IN_PRECOMPILE_STAGE = get_bool_env_var("SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE", "false") # Force redirect deep_gemm cache_dir os.environ["DG_JIT_CACHE_DIR"] = os.getenv( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 6400a7c87..c9895f63c 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -1464,7 +1464,7 @@ class DeepseekV2AttentionMLA(nn.Module): # TODO: Design a finer way to determine the threshold self.chunked_prefix_cache_threshold = get_int_env_var( - "SGL_CHUNKED_PREFIX_CACHE_THRESHOLD", 8192 + "SGLANG_CHUNKED_PREFIX_CACHE_THRESHOLD", 8192 ) # If we have self.fused_qkv_a_proj_with_mqa and we're running on CPU, we will choose the torch.ops.sgl_kernel.qkv_proj_with_rope_fused_weight kernel