chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[1/2] deepseek deterministic: support deterministic inference for deepseek arch models on a single GPU (#12000)

Browse Source
This commit is contained in:
Minglei Zhu
2025-10-24 15:17:28 -07:00
committed by GitHub
parent 4463e90dd2
commit f4b78d137c
3 changed files with 64 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
python/sglang/srt/server_args.py
View File

@@ -1532,13 +1532,30 @@ class ServerArgs:
logger.warning(
"Sampling backend is set to pytorch for deterministic inference."
)
is_deepseek_model = False
if parse_connector_type(self.model_path) != ConnectorType.INSTANCE:
try:
hf_config = self.get_hf_config()
model_arch = hf_config.architectures[0]
is_deepseek_model = model_arch in [
"DeepseekV2ForCausalLM",
"DeepseekV3ForCausalLM",
"DeepseekV32ForCausalLM",
]
except Exception:
pass
# Check attention backend
if self.attention_backend is None:
# User didn't specify attention backend, fallback based on GPU architecture
if is_sm100_supported() or is_sm120_supported():
# Blackwell and newer architectures
self.attention_backend = "flashinfer"
if is_deepseek_model:
# fallback to triton for DeepSeek models because flashinfer doesn't support deterministic inference for DeepSeek models yet
self.attention_backend = "triton"
else:
# fallback to flashinfer on Blackwell for non-DeepSeek models
self.attention_backend = "flashinfer"
else:
# Hopper (SM90) and older architectures
self.attention_backend = "fa3"
@@ -1553,8 +1570,13 @@ class ServerArgs:
f"but you explicitly specified '{self.attention_backend}'."
)
# Currently, only FA3 and Triton supports radix cache. Support for other backends is in progress
if self.attention_backend not in ["fa3", "triton"]:
if is_deepseek_model:
raise ValueError(
f"Currently only fa3 and triton attention backends are supported for deterministic inference with DeepSeek models. But you're using {self.attention_backend}."
)
# Currently, only FA3 and Triton supports radix cache. Support for other backends is in progress
self.disable_radix_cache = True
logger.warning(
f"Currently radix cache is not compatible with {self.attention_backend} attention backend for deterministic inference. It will be supported in the future."