chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[1/2] deepseek deterministic: support deterministic inference for deepseek arch models on a single GPU (#12000)

Browse Source
This commit is contained in:
Minglei Zhu
2025-10-24 15:17:28 -07:00
committed by GitHub
parent 4463e90dd2
commit f4b78d137c
3 changed files with 64 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/models/deepseek_v2.py
View File

@@ -350,7 +350,11 @@ def handle_attention_flashinfer(attn, forward_batch):
def handle_attention_fa3(attn, forward_batch):
return _handle_attention_backend(attn, forward_batch, "fa3")
# when deterministic inference is enabled, use MLA
if get_global_server_args().enable_deterministic_inference:
return _dispatch_mla_subtype(attn, forward_batch)
else:
return _handle_attention_backend(attn, forward_batch, "fa3")
def handle_attention_flashmla(attn, forward_batch):
@@ -394,6 +398,10 @@ def handle_attention_nsa(attn, forward_batch):
def handle_attention_triton(attn, forward_batch):
# when deterministic inference is enabled, use MLA
if get_global_server_args().enable_deterministic_inference:
return _dispatch_mla_subtype(attn, forward_batch)
if (
_is_extend_without_speculative(forward_batch)
and sum(forward_batch.extend_prefix_lens_cpu) == 0