[Spec Decoding] Support MTP for dsv3.2 (#11652)
Co-authored-by: Paiiiiiiiiiiiiii <zengpai@baidu.com>
This commit is contained in:
co-authored by
Paiiiiiiiiiiiiii
parent
d658f0497e
commit
efa473348b
@@ -48,6 +48,7 @@ class DraftBackendFactory:
|
||||
"flashmla": self._create_flashmla_decode_backend,
|
||||
"trtllm_mha": self._create_trtllm_mha_decode_backend,
|
||||
"trtllm_mla": self._create_trtllm_mla_decode_backend,
|
||||
"nsa": self._create_nsa_decode_backend,
|
||||
}
|
||||
|
||||
return self._create_backend(
|
||||
@@ -70,6 +71,7 @@ class DraftBackendFactory:
|
||||
"flashmla": self._create_flashmla_prefill_backend,
|
||||
"trtllm_mha": self._create_trtllm_mha_prefill_backend,
|
||||
"trtllm_mla": self._create_trtllm_mla_prefill_backend,
|
||||
"nsa": self._create_nsa_prefill_backend,
|
||||
}
|
||||
backend_name = (
|
||||
"decode_attention_backend"
|
||||
@@ -82,6 +84,20 @@ class DraftBackendFactory:
|
||||
"EAGLE is not supported in attention backend {backend_type}",
|
||||
)
|
||||
|
||||
def _create_nsa_decode_backend(self):
|
||||
from sglang.srt.layers.attention.nsa_backend import (
|
||||
NativeSparseAttnMultiStepBackend,
|
||||
)
|
||||
|
||||
return NativeSparseAttnMultiStepBackend(
|
||||
self.draft_model_runner, self.topk, self.speculative_num_steps
|
||||
)
|
||||
|
||||
def _create_nsa_prefill_backend(self):
|
||||
from sglang.srt.layers.attention.nsa_backend import NativeSparseAttnBackend
|
||||
|
||||
return NativeSparseAttnBackend(self.draft_model_runner, skip_prefill=False)
|
||||
|
||||
def _create_flashinfer_decode_backend(self):
|
||||
if not get_global_server_args().use_mla_backend:
|
||||
from sglang.srt.layers.attention.flashinfer_backend import (
|
||||
|
||||
Reference in New Issue
Block a user