Support using different attention backend for draft decoding. (#14843)

This commit is contained in:
Charles Chen
2025-12-18 16:49:11 -08:00
committed by GitHub
parent 216067c0cb
commit 9e0ef04e5b
3 changed files with 23 additions and 1 deletions

View File

@@ -2147,6 +2147,16 @@ class ModelRunner:
def _get_attention_backend(self, init_new_workspace: bool = False):
"""Init attention kernel backend."""
draft_attn_backend = self.server_args.speculative_draft_attention_backend
if self.is_draft_worker and draft_attn_backend:
logger.warning(
f"Overriding draft attention backend to {draft_attn_backend}."
)
return self._get_attention_backend_from_str(
draft_attn_backend,
init_new_workspace=init_new_workspace,
)
self.prefill_attention_backend_str, self.decode_attention_backend_str = (
self.server_args.get_attention_backends()
)