Support using different attention backend for draft decoding. (#14843)
This commit is contained in:
@@ -2147,6 +2147,16 @@ class ModelRunner:
|
||||
|
||||
def _get_attention_backend(self, init_new_workspace: bool = False):
|
||||
"""Init attention kernel backend."""
|
||||
draft_attn_backend = self.server_args.speculative_draft_attention_backend
|
||||
if self.is_draft_worker and draft_attn_backend:
|
||||
logger.warning(
|
||||
f"Overriding draft attention backend to {draft_attn_backend}."
|
||||
)
|
||||
return self._get_attention_backend_from_str(
|
||||
draft_attn_backend,
|
||||
init_new_workspace=init_new_workspace,
|
||||
)
|
||||
|
||||
self.prefill_attention_backend_str, self.decode_attention_backend_str = (
|
||||
self.server_args.get_attention_backends()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user