feat: support flashinfer kernel autotune (#12306)
Co-authored-by: Qiaolin Yu <liin1211@outlook.com> Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com>
This commit is contained in:
co-authored by
Qiaolin Yu
Kangyan-Zhou
parent
ce6b7dfce7
commit
848ee57067
@@ -378,6 +378,7 @@ class ServerArgs:
|
||||
mm_attention_backend: Optional[str] = None
|
||||
nsa_prefill_backend: str = "flashmla_sparse"
|
||||
nsa_decode_backend: str = "fa3"
|
||||
enable_flashinfer_autotune: bool = False
|
||||
|
||||
# Speculative decoding
|
||||
speculative_algorithm: Optional[str] = None
|
||||
@@ -2882,6 +2883,12 @@ class ServerArgs:
|
||||
type=str,
|
||||
choices=NSA_CHOICES,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-flashinfer-autotune",
|
||||
default=ServerArgs.enable_flashinfer_autotune,
|
||||
action="store_true",
|
||||
help="Enable FlashInfer autotuning for optimal kernel selection.",
|
||||
)
|
||||
|
||||
# Speculative decoding
|
||||
parser.add_argument(
|
||||
|
||||
Reference in New Issue
Block a user