[Refactor] Set fp4-gemm-backend=auto on SM100 and rename fp4-gemm-backend with flashinfer_ prefix (#17309)
This commit is contained in:
@@ -16,21 +16,35 @@ class Fp4GemmRunnerBackend(Enum):
|
||||
"""Enum for FP4 GEMM runner backend selection."""
|
||||
|
||||
AUTO = "auto"
|
||||
CUDNN = "cudnn"
|
||||
CUTLASS = "cutlass"
|
||||
TRTLLM = "trtllm"
|
||||
FLASHINFER_CUDNN = "flashinfer_cudnn"
|
||||
FLASHINFER_CUTLASS = "flashinfer_cutlass"
|
||||
FLASHINFER_TRTLLM = "flashinfer_trtllm"
|
||||
|
||||
def is_auto(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.AUTO
|
||||
|
||||
def is_cudnn(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.CUDNN
|
||||
def is_flashinfer_cudnn(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.FLASHINFER_CUDNN
|
||||
|
||||
def is_cutlass(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.CUTLASS
|
||||
def is_flashinfer_cutlass(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.FLASHINFER_CUTLASS
|
||||
|
||||
def is_trtllm(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.TRTLLM
|
||||
def is_flashinfer_trtllm(self) -> bool:
|
||||
return self == Fp4GemmRunnerBackend.FLASHINFER_TRTLLM
|
||||
|
||||
def get_flashinfer_backend(self) -> str:
|
||||
"""Get the backend string to pass to FlashInfer's mm_fp4 API.
|
||||
|
||||
This remaps SGLang's user-facing backend names to FlashInfer's API names.
|
||||
Examples:
|
||||
'flashinfer_trtllm' -> 'trtllm'
|
||||
'flashinfer_cutlass' -> 'cutlass'
|
||||
'flashinfer_cudnn' -> 'cudnn'
|
||||
"""
|
||||
if self.value.startswith("flashinfer_"):
|
||||
return self.value.removeprefix("flashinfer_")
|
||||
else:
|
||||
return self.value
|
||||
|
||||
|
||||
FP4_GEMM_RUNNER_BACKEND: Fp4GemmRunnerBackend | None = None
|
||||
|
||||
Reference in New Issue
Block a user