[Revision] Replace enable_flashinfer_mla argument with attention_backend (#5052)
This commit is contained in:
@@ -179,7 +179,7 @@ class ServerArgs:
|
||||
tool_call_parser: Optional[str] = None
|
||||
enable_hierarchical_cache: bool = False
|
||||
hicache_ratio: float = 2.0
|
||||
enable_flashinfer_mla: bool = False
|
||||
enable_flashinfer_mla: bool = False # TODO: remove this argument
|
||||
enable_flashmla: bool = False
|
||||
flashinfer_mla_disable_ragged: bool = False
|
||||
warmups: Optional[str] = None
|
||||
@@ -267,15 +267,11 @@ class ServerArgs:
|
||||
else:
|
||||
self.cuda_graph_max_bs = 160
|
||||
|
||||
# Choose kernel backends
|
||||
# Set kernel backends for hpu device
|
||||
if self.device == "hpu":
|
||||
self.attention_backend = "torch_native"
|
||||
self.sampling_backend = "pytorch"
|
||||
|
||||
if self.attention_backend is None:
|
||||
self.attention_backend = (
|
||||
"flashinfer" if is_flashinfer_available() else "triton"
|
||||
)
|
||||
if self.sampling_backend is None:
|
||||
self.sampling_backend = (
|
||||
"flashinfer" if is_flashinfer_available() else "pytorch"
|
||||
@@ -842,7 +838,7 @@ class ServerArgs:
|
||||
parser.add_argument(
|
||||
"--enable-flashinfer-mla",
|
||||
action="store_true",
|
||||
help="Enable FlashInfer MLA optimization",
|
||||
help="Enable FlashInfer MLA optimization. This argument will be deprecated soon! Please use '--attention-backend flashinfer' instead for switching on flashfiner mla!",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-flashmla",
|
||||
|
||||
Reference in New Issue
Block a user