Add env var to control custom Triton kernel cache and set CSGMV as default backend. (#12176)
This commit is contained in:
@@ -331,7 +331,7 @@ class ServerArgs:
|
||||
max_loaded_loras: Optional[int] = None
|
||||
max_loras_per_batch: int = 8
|
||||
lora_eviction_policy: str = "lru"
|
||||
lora_backend: str = "triton"
|
||||
lora_backend: str = "csgmv"
|
||||
max_lora_chunk_size: Optional[int] = 16
|
||||
|
||||
# Kernel backend
|
||||
|
||||
Reference in New Issue
Block a user