Add env var to control custom Triton kernel cache and set CSGMV as default backend. (#12176)

This commit is contained in:
Lifu Huang
2025-10-27 17:49:32 -07:00
committed by GitHub
parent 88596739a4
commit ce832d7034
13 changed files with 24 additions and 25 deletions

View File

@@ -331,7 +331,7 @@ class ServerArgs:
max_loaded_loras: Optional[int] = None
max_loras_per_batch: int = 8
lora_eviction_policy: str = "lru"
lora_backend: str = "triton"
lora_backend: str = "csgmv"
max_lora_chunk_size: Optional[int] = 16
# Kernel backend