Enable RadixCache for Mamba2 models (#13584)

This commit is contained in:
roikoren755
2025-12-05 12:23:58 +02:00
committed by GitHub
parent 35ba6fe19e
commit 2ce121a1c3
6 changed files with 29 additions and 19 deletions

View File

@@ -1278,6 +1278,18 @@ class ServerArgs:
logger.info(
"Use flashinfer_trtllm as MoE runner backend on sm100 for Qwen3NextForCausalLM"
)
elif model_arch in [
"NemotronHForCausalLM",
"FalconH1ForCausalLM",
"JetNemotronForCausalLM",
"JetVLMForConditionalGeneration",
]:
if not self.disable_radix_cache:
logger.warning(
"Disabling overlap schedule since MambaRadixCache is not compatible with "
"overlap schedule currently, try to use --disable-radix-cache if overlap schedule is necessary"
)
self.disable_overlap_schedule = True
def _handle_sampling_backend(self):
if self.sampling_backend is None: