Separate swa and local attention chunk cache eviction (#15820)

This commit is contained in:
Ke Bao
2025-12-26 09:34:22 +08:00
committed by GitHub
parent 2f66b0671b
commit 7b7e357f61
8 changed files with 50 additions and 29 deletions
@@ -160,6 +160,7 @@ class ModelConfig:
self.attention_chunk_size = getattr(
self.hf_text_config, "attention_chunk_size", None
)
self.sliding_window_size = self._get_sliding_window_size()
self.is_generation = is_generation_model(
self.hf_config.architectures, is_embedding
)
@@ -670,6 +671,12 @@ class ModelConfig:
else:
return "fp8" # Default fallback
def _get_sliding_window_size(self) -> Optional[int]:
sliding_window_size = getattr(self.hf_text_config, "sliding_window_size", None)
if sliding_window_size is None:
sliding_window_size = getattr(self.hf_text_config, "sliding_window", None)
return sliding_window_size
def _validate_quantize_and_serve_config(self):
"""Validate quantize_and_serve configuration."""
if not self.quantize_and_serve: