[Feature] Support mamba radix cache v0 (#11214)
Co-authored-by: hanming-lu <hanming@x.ai> Co-authored-by: hzh0425 <hzh0425@apache.org> Co-authored-by: thalahors <ericalcaide1@gmail.com>
This commit is contained in:
@@ -362,6 +362,7 @@ class ServerArgs:
|
||||
# Mamba cache
|
||||
max_mamba_cache_size: Optional[int] = None
|
||||
mamba_ssm_dtype: str = "float32"
|
||||
mamba_full_memory_ratio: float = 0.2
|
||||
|
||||
# Hierarchical cache
|
||||
enable_hierarchical_cache: bool = False
|
||||
@@ -2433,6 +2434,12 @@ class ServerArgs:
|
||||
choices=["float32", "bfloat16"],
|
||||
help="The data type of the SSM states in mamba cache.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mamba-full-memory-ratio",
|
||||
type=float,
|
||||
default=ServerArgs.mamba_full_memory_ratio,
|
||||
help="The ratio of mamba state memory to full kv cache memory.",
|
||||
)
|
||||
# Args for multi-item-scoring
|
||||
parser.add_argument(
|
||||
"--multi-item-scoring-delimiter",
|
||||
|
||||
Reference in New Issue
Block a user