Sync code and test CI; rename some env vars (#11686)

This commit is contained in:
Lianmin Zheng
2025-10-15 18:37:03 -07:00
committed by GitHub
parent 729b7edf72
commit cd7e1bd591
17 changed files with 66 additions and 34 deletions

View File

@@ -2484,13 +2484,7 @@ class ServerArgs:
default=ServerArgs.mamba_full_memory_ratio,
help="The ratio of mamba state memory to full kv cache memory.",
)
# Args for multi-item-scoring
parser.add_argument(
"--multi-item-scoring-delimiter",
type=int,
default=ServerArgs.multi_item_scoring_delimiter,
help="Delimiter token ID for multi-item scoring. Used to combine Query and Items into a single sequence: Query<delimiter>Item1<delimiter>Item2<delimiter>... This enables efficient batch processing of multiple items against a single query.",
)
# Hierarchical cache
parser.add_argument(
"--enable-hierarchical-cache",
@@ -2636,6 +2630,14 @@ class ServerArgs:
help="Mode of offloading.",
)
# Args for multi-item-scoring
parser.add_argument(
"--multi-item-scoring-delimiter",
type=int,
default=ServerArgs.multi_item_scoring_delimiter,
help="Delimiter token ID for multi-item scoring. Used to combine Query and Items into a single sequence: Query<delimiter>Item1<delimiter>Item2<delimiter>... This enables efficient batch processing of multiple items against a single query.",
)
# Optimization/debug options
parser.add_argument(
"--disable-radix-cache",