Support min load besides random routing key assignment policy in ManualPolicy (#16767)

This commit is contained in:
fzyzcjy
2026-01-12 18:31:18 -08:00
committed by GitHub
parent a83484275d
commit 9d3018f484
8 changed files with 163 additions and 15 deletions
@@ -36,6 +36,7 @@ class RouterArgs:
eviction_interval_secs: int = 60
max_tree_size: int = 2**26
max_idle_secs: int = 4 * 3600
assignment_mode: str = "random"
max_payload_size: int = 512 * 1024 * 1024 # 512MB default for large batches
bucket_adjust_interval_secs: int = 5
dp_aware: bool = False
@@ -310,6 +311,13 @@ class RouterArgs:
default=RouterArgs.max_idle_secs,
help="Maximum idle time in seconds before eviction (for manual policy)",
)
routing_group.add_argument(
f"--{prefix}assignment-mode",
type=str,
default=RouterArgs.assignment_mode,
choices=["random", "min_load"],
help="Mode for assigning new routing keys in manual policy: random (default), min_load (worker with fewest requests)",
)
routing_group.add_argument(
f"--{prefix}max-payload-size",
type=int,