Support cache eviction for Manual Policy (#16263)

2026-01-02 08:52:01 +08:00
parent d7a8257ba5
commit 00562ee14a
8 changed files with 246 additions and 65 deletions
@@ -33,8 +33,9 @@ class RouterArgs:
    cache_threshold: float = 0.3
    balance_abs_threshold: int = 64
    balance_rel_threshold: float = 1.5
-    eviction_interval_secs: int = 120
+    eviction_interval_secs: int = 60
    max_tree_size: int = 2**26
+    max_idle_secs: int = 4 * 3600
    max_payload_size: int = 512 * 1024 * 1024  # 512MB default for large batches
    bucket_adjust_interval_secs: int = 5
    dp_aware: bool = False
@@ -297,6 +298,12 @@ class RouterArgs:
            default=RouterArgs.max_tree_size,
            help="Maximum size of the approximation tree for cache-aware routing",
        )
+        routing_group.add_argument(
+            f"--{prefix}max-idle-secs",
+            type=int,
+            default=RouterArgs.max_idle_secs,
+            help="Maximum idle time in seconds before eviction (for manual policy)",
+        )
        routing_group.add_argument(
            f"--{prefix}max-payload-size",
            type=int,