Support cache eviction for Manual Policy (#16263)

sgl-model-gateway/bindings/python/sglang_router/router_args.py

@@ -33,8 +33,9 @@ class RouterArgs:
     cache_threshold: float = 0.3
     balance_abs_threshold: int = 64
     balance_rel_threshold: float = 1.5
-    eviction_interval_secs: int = 120
+    eviction_interval_secs: int = 60
     max_tree_size: int = 2**26
+    max_idle_secs: int = 4 * 3600
     max_payload_size: int = 512 * 1024 * 1024  # 512MB default for large batches
     bucket_adjust_interval_secs: int = 5
     dp_aware: bool = False
@@ -297,6 +298,12 @@ class RouterArgs:
             default=RouterArgs.max_tree_size,
             help="Maximum size of the approximation tree for cache-aware routing",
         )
+        routing_group.add_argument(
+            f"--{prefix}max-idle-secs",
+            type=int,
+            default=RouterArgs.max_idle_secs,
+            help="Maximum idle time in seconds before eviction (for manual policy)",
+        )
         routing_group.add_argument(
             f"--{prefix}max-payload-size",
             type=int,

sgl-model-gateway/bindings/python/src/lib.rs

@@ -312,6 +312,7 @@ struct Router {
     balance_rel_threshold: f32,
     eviction_interval_secs: u64,
     max_tree_size: usize,
+    max_idle_secs: u64,
     max_payload_size: usize,
     dp_aware: bool,
     api_key: Option<String>,
@@ -417,7 +418,10 @@ impl Router {
                     balance_rel_threshold: self.balance_rel_threshold,
                     bucket_adjust_interval_secs: self.bucket_adjust_interval_secs,
                 },
-                PolicyType::Manual => ConfigPolicyConfig::Manual,
+                PolicyType::Manual => ConfigPolicyConfig::Manual {
+                    eviction_interval_secs: self.eviction_interval_secs,
+                    max_idle_secs: self.max_idle_secs,
+                },
                 PolicyType::ConsistentHashing => ConfigPolicyConfig::ConsistentHashing,
                 PolicyType::PrefixHash => ConfigPolicyConfig::PrefixHash {
                     prefix_token_count: 256,
@@ -589,6 +593,7 @@ impl Router {
         balance_rel_threshold = 1.5,
         eviction_interval_secs = 120,
         max_tree_size = 2usize.pow(26),
+        max_idle_secs = 14400,
         max_payload_size = 512 * 1024 * 1024,
         dp_aware = false,
         api_key = None,
@@ -671,6 +676,7 @@ impl Router {
         balance_rel_threshold: f32,
         eviction_interval_secs: u64,
         max_tree_size: usize,
+        max_idle_secs: u64,
         max_payload_size: usize,
         dp_aware: bool,
         api_key: Option<String>,
@@ -766,6 +772,7 @@ impl Router {
             balance_rel_threshold,
             eviction_interval_secs,
             max_tree_size,
+            max_idle_secs,
             max_payload_size,
             dp_aware,
             api_key,