[model-gateway] Add PrefixHash load balancing policy for KV cache-aware routing (#15935)

2025-12-27 05:58:05 -05:00
parent ca740a41f3
commit 3645ed0f73
14 changed files with 532 additions and 60 deletions
@@ -14,6 +14,7 @@ pub enum PolicyType {
    Bucket,
    Manual,
    ConsistentHashing,
+    PrefixHash,
 }

 #[pyclass(eq)]
@@ -418,6 +419,10 @@ impl Router {
                },
                PolicyType::Manual => ConfigPolicyConfig::Manual,
                PolicyType::ConsistentHashing => ConfigPolicyConfig::ConsistentHashing,
+                PolicyType::PrefixHash => ConfigPolicyConfig::PrefixHash {
+                    prefix_token_count: 256,
+                    load_factor: 1.25,
+                },
            }
        };