[model-gateway] Add PrefixHash load balancing policy for KV cache-aware routing (#15935)

2025-12-27 05:58:05 -05:00
parent ca740a41f3
commit 3645ed0f73
14 changed files with 532 additions and 60 deletions
@@ -27,6 +27,7 @@ def policy_from_str(policy_str: Optional[str]) -> PolicyType:
        "bucket": PolicyType.Bucket,
        "manual": PolicyType.Manual,
        "consistent_hashing": PolicyType.ConsistentHashing,
+        "prefix_hash": PolicyType.PrefixHash,
    }
    return policy_map[policy_str]