From 2e3946d889e7b9368db438d8eaa7db332947b205 Mon Sep 17 00:00:00 2001
From: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com>
Date: Tue, 9 Dec 2025 05:33:34 +0800
Subject: [PATCH] Fix cache-aware router should pick min load instead of min
 tenant size (#14650)

---
 sgl-model-gateway/src/policies/cache_aware.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sgl-model-gateway/src/policies/cache_aware.rs b/sgl-model-gateway/src/policies/cache_aware.rs
index 0b5400148..07dfe3f68 100644
--- a/sgl-model-gateway/src/policies/cache_aware.rs
+++ b/sgl-model-gateway/src/policies/cache_aware.rs
@@ -311,7 +311,10 @@ impl LoadBalancingPolicy for CacheAwarePolicy {
                 matched_worker.to_string()
             } else {
                 RouterMetrics::record_cache_miss();
-                tree.get_smallest_tenant()
+                let min_load_idx = *healthy_indices
+                    .iter()
+                    .min_by_key(|&&idx| workers[idx].load())?;
+                workers[min_load_idx].url().to_string()
             };
 
             // Find the index of the selected worker