Fix decode OOM caused by retraction (#14939)

2025-12-13 13:59:17 +09:00
parent 8698867479
commit 01e3b3f3a3
4 changed files with 10 additions and 4 deletions
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -1666,7 +1666,9 @@ class ModelRunner:
        self.max_total_num_tokens = self.profile_max_num_token(total_gpu_memory)

        if (small_kv_size := envs.SGLANG_CI_SMALL_KV_SIZE.get()) > 0:
-            # Use a small KV cache pool size for local tests
+            logger.info(
+                f"Use a small KV cache pool size ({small_kv_size}) for local tests"
+            )
            self.max_total_num_tokens = small_kv_size

        if max_num_reqs is None: