Correctly abort the failed grammar requests & Improve the handling of abort (#6803)

2025-06-01 19:00:07 -07:00
parent 6a47b73024
commit 20fd53b8f6
16 changed files with 199 additions and 142 deletions
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -918,7 +918,7 @@ class ModelRunner:

        if self.req_to_token_pool is None:
            self.req_to_token_pool = ReqToTokenPool(
-                size=max_num_reqs + 1,
+                size=max_num_reqs,
                max_context_len=self.model_config.context_len + 4,
                device=self.device,
                enable_memory_saver=self.server_args.enable_memory_saver,