Improve overlap scheduling for better TTFT (#11856)

Co-authored-by: Peng Wang <peng_wang@linux.alibaba.com>
2025-11-12 11:24:04 +08:00
parent 28b8c5792d
commit 8f01a12d43
3 changed files with 23 additions and 3 deletions
@@ -162,6 +162,7 @@ class Envs:

    # Scheduler: others:
    SGLANG_EMPTY_CACHE_INTERVAL = EnvFloat(-1)  # in seconds. Set if you observe high memory accumulation over a long serving period.
+    SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP = EnvBool(False)
    SGLANG_EXPERIMENTAL_CPP_RADIX_TREE = EnvBool(False)

    # Test: pd-disaggregation