Support token low usage watermark in prefill delayer (#16814)

2026-01-09 23:07:18 +08:00
parent 71e9c31cb8
commit 8eeffbe9aa
5 changed files with 241 additions and 22 deletions
@@ -224,6 +224,7 @@ class Envs:
    SGLANG_SCHEDULER_SKIP_ALL_GATHER = EnvBool(False)
    SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE = EnvBool(False)
    SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES = EnvInt(30)
+    SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK = EnvFloat(None)
    SGLANG_DATA_PARALLEL_BUDGET_INTERVAL = EnvInt(1)

    # Test: pd-disaggregation