Warmup before profiling prefill latency for dynamic chunk sizing (#17198)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Xiaowei Wang
2026-02-03 17:45:23 +08:00
committed by GitHub
parent 8e933e1914
commit cc69ac9e7a

View File

@@ -1232,8 +1232,9 @@ class ChunkSizePredictor:
def fit(self, seq_lens: List[int], latencies: List[float]):
"""Fit quadratic coefficients f(l) = al^2 + bl + c from data points."""
L = np.array(seq_lens, dtype=np.float64)
T = np.array(latencies, dtype=np.float64)
# Skip the first data point to reduce fitting bias, as the first run is slower without warmup
L = np.array(seq_lens[1:], dtype=np.float64)
T = np.array(latencies[1:], dtype=np.float64)
if len(L) < 8:
raise ValueError(