Warmup before profiling prefill latency for dynamic chunk sizing (#17198)
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
@@ -1232,8 +1232,9 @@ class ChunkSizePredictor:
|
||||
|
||||
def fit(self, seq_lens: List[int], latencies: List[float]):
|
||||
"""Fit quadratic coefficients f(l) = al^2 + bl + c from data points."""
|
||||
L = np.array(seq_lens, dtype=np.float64)
|
||||
T = np.array(latencies, dtype=np.float64)
|
||||
# Skip the first data point to reduce fitting bias, as the first run is slower without warmup
|
||||
L = np.array(seq_lens[1:], dtype=np.float64)
|
||||
T = np.array(latencies[1:], dtype=np.float64)
|
||||
|
||||
if len(L) < 8:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user