Clean up wrapper in flashinfer backend (#2638)
This commit is contained in:
@@ -843,8 +843,8 @@ class ScheduleBatch:
|
||||
# TODO (lianmin): Revisit this. It should be seq_len - 1
|
||||
self.extend_logprob_start_lens.extend([0] * running_bs)
|
||||
|
||||
def check_decode_mem(self):
|
||||
bs = len(self.reqs)
|
||||
def check_decode_mem(self, buf_multiplier=1):
|
||||
bs = len(self.reqs) * buf_multiplier
|
||||
if self.token_to_kv_pool.available_size() >= bs:
|
||||
return True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user