[Bugfix] Add rids to the batch filtering for two batch overlap (#19418)
This commit is contained in:
@@ -653,6 +653,7 @@ class TboForwardBatchPreparer:
|
||||
"extend_seq_lens_cpu",
|
||||
"extend_logprob_start_lens_cpu",
|
||||
"lora_ids",
|
||||
"rids",
|
||||
]:
|
||||
old_value = getattr(batch, key)
|
||||
if old_value is None:
|
||||
|
||||
@@ -1048,7 +1048,7 @@ class ServerArgs:
|
||||
reserved_mem += 6 * 1024
|
||||
elif self.speculative_algorithm != "NGRAM":
|
||||
# eagle draft models and cuda graphs
|
||||
reserved_mem += 2 * 1024
|
||||
reserved_mem += 4 * 1024
|
||||
|
||||
self.mem_fraction_static = (
|
||||
round((gpu_mem - reserved_mem) / gpu_mem, 3)
|
||||
|
||||
Reference in New Issue
Block a user