Fix spec info's filter when reqs are finished right after prefill (#14742)
This commit is contained in:
@@ -1796,6 +1796,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
self,
|
||||
chunked_req_to_exclude: Optional[Union[Req, List[Req]]] = None,
|
||||
keep_indices: Optional[List[int]] = None,
|
||||
# FIXME(lsyin): deprecate this API after spec v1 is deprecated
|
||||
v1_spec_info_filtered: Optional[bool] = False,
|
||||
):
|
||||
# FIXME(lsyin): used here to get the correct seq_lens
|
||||
# The batch has been launched but we need it verified to get correct next batch info
|
||||
@@ -1852,11 +1854,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
self.has_grammar = any(req.grammar for req in self.reqs)
|
||||
|
||||
self.sampling_info.filter_batch(keep_indices, keep_indices_device)
|
||||
# NOTE: spec_info filtered before batch filtering only happens in:
|
||||
# - Spec v1's verify phase
|
||||
# - Only for decode batch (running_batch)
|
||||
has_been_filtered = v1_spec_info_filtered and not self.is_v2_eagle
|
||||
|
||||
if self.spec_info:
|
||||
if chunked_req_to_exclude is not None and len(chunked_req_to_exclude) > 0:
|
||||
has_been_filtered = False
|
||||
else:
|
||||
has_been_filtered = True
|
||||
self.spec_info.filter_batch(
|
||||
new_indices=keep_indices_device,
|
||||
has_been_filtered=has_been_filtered,
|
||||
|
||||
Reference in New Issue
Block a user