Support returning customized info (#16144)

This commit is contained in:
Qiaolin Yu
2026-01-02 02:32:18 -05:00
committed by GitHub
parent 6c8587b5db
commit dd93e4451c
8 changed files with 58 additions and 2 deletions

View File

@@ -187,6 +187,10 @@ class SamplingBatchInfo:
def adjusted_from_schedule_batch(self, batch: ScheduleBatch, vocab_size: int):
pass
# placeholder for override
def adjusted_merge_batch(self, other: "SamplingBatchInfo"):
pass
def __len__(self):
return len(self.temperatures)
@@ -372,6 +376,8 @@ class SamplingBatchInfo:
self.need_top_k_sampling |= other.need_top_k_sampling
self.need_min_p_sampling |= other.need_min_p_sampling
self.adjusted_merge_batch(other)
def copy_for_forward(self):
# Accumulate the penalty into a pre-allocated buffer to get rid of the dependency of `penalizer_orchestrator` later
self.update_penalties()