Support returning customized info (#16144)
This commit is contained in:
@@ -187,6 +187,10 @@ class SamplingBatchInfo:
|
||||
def adjusted_from_schedule_batch(self, batch: ScheduleBatch, vocab_size: int):
|
||||
pass
|
||||
|
||||
# placeholder for override
|
||||
def adjusted_merge_batch(self, other: "SamplingBatchInfo"):
|
||||
pass
|
||||
|
||||
def __len__(self):
|
||||
return len(self.temperatures)
|
||||
|
||||
@@ -372,6 +376,8 @@ class SamplingBatchInfo:
|
||||
self.need_top_k_sampling |= other.need_top_k_sampling
|
||||
self.need_min_p_sampling |= other.need_min_p_sampling
|
||||
|
||||
self.adjusted_merge_batch(other)
|
||||
|
||||
def copy_for_forward(self):
|
||||
# Accumulate the penalty into a pre-allocated buffer to get rid of the dependency of `penalizer_orchestrator` later
|
||||
self.update_penalties()
|
||||
|
||||
Reference in New Issue
Block a user