Revert "feat: reduce constrained-decoding overhead in TP" (#16845)
This commit is contained in:
@@ -201,11 +201,7 @@ class SamplingBatchInfo:
|
||||
return
|
||||
|
||||
# Find a grammar from the list
|
||||
first_grammar = next((grammar for grammar in self.grammars if grammar), None)
|
||||
if first_grammar is None:
|
||||
self.vocab_mask = None
|
||||
self.apply_mask_func = None
|
||||
return
|
||||
first_grammar = next(grammar for grammar in self.grammars if grammar)
|
||||
|
||||
# TODO(lianmin): Maybe we can reuse the existing mask?
|
||||
self.vocab_mask = first_grammar.allocate_vocab_mask(
|
||||
@@ -236,7 +232,7 @@ class SamplingBatchInfo:
|
||||
else:
|
||||
self.acc_linear_penalties = None
|
||||
|
||||
def apply_logits_bias(self, logits: torch.Tensor, apply_vocab_mask: bool = True):
|
||||
def apply_logits_bias(self, logits: torch.Tensor):
|
||||
if self.acc_linear_penalties is not None:
|
||||
# Used in the overlap mode
|
||||
logits.add_(self.acc_linear_penalties)
|
||||
@@ -245,7 +241,7 @@ class SamplingBatchInfo:
|
||||
# Used in the non-overlap mode
|
||||
self.penalizer_orchestrator.apply(logits)
|
||||
|
||||
if apply_vocab_mask and self.vocab_mask is not None:
|
||||
if self.vocab_mask is not None:
|
||||
self.apply_mask_func(logits=logits, vocab_mask=self.vocab_mask)
|
||||
|
||||
if self.logit_bias is not None:
|
||||
|
||||
@@ -102,19 +102,6 @@ class SamplingParams:
|
||||
if self.top_k == -1:
|
||||
self.top_k = TOP_K_ALL # whole vocabulary
|
||||
|
||||
@property
|
||||
def has_grammar_constraint(self) -> bool:
|
||||
"""
|
||||
Helper property to check if any grammar constraint
|
||||
exists for these SamplingParams.
|
||||
"""
|
||||
return (
|
||||
self.json_schema is not None
|
||||
or self.regex is not None
|
||||
or self.ebnf is not None
|
||||
or self.structural_tag is not None
|
||||
)
|
||||
|
||||
def verify(self, vocab_size):
|
||||
if self.temperature < 0.0:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user