chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add per-request decode tp size (#14678)

Browse Source 
Co-authored-by: Byron Hsu <byronhsu1230@gmail.com>
This commit is contained in:
Lianmin Zheng
2025-12-08 20:24:31 -08:00
committed by GitHub
parent 0e0b0c0566
commit ce4e836be5
2 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/managers/io_struct.py
View File

@@ -196,6 +196,7 @@ class GenerateReqInput(BaseReq):
bootstrap_port: Optional[Union[List[Optional[int]], int]] = None
bootstrap_room: Optional[Union[List[int], int]] = None
bootstrap_pair_key: Optional[Union[List[str], str]] = None
decode_tp_size: Optional[Union[List[Optional[int]], int]] = None
# For reasoning
reasoning: bool = False
@@ -619,6 +620,9 @@ class GenerateReqInput(BaseReq):
if self.bootstrap_pair_key is not None
else None
),
decode_tp_size=(
self.decode_tp_size[i] if self.decode_tp_size is not None else None
),
validation_time=self.validation_time,
data_parallel_rank=(
self.data_parallel_rank if self.data_parallel_rank is not None else None
@@ -677,6 +681,7 @@ class TokenizedGenerateReqInput(BaseReq):
bootstrap_port: Optional[int] = None
bootstrap_room: Optional[int] = None
bootstrap_pair_key: Optional[str] = None
decode_tp_size: Optional[int] = None
# For reasoning
reasoning: bool = False

5
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -841,6 +841,9 @@ class TokenizerManager(TokenizerCommunicatorMixin):
f"The input_ids {input_ids} contains values greater than the vocab size ({vocab_size})."
)
def _get_sampling_params(self, sampling_kwargs: Dict) -> SamplingParams:
return SamplingParams(**sampling_kwargs)
def _create_tokenized_object(
self,
obj: Union[GenerateReqInput, EmbeddingReqInput],
@@ -858,7 +861,7 @@ class TokenizerManager(TokenizerCommunicatorMixin):
sampling_kwargs = {**self.preferred_sampling_params, **obj.sampling_params}
else:
sampling_kwargs = obj.sampling_params
sampling_params = SamplingParams(**sampling_kwargs)
sampling_params = self._get_sampling_params(sampling_kwargs)
sampling_params.normalize(self.tokenizer)
sampling_params.verify(self.model_config.vocab_size)