chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix flaky streaming logprobs test by handling detokenizer text buffering (#17687)

Browse Source 
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Kangyan-Zhou
2026-01-25 15:09:06 -08:00
committed by GitHub
parent bf139af37a
commit 592603d77b
3 changed files with 54 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
python/sglang/srt/entrypoints/openai/serving_chat.py
View File

@@ -627,16 +627,22 @@ class OpenAIServingChat(OpenAIServingBase):
routed_experts[index] = content["meta_info"].get("routed_experts", None)
# Handle logprobs
finish_reason = content["meta_info"]["finish_reason"]
choice_logprobs = None
if request.logprobs:
choice_logprobs = self._process_streaming_logprobs(
content, n_prev_tokens.get(index, 0)
)
n_prev_tokens[index] = len(
n_prev_token = n_prev_tokens.get(index, 0)
total_output_logprobs = len(
content["meta_info"]["output_token_logprobs"]
)
finish_reason = content["meta_info"]["finish_reason"]
# When finish_reason is set and all logprobs have been sent,
# any remaining text is just buffered text being flushed by the
# detokenizer (it holds back text at word boundaries). Return None
# for logprobs since no new tokens were generated for this text.
if n_prev_token < total_output_logprobs or finish_reason is None:
choice_logprobs = self._process_streaming_logprobs(
content, n_prev_token
)
n_prev_tokens[index] = total_output_logprobs
finish_reason_type = finish_reason["type"] if finish_reason else None
# Track finish_reason for each index

37
python/sglang/srt/entrypoints/openai/serving_completions.py
View File

@@ -242,19 +242,34 @@ class OpenAIServingCompletion(OpenAIServingBase):
input_top_logprobs = None
n_prev_token = n_prev_tokens.get(index, 0)
logprobs = to_openai_style_logprobs(
input_token_logprobs=input_token_logprobs,
input_top_logprobs=input_top_logprobs,
output_token_logprobs=content["meta_info"][
"output_token_logprobs"
][n_prev_token:],
output_top_logprobs=content["meta_info"].get(
"output_top_logprobs", []
)[n_prev_token:],
)
n_prev_tokens[index] = len(
total_output_logprobs = len(
content["meta_info"]["output_token_logprobs"]
)
output_logprobs_slice = content["meta_info"][
"output_token_logprobs"
][n_prev_token:]
finish_reason_for_logprobs = content["meta_info"]["finish_reason"]
# When finish_reason is set and all logprobs have been sent,
# any remaining text is just buffered text being flushed by the
# detokenizer (it holds back text at word boundaries). Return None
# for logprobs since no new tokens were generated for this text.
if (
len(output_logprobs_slice) == 0
and finish_reason_for_logprobs is not None
and input_token_logprobs is None
):
logprobs = None
else:
logprobs = to_openai_style_logprobs(
input_token_logprobs=input_token_logprobs,
input_top_logprobs=input_top_logprobs,
output_token_logprobs=output_logprobs_slice,
output_top_logprobs=content["meta_info"].get(
"output_top_logprobs", []
)[n_prev_token:],
)
n_prev_tokens[index] = total_output_logprobs
# Generate delta
delta = text[len(stream_buffer) :]