chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix flaky streaming logprobs test by handling detokenizer text buffering (#17687)

Browse Source 
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Kangyan-Zhou
2026-01-25 15:09:06 -08:00
committed by GitHub
co-authored by Xinyuan Tong
parent bf139af37a
commit 592603d77b
3 changed files with 54 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/openai_server/basic/test_openai_server.py
+16 -16
View File
@@ -156,23 +156,23 @@ class TestOpenAIServer(CustomTestCase):
is_first = is_firsts.get(index, True)
if logprobs:
assert response.choices[0].logprobs, f"no logprobs in response"
print(
f"{response=}, {response.choices[0]=}, {response.choices[0].logprobs=}"
)
assert isinstance(
response.choices[0].logprobs.tokens[0], str
), f"{response.choices[0].logprobs.tokens[0]} is not a string"
if not (is_first and echo):
# When finish_reason is set, logprobs may be None if this chunk
# only contains buffered text being flushed (no new tokens generated).
# The detokenizer holds back text at word boundaries during streaming.
if response.choices[0].logprobs is not None:
assert isinstance(
response.choices[0].logprobs.top_logprobs[0], dict
), f"top_logprobs was not a dictionary"
ret_num_top_logprobs = len(
response.choices[0].logprobs.top_logprobs[0]
)
# FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map
# assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}"
assert ret_num_top_logprobs > 0, f"ret_num_top_logprobs was 0"
response.choices[0].logprobs.tokens[0], str
), f"{response.choices[0].logprobs.tokens[0]} is not a string"
if not (is_first and echo):
assert isinstance(
response.choices[0].logprobs.top_logprobs[0], dict
), f"top_logprobs was not a dictionary"
ret_num_top_logprobs = len(
response.choices[0].logprobs.top_logprobs[0]
)
# FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map
# assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}"
assert ret_num_top_logprobs > 0, f"ret_num_top_logprobs was 0"
if is_first:
if echo: