diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 2acf07cd5..3b8773a55 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -627,16 +627,22 @@ class OpenAIServingChat(OpenAIServingBase): routed_experts[index] = content["meta_info"].get("routed_experts", None) # Handle logprobs + finish_reason = content["meta_info"]["finish_reason"] choice_logprobs = None if request.logprobs: - choice_logprobs = self._process_streaming_logprobs( - content, n_prev_tokens.get(index, 0) - ) - n_prev_tokens[index] = len( + n_prev_token = n_prev_tokens.get(index, 0) + total_output_logprobs = len( content["meta_info"]["output_token_logprobs"] ) - - finish_reason = content["meta_info"]["finish_reason"] + # When finish_reason is set and all logprobs have been sent, + # any remaining text is just buffered text being flushed by the + # detokenizer (it holds back text at word boundaries). Return None + # for logprobs since no new tokens were generated for this text. + if n_prev_token < total_output_logprobs or finish_reason is None: + choice_logprobs = self._process_streaming_logprobs( + content, n_prev_token + ) + n_prev_tokens[index] = total_output_logprobs finish_reason_type = finish_reason["type"] if finish_reason else None # Track finish_reason for each index diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py index 10c04552c..6f437e4e2 100644 --- a/python/sglang/srt/entrypoints/openai/serving_completions.py +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -242,19 +242,34 @@ class OpenAIServingCompletion(OpenAIServingBase): input_top_logprobs = None n_prev_token = n_prev_tokens.get(index, 0) - logprobs = to_openai_style_logprobs( - input_token_logprobs=input_token_logprobs, - input_top_logprobs=input_top_logprobs, - output_token_logprobs=content["meta_info"][ - "output_token_logprobs" - ][n_prev_token:], - output_top_logprobs=content["meta_info"].get( - "output_top_logprobs", [] - )[n_prev_token:], - ) - n_prev_tokens[index] = len( + total_output_logprobs = len( content["meta_info"]["output_token_logprobs"] ) + output_logprobs_slice = content["meta_info"][ + "output_token_logprobs" + ][n_prev_token:] + finish_reason_for_logprobs = content["meta_info"]["finish_reason"] + + # When finish_reason is set and all logprobs have been sent, + # any remaining text is just buffered text being flushed by the + # detokenizer (it holds back text at word boundaries). Return None + # for logprobs since no new tokens were generated for this text. + if ( + len(output_logprobs_slice) == 0 + and finish_reason_for_logprobs is not None + and input_token_logprobs is None + ): + logprobs = None + else: + logprobs = to_openai_style_logprobs( + input_token_logprobs=input_token_logprobs, + input_top_logprobs=input_top_logprobs, + output_token_logprobs=output_logprobs_slice, + output_top_logprobs=content["meta_info"].get( + "output_top_logprobs", [] + )[n_prev_token:], + ) + n_prev_tokens[index] = total_output_logprobs # Generate delta delta = text[len(stream_buffer) :] diff --git a/test/registered/openai_server/basic/test_openai_server.py b/test/registered/openai_server/basic/test_openai_server.py index 235c92d73..33032f134 100644 --- a/test/registered/openai_server/basic/test_openai_server.py +++ b/test/registered/openai_server/basic/test_openai_server.py @@ -156,23 +156,23 @@ class TestOpenAIServer(CustomTestCase): is_first = is_firsts.get(index, True) if logprobs: - assert response.choices[0].logprobs, f"no logprobs in response" - print( - f"{response=}, {response.choices[0]=}, {response.choices[0].logprobs=}" - ) - assert isinstance( - response.choices[0].logprobs.tokens[0], str - ), f"{response.choices[0].logprobs.tokens[0]} is not a string" - if not (is_first and echo): + # When finish_reason is set, logprobs may be None if this chunk + # only contains buffered text being flushed (no new tokens generated). + # The detokenizer holds back text at word boundaries during streaming. + if response.choices[0].logprobs is not None: assert isinstance( - response.choices[0].logprobs.top_logprobs[0], dict - ), f"top_logprobs was not a dictionary" - ret_num_top_logprobs = len( - response.choices[0].logprobs.top_logprobs[0] - ) - # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map - # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" - assert ret_num_top_logprobs > 0, f"ret_num_top_logprobs was 0" + response.choices[0].logprobs.tokens[0], str + ), f"{response.choices[0].logprobs.tokens[0]} is not a string" + if not (is_first and echo): + assert isinstance( + response.choices[0].logprobs.top_logprobs[0], dict + ), f"top_logprobs was not a dictionary" + ret_num_top_logprobs = len( + response.choices[0].logprobs.top_logprobs[0] + ) + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0, f"ret_num_top_logprobs was 0" if is_first: if echo: