From 38dc372dae00fceb372e0cca75312c136f156713 Mon Sep 17 00:00:00 2001 From: Jun Liu Date: Sun, 1 Mar 2026 13:21:35 +0900 Subject: [PATCH] [Bugfix] Fix KeyError: 'prompt_tokens' when streaming requests are aborted (#19514) --- python/sglang/srt/entrypoints/openai/serving_chat.py | 6 ++++-- python/sglang/srt/entrypoints/openai/serving_completions.py | 6 ++++-- python/sglang/srt/entrypoints/openai/usage_processor.py | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index fde200f82..6e50f3278 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -632,8 +632,10 @@ class OpenAIServingChat(OpenAIServingBase): ): index = content.get("index", 0) - prompt_tokens[index] = content["meta_info"]["prompt_tokens"] - completion_tokens[index] = content["meta_info"]["completion_tokens"] + prompt_tokens[index] = content["meta_info"].get("prompt_tokens", 0) + completion_tokens[index] = content["meta_info"].get( + "completion_tokens", 0 + ) cached_tokens[index] = content["meta_info"].get("cached_tokens", 0) hidden_states[index] = content["meta_info"].get("hidden_states", None) routed_experts[index] = content["meta_info"].get("routed_experts", None) diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py index fd61942d7..a639d2bed 100644 --- a/python/sglang/srt/entrypoints/openai/serving_completions.py +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -209,8 +209,10 @@ class OpenAIServingCompletion(OpenAIServingBase): index = content.get("index", 0) text = content["text"] - prompt_tokens[index] = content["meta_info"]["prompt_tokens"] - completion_tokens[index] = content["meta_info"]["completion_tokens"] + prompt_tokens[index] = content["meta_info"].get("prompt_tokens", 0) + completion_tokens[index] = content["meta_info"].get( + "completion_tokens", 0 + ) cached_tokens[index] = content["meta_info"].get("cached_tokens", 0) hidden_states[index] = content["meta_info"].get("hidden_states", None) routed_experts[index] = content["meta_info"].get("routed_experts", None) diff --git a/python/sglang/srt/entrypoints/openai/usage_processor.py b/python/sglang/srt/entrypoints/openai/usage_processor.py index f02b858eb..de88e9eb0 100644 --- a/python/sglang/srt/entrypoints/openai/usage_processor.py +++ b/python/sglang/srt/entrypoints/openai/usage_processor.py @@ -20,10 +20,12 @@ class UsageProcessor: n_choices: int = 1, enable_cache_report: bool = False, ) -> UsageInfo: - completion_tokens = sum(r["meta_info"]["completion_tokens"] for r in responses) + completion_tokens = sum( + r["meta_info"].get("completion_tokens", 0) for r in responses + ) prompt_tokens = sum( - responses[i]["meta_info"]["prompt_tokens"] + responses[i]["meta_info"].get("prompt_tokens", 0) for i in range(0, len(responses), n_choices) )