From 7f5055ed5e896d8495d462ddde6b00dc7eafc9bb Mon Sep 17 00:00:00 2001 From: Frank Minors <59649479+FrankMinions@users.noreply.github.com> Date: Wed, 12 Nov 2025 01:02:43 +0800 Subject: [PATCH] Fix cached tokens usage bug (#12814) Co-authored-by: FrankMinions --- python/sglang/srt/entrypoints/openai/usage_processor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/usage_processor.py b/python/sglang/srt/entrypoints/openai/usage_processor.py index cdd14d74e..cd7a18b80 100644 --- a/python/sglang/srt/entrypoints/openai/usage_processor.py +++ b/python/sglang/srt/entrypoints/openai/usage_processor.py @@ -30,7 +30,8 @@ class UsageProcessor: cached_details = None if enable_cache_report: cached_total = sum( - r["meta_info"].get("cached_tokens", 0) for r in responses + responses[i]["meta_info"].get("cached_tokens", 0) + for i in range(0, len(responses), n_choices) ) cached_details = UsageProcessor._details_if_cached(cached_total) @@ -55,7 +56,9 @@ class UsageProcessor: total_completion_tokens = sum(completion_tokens.values()) cached_details = ( - UsageProcessor._details_if_cached(sum(cached_tokens.values())) + UsageProcessor._details_if_cached( + sum(tok for idx, tok in cached_tokens.items() if idx % n_choices == 0) + ) if enable_cache_report else None )