chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Bugfix] Fix embedding model hangs with --enable-metrics (#2822)

Browse Source
This commit is contained in:
Chang Su
2025-01-10 13:14:51 -08:00
committed by GitHub
parent 8f15789314
commit f290bd4332
4 changed files with 49 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -688,7 +688,7 @@ class TokenizerManager:
if self.enable_metrics:
completion_tokens = (
recv_obj.completion_tokens[i]
if recv_obj.completion_tokens
if getattr(recv_obj, "completion_tokens", None)
else 0
)
@@ -716,7 +716,11 @@ class TokenizerManager:
time.time() - state.created_time
)
# Compute time_per_output_token for the non-streaming case
if not state.obj.stream and completion_tokens >= 1:
if (
hasattr(state.obj, "stream")
and not state.obj.stream
and completion_tokens >= 1
):
self.metrics_collector.observe_time_per_output_token(
(time.time() - state.created_time)
/ completion_tokens