Improve the control of streaming and improve the first token latency in streaming (#117)

2024-01-29 17:05:42 -08:00
parent cd6872334e
commit 6f560c761b
12 changed files with 46 additions and 23 deletions
--- a/python/sglang/srt/managers/router/manager.py
+++ b/python/sglang/srt/managers/router/manager.py
@@ -40,7 +40,7 @@ class RouterManager:
            for obj in out_pyobjs:
                self.send_to_detokenizer.send_pyobj(obj)

-            # async sleep for recving the subsequent request, and avoiding cache miss
+            # async sleep for receiving the subsequent request and avoiding cache miss
            if len(out_pyobjs) != 0:
                has_finished = any([obj.finished for obj in out_pyobjs])
                if has_finished: