perf: reduce ttft and itl with stream_interval 1 (#658)

2024-07-19 09:14:22 -07:00
parent e1792cca24
commit ac971ff633
1 changed files with 1 additions and 1 deletions
@@ -33,7 +33,7 @@ class ServerArgs:

    # Other runtime options
    tp_size: int = 1
-    stream_interval: int = 8
+    stream_interval: int = 1
    random_seed: Optional[int] = None

    # Logging