Improve streaming, log_level, memory report, weight loading, and benchmark script (#7632)

Co-authored-by: Kan Wu <wukanustc@gmail.com>
This commit is contained in:
Lianmin Zheng
2025-06-29 23:16:19 -07:00
committed by GitHub
co-authored by Kan Wu
parent c5131f7a2f
commit 22352d47a9
24 changed files with 626 additions and 160 deletions
@@ -521,11 +521,17 @@ class SchedulerOutputProcessorMixin:
stream_interval = (
req.sampling_params.stream_interval or self.stream_interval
)
should_output = len(req.output_ids) % stream_interval == 0
should_output = (
len(req.output_ids) % stream_interval == 1
if not self.model_config.is_multimodal_gen
and stream_interval > 1
else len(req.output_ids) % stream_interval == 0
)
else:
should_output = (
len(req.output_ids) % DEFAULT_FORCE_STREAM_INTERVAL == 0
and not self.model_config.is_multimodal_gen
if not self.model_config.is_multimodal_gen
else False
)
if should_output: