Improve streaming, log_level, memory report, weight loading, and benchmark script (#7632)
Co-authored-by: Kan Wu <wukanustc@gmail.com>
This commit is contained in:
@@ -521,11 +521,17 @@ class SchedulerOutputProcessorMixin:
|
||||
stream_interval = (
|
||||
req.sampling_params.stream_interval or self.stream_interval
|
||||
)
|
||||
should_output = len(req.output_ids) % stream_interval == 0
|
||||
should_output = (
|
||||
len(req.output_ids) % stream_interval == 1
|
||||
if not self.model_config.is_multimodal_gen
|
||||
and stream_interval > 1
|
||||
else len(req.output_ids) % stream_interval == 0
|
||||
)
|
||||
else:
|
||||
should_output = (
|
||||
len(req.output_ids) % DEFAULT_FORCE_STREAM_INTERVAL == 0
|
||||
and not self.model_config.is_multimodal_gen
|
||||
if not self.model_config.is_multimodal_gen
|
||||
else False
|
||||
)
|
||||
|
||||
if should_output:
|
||||
|
||||
Reference in New Issue
Block a user