Add constrained_json_whitespace_pattern to ServerArgs (#1438)

This commit is contained in:
zifeitong
2024-09-16 13:29:18 -07:00
committed by GitHub
parent 2abe4f1cb6
commit 93dffd699b
3 changed files with 17 additions and 7 deletions

View File

@@ -198,6 +198,7 @@ class ModelTpServer:
"trust_remote_code": server_args.trust_remote_code,
},
skip_tokenizer_init=server_args.skip_tokenizer_init,
constrained_json_whitespace_pattern=server_args.constrained_json_whitespace_pattern,
)
self.jump_forward_cache = JumpForwardCache()
@@ -807,12 +808,10 @@ class ModelTpServer:
unfinished_indices.append(i)
if req.finished() or (
(
req.stream
and (
self.decode_forward_ct % self.stream_interval == 0
or len(req.output_ids) == 1
)
req.stream
and (
self.decode_forward_ct % self.stream_interval == 0
or len(req.output_ids) == 1
)
):
output_rids.append(req.rid)