Support penalty in overlap mode; return logprob with chunked prefill; improve benchmark scripts (#3988)
Co-authored-by: SangBin Cho <rkooo567@gmail.com> Co-authored-by: dhou-xai <dhou@x.ai> Co-authored-by: Hanming Lu <hanming_lu@berkeley.edu>
This commit is contained in:
@@ -35,12 +35,12 @@ class SessionReqNode:
|
||||
for req_node in self.childs:
|
||||
req_node.clear(req_dict)
|
||||
|
||||
if self.req.finished_reason == None:
|
||||
if self.req.finished_reason is None:
|
||||
self.req.to_abort = True
|
||||
del req_dict[self.req.rid]
|
||||
|
||||
def abort(self):
|
||||
if self.req.finished_reason == None:
|
||||
if self.req.finished_reason is None:
|
||||
self.req.to_abort = True
|
||||
|
||||
def __str__(self):
|
||||
@@ -132,6 +132,10 @@ class Session:
|
||||
lora_path=req.lora_path,
|
||||
session_id=self.session_id,
|
||||
custom_logit_processor=req.custom_logit_processor,
|
||||
stream=req.stream,
|
||||
return_logprob=req.return_logprob,
|
||||
top_logprobs_num=req.top_logprobs_num,
|
||||
token_ids_logprob=req.token_ids_logprob,
|
||||
)
|
||||
if last_req is not None:
|
||||
new_req.image_inputs = last_req.image_inputs
|
||||
|
||||
Reference in New Issue
Block a user