chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix input_ids && rename to fill_ids (#1021)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-08-10 16:24:12 -07:00
committed by GitHub
parent 54fb1c80c0
commit 43fbb6d919
7 changed files with 28 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/tp_worker.py
+4 -4
View File
@@ -515,7 +515,7 @@ class ModelTpServer:
self.handle_finished_requests(batch)
def add_logprob_return_values(self, i, req, pt, next_token_ids, output):
def add_logprob_return_values(self, i, req: Req, pt, next_token_ids, output):
if req.normalized_prompt_logprob is None:
req.normalized_prompt_logprob = output.normalized_prompt_logprobs[i]
@@ -524,12 +524,12 @@ class ModelTpServer:
req.input_token_logprobs = list(
zip(
output.input_token_logprobs[pt : pt + req.extend_input_len - 1],
req.input_ids[-req.extend_input_len + 1 :],
req.fill_ids[-req.extend_input_len + 1 :],
)
)
if req.logprob_start_len == 0:
req.input_token_logprobs = [
(None, req.input_ids[0])
(None, req.fill_ids[0])
] + req.input_token_logprobs
if req.last_update_decode_tokens != 0:
@@ -543,7 +543,7 @@ class ModelTpServer:
+ req.extend_input_len
- 1
],
req.input_ids[-req.last_update_decode_tokens + 1 :],
req.fill_ids[-req.last_update_decode_tokens + 1 :],
)
)
)