chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Remove monkey_patch_vllm_dummy_weight_loader (#2064)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-17 15:48:12 -08:00
committed by GitHub
parent c1f401fc58
commit 38625e2139
6 changed files with 17 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/scheduler.py
+2 -2
View File
@@ -895,7 +895,7 @@ class Scheduler:
logits_output, next_token_ids, bid = result
if self.enable_overlap:
logits_output, next_token_ids = self.tp_worker.resulve_batch_result(bid)
logits_output, next_token_ids = self.tp_worker.resolve_batch_result(bid)
else:
# Move next_token_ids and logprobs to cpu
if batch.return_logprob:
@@ -970,7 +970,7 @@ class Scheduler:
self.num_generated_tokens += len(batch.reqs)
if self.enable_overlap:
logits_output, next_token_ids = self.tp_worker.resulve_batch_result(bid)
logits_output, next_token_ids = self.tp_worker.resolve_batch_result(bid)
next_token_logprobs = logits_output.next_token_logprobs
else:
# Move next_token_ids and logprobs to cpu
python/sglang/srt/managers/tp_worker_overlap_thread.py
+1 -1
View File
@@ -141,7 +141,7 @@ class TpModelWorkerClient:
self.launch_event.set()
self.output_queue.put((copy_event, logits_output, next_token_ids))
def resulve_batch_result(self, bid: int):
def resolve_batch_result(self, bid: int):
copy_event, logits_output, next_token_ids = self.output_queue.get()
while not copy_event.query():
time.sleep(1e-5)