Simplify the event loop and expose --num-continuous-decode-steps as an argument (#1652)

2024-10-12 21:35:30 -07:00
parent 9610fcd469
commit 7ee6c259ff
5 changed files with 85 additions and 62 deletions
--- a/python/sglang/srt/managers/tp_worker.py
+++ b/python/sglang/srt/managers/tp_worker.py
@@ -118,7 +118,7 @@ class TpModelWorker:
    def forward_batch_embedding(self, model_worker_batch: ModelWorkerBatch):
        forward_batch = ForwardBatch.init_new(model_worker_batch, self.model_runner)
        logits_output = self.model_runner.forward(forward_batch)
-        embeddings = logits_output.embeddings.tolist()
+        embeddings = logits_output.embeddings
        return embeddings

    def update_weights(self, recv_req: UpdateWeightReqInput):