Add openai embedding API (#997)

2024-08-09 11:19:18 -07:00
parent 05c50a82b8
commit b16e856f11
8 changed files with 135 additions and 19 deletions
@@ -499,6 +499,8 @@ class ModelTpServer:
                req.embedding = embeddings[i]
                if req is not self.current_inflight_req:
                    # Inflight reqs' prefill is not finished
+                    # dummy output token for embedding models
+                    req.output_ids.append(0)
                    req.check_finished()

                if req.finished():