[model-gateway]: add gRPC router embeddings endpoint implementation (#15273)

This commit is contained in:
ratish
2025-12-23 21:12:22 +04:00
committed by GitHub
parent 80ae2229d3
commit 5f3a47d8a7
30 changed files with 1097 additions and 35 deletions

View File

@@ -455,10 +455,22 @@ class SGLangSchedulerServicer(sglang_scheduler_pb2_grpc.SglangSchedulerServicer)
input_text = grpc_req.tokenized.original_text
input_ids = list(grpc_req.tokenized.input_ids)
# Convert sampling params
sampling_params = self._convert_sampling_params(grpc_req.sampling_params)
# For embedding requests, max_new_tokens should be 0.
# The scheduler logic expects an integer, not None.
sampling_params.max_new_tokens = 0
sampling_params.normalize(tokenizer=None)
return TokenizedEmbeddingReqInput(
rid=grpc_req.request_id,
input_text=input_text,
input_ids=input_ids,
image_inputs={"mm_items": []},
token_type_ids=list(grpc_req.token_type_ids),
sampling_params=sampling_params,
)
def _convert_sampling_params(