[model-gateway]: add gRPC router embeddings endpoint implementation (#15273)
This commit is contained in:
@@ -455,10 +455,22 @@ class SGLangSchedulerServicer(sglang_scheduler_pb2_grpc.SglangSchedulerServicer)
|
||||
input_text = grpc_req.tokenized.original_text
|
||||
input_ids = list(grpc_req.tokenized.input_ids)
|
||||
|
||||
# Convert sampling params
|
||||
sampling_params = self._convert_sampling_params(grpc_req.sampling_params)
|
||||
|
||||
# For embedding requests, max_new_tokens should be 0.
|
||||
# The scheduler logic expects an integer, not None.
|
||||
sampling_params.max_new_tokens = 0
|
||||
|
||||
sampling_params.normalize(tokenizer=None)
|
||||
|
||||
return TokenizedEmbeddingReqInput(
|
||||
rid=grpc_req.request_id,
|
||||
input_text=input_text,
|
||||
input_ids=input_ids,
|
||||
image_inputs={"mm_items": []},
|
||||
token_type_ids=list(grpc_req.token_type_ids),
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
|
||||
def _convert_sampling_params(
|
||||
|
||||
Reference in New Issue
Block a user