Refine OpenAI serving entrypoint to remove batch requests (#7372)
Signed-off-by: Xinyuan Tong <justinning0323@outlook.com> Co-authored-by: Chang Su <csu272@usc.edu>
This commit is contained in:
@@ -54,35 +54,25 @@ class OpenAIServingEmbedding(OpenAIServingBase):
|
||||
return f"All items in input list must be integers"
|
||||
if item < 0:
|
||||
return f"Token ID at index {i} must be non-negative"
|
||||
elif isinstance(first_item, list):
|
||||
# List of lists (multiple token sequences)
|
||||
for i, item in enumerate(input):
|
||||
if not isinstance(item, list):
|
||||
return f"Input at index {i} must be a list"
|
||||
if not item:
|
||||
return f"Input at index {i} cannot be empty"
|
||||
if not all(isinstance(token, int) for token in item):
|
||||
return f"Input at index {i} must contain only integers"
|
||||
if any(token < 0 for token in item):
|
||||
return f"Input at index {i} contains negative token IDs"
|
||||
# Note: MultimodalEmbeddingInput validation would be handled by Pydantic
|
||||
|
||||
return None
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: EmbeddingRequest,
|
||||
request_id: str,
|
||||
) -> tuple[EmbeddingReqInput, Union[EmbeddingRequest, List[EmbeddingRequest]]]:
|
||||
) -> tuple[EmbeddingReqInput, EmbeddingRequest]:
|
||||
"""Convert OpenAI embedding request to internal format"""
|
||||
prompt = request.input
|
||||
|
||||
if isinstance(prompt, str):
|
||||
# Single string input
|
||||
prompt_kwargs = {"text": prompt}
|
||||
elif isinstance(prompt, list):
|
||||
if len(prompt) > 0 and isinstance(prompt[0], str):
|
||||
# List of strings
|
||||
prompt_kwargs = {"text": prompt}
|
||||
# List of strings - if it's a single string in a list, treat as single string
|
||||
if len(prompt) == 1:
|
||||
prompt_kwargs = {"text": prompt[0]}
|
||||
else:
|
||||
prompt_kwargs = {"text": prompt}
|
||||
elif len(prompt) > 0 and isinstance(prompt[0], MultimodalEmbeddingInput):
|
||||
# Handle multimodal embedding inputs
|
||||
texts = []
|
||||
@@ -94,7 +84,6 @@ class OpenAIServingEmbedding(OpenAIServingBase):
|
||||
|
||||
generate_prompts = []
|
||||
# Check if we have a chat template for multimodal embeddings
|
||||
# This would need to be passed in from the server configuration
|
||||
chat_template_name = getattr(
|
||||
self.tokenizer_manager, "chat_template_name", None
|
||||
)
|
||||
@@ -121,6 +110,7 @@ class OpenAIServingEmbedding(OpenAIServingBase):
|
||||
else:
|
||||
# Other types (should not happen but handle gracefully)
|
||||
prompt_kwargs = {"input_ids": prompt}
|
||||
|
||||
adapted_request = EmbeddingReqInput(
|
||||
**prompt_kwargs,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user