feat: support EPD disaggregation (#12263)
Co-authored-by: liusy58 <liusy58@linux.alibaba.com> Co-authored-by: ZhengWG <zwg0606@gmail.com> Co-authored-by: Nicholas <45984215+liusy58@users.noreply.github.com> Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com>
This commit is contained in:
co-authored by
liusy58
ZhengWG
Nicholas
Shangming Cai
Yuhao Yang
parent
a9ce1623cd
commit
9acb21ae27
@@ -0,0 +1,544 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import pickle
|
||||
import random
|
||||
import threading
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
import aiohttp
|
||||
import torch
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
|
||||
from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferEngine
|
||||
from sglang.srt.managers.io_struct import TokenizedGenerateReqInput
|
||||
from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import get_local_ip_auto, get_zmq_socket_on_host
|
||||
from sglang.srt.utils.hf_transformers_utils import get_processor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EmbeddingData:
|
||||
def __init__(self, req_id, num_parts, part_idx, image_grid_dim, embedding=None):
|
||||
self.req_id = req_id
|
||||
self.num_parts = num_parts
|
||||
self.part_idx = part_idx
|
||||
self.image_grid_dim = image_grid_dim
|
||||
self.embedding = embedding
|
||||
self.send_time = None
|
||||
self.dtype = embedding.dtype if embedding is not None else None
|
||||
self.shape = list(embedding.shape) if embedding is not None else None
|
||||
# aggregated data
|
||||
self.ready_list = [i == self.part_idx for i in range(self.num_parts)]
|
||||
self.embedding_list = [
|
||||
embedding if i == self.part_idx else None for i in range(self.num_parts)
|
||||
]
|
||||
self.image_grid_dim_list = [
|
||||
self.image_grid_dim if i == self.part_idx else None
|
||||
for i in range(self.num_parts)
|
||||
]
|
||||
|
||||
def add(self, embedding_data):
|
||||
assert self.req_id == embedding_data.req_id
|
||||
assert not self.ready_list[embedding_data.part_idx]
|
||||
self.ready_list[embedding_data.part_idx] = True
|
||||
self.image_grid_dim_list[embedding_data.part_idx] = (
|
||||
embedding_data.image_grid_dim
|
||||
)
|
||||
self.embedding_list[embedding_data.part_idx] = embedding_data.embedding
|
||||
|
||||
def get_embedding(self, is_concat=False):
|
||||
if is_concat:
|
||||
return torch.concat([embedding.cuda() for embedding in self.embedding_list])
|
||||
else:
|
||||
return self.embedding_list
|
||||
|
||||
def get_img_grid(self):
|
||||
return torch.concatenate(self.image_grid_dim_list)
|
||||
|
||||
@property
|
||||
def ready(self):
|
||||
return sum(self.ready_list) == self.num_parts
|
||||
|
||||
def __repr__(self):
|
||||
return f"EmbeddingData(req_id={self.req_id}, num_parts={self.num_parts}, part_idx={self.part_idx})"
|
||||
|
||||
def copy_without_embedding(self):
|
||||
new_data = EmbeddingData(
|
||||
req_id=self.req_id,
|
||||
num_parts=self.num_parts,
|
||||
part_idx=self.part_idx,
|
||||
image_grid_dim=self.image_grid_dim,
|
||||
)
|
||||
new_data.send_time = self.send_time
|
||||
new_data.dtype = self.dtype
|
||||
new_data.shape = self.shape
|
||||
return new_data
|
||||
|
||||
|
||||
# For zmq_to_scheduler
|
||||
class WaitingImageRequest:
|
||||
def __init__(
|
||||
self,
|
||||
rid: str,
|
||||
recv_req: TokenizedGenerateReqInput,
|
||||
mm_processor,
|
||||
encoder_urls,
|
||||
host_name,
|
||||
receive_count,
|
||||
embedding_port=None,
|
||||
):
|
||||
self.rid = rid
|
||||
self.recv_req = recv_req
|
||||
self.mm_inputs = None
|
||||
self.error = None
|
||||
self.thread = None
|
||||
self.mm_processor = mm_processor
|
||||
self.encoder_urls = encoder_urls
|
||||
self.host_name = host_name
|
||||
self.receive_count = receive_count
|
||||
self.num_items_assigned = recv_req.num_items_assigned
|
||||
self.embedding_port, self.recv_socket = get_zmq_socket_on_host(
|
||||
zmq.Context(), zmq.PULL
|
||||
)
|
||||
logger.info(f"Waiting for input {self.embedding_port = }")
|
||||
self.recv_embedding_data = None
|
||||
self.ready = False
|
||||
|
||||
def send_encode_request(self):
|
||||
async def _send_single_request(session, url, payload):
|
||||
try:
|
||||
async with session.post(url, json=payload) as response:
|
||||
response.raise_for_status()
|
||||
return await response.text()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send request to {url}: {e}")
|
||||
raise
|
||||
|
||||
async def send_embedding_port(req_id, receive_count, host_name, embedding_port):
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=1800)
|
||||
) as session:
|
||||
tasks = []
|
||||
logger.info(f"{self.num_items_assigned = } ")
|
||||
for idx, assigned_num in enumerate(self.num_items_assigned):
|
||||
if assigned_num == 0:
|
||||
continue
|
||||
encoder_url = self.encoder_urls[idx]
|
||||
target_url = f"{encoder_url}/scheduler_receive_url"
|
||||
payload = {
|
||||
"req_id": req_id,
|
||||
"receive_count": receive_count,
|
||||
"receive_url": f"{host_name}:{embedding_port}",
|
||||
}
|
||||
|
||||
logger.info(f"Preparing to send to {target_url}")
|
||||
|
||||
task = _send_single_request(session, target_url, payload)
|
||||
tasks.append(task)
|
||||
|
||||
if not tasks:
|
||||
logger.info("No tasks to send.")
|
||||
return
|
||||
logger.info(f"Concurrently sending {len(tasks)} requests...")
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if isinstance(result, Exception):
|
||||
logger.error(f"Request {i} failed: {result}")
|
||||
else:
|
||||
logger.debug(f"Request {i} succeeded.")
|
||||
|
||||
asyncio.run(
|
||||
send_embedding_port(
|
||||
self.recv_req.rid,
|
||||
self.receive_count,
|
||||
self.host_name,
|
||||
self.embedding_port,
|
||||
)
|
||||
)
|
||||
|
||||
def _try_recv_mm_data(self):
|
||||
if self.ready:
|
||||
return
|
||||
while self.recv_embedding_data is None or not self.recv_embedding_data.ready:
|
||||
try:
|
||||
parts = self.recv_socket.recv_multipart(flags=zmq.NOBLOCK, copy=False)
|
||||
except zmq.Again:
|
||||
# No data available yet, wait a bit and retry
|
||||
return
|
||||
|
||||
recv_obj: EmbeddingData = pickle.loads(parts[0])
|
||||
buffer = parts[1].buffer if hasattr(parts[1], "buffer") else parts[1]
|
||||
recv_obj.embedding = torch.frombuffer(buffer, dtype=recv_obj.dtype).reshape(
|
||||
recv_obj.shape
|
||||
)
|
||||
recv_obj.embedding_list[recv_obj.part_idx] = recv_obj.embedding
|
||||
if self.recv_embedding_data is None:
|
||||
self.recv_embedding_data = recv_obj
|
||||
else:
|
||||
self.recv_embedding_data.add(recv_obj)
|
||||
|
||||
recv_embedding = self.recv_embedding_data.get_embedding(is_concat=True)
|
||||
img_grid_thw = self.recv_embedding_data.get_img_grid()
|
||||
|
||||
mm_inputs = self.mm_processor.get_mm_data(
|
||||
self.recv_req.input_text, recv_embedding, img_grid_thw
|
||||
)
|
||||
self.recv_req.mm_inputs = mm_inputs
|
||||
self.recv_req.input_ids = mm_inputs["input_ids"]
|
||||
self.ready = True
|
||||
self.recv_socket.close()
|
||||
|
||||
|
||||
def _determine_tensor_transport_mode(server_args):
|
||||
is_cross_node = server_args.dist_init_addr
|
||||
|
||||
if is_cross_node:
|
||||
# Fallback to default CPU transport for multi-node
|
||||
return "default"
|
||||
else:
|
||||
return "cuda_ipc"
|
||||
|
||||
|
||||
class MMReceiver:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
dtype=None,
|
||||
hf_config=None,
|
||||
pp_rank=None,
|
||||
tp_rank=None,
|
||||
):
|
||||
self.context = zmq.asyncio.Context(20)
|
||||
self.encoder_transfer_backend = server_args.encoder_transfer_backend
|
||||
self.encode_urls = server_args.encoder_urls
|
||||
self.encode_idx = list(range(len(self.encode_urls)))
|
||||
self.host = server_args.host
|
||||
if self.encoder_transfer_backend == "mooncake":
|
||||
self.dtype = dtype
|
||||
self.embeddings_engine = MooncakeTransferEngine(
|
||||
hostname=get_local_ip_auto(),
|
||||
gpu_id=None,
|
||||
ib_device=server_args.disaggregation_ib_device,
|
||||
)
|
||||
self.embeddings_buffer = dict()
|
||||
elif self.encoder_transfer_backend == "zmq_to_scheduler":
|
||||
self.pp_rank = pp_rank
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = server_args.tp_size
|
||||
self.nnodes = server_args.nnodes
|
||||
self.hostname = get_local_ip_auto()
|
||||
self.world_size = server_args.pp_size * server_args.tp_size
|
||||
self.waiting_list: List[WaitingImageRequest] = []
|
||||
if hf_config is not None:
|
||||
transport_mode = _determine_tensor_transport_mode(server_args)
|
||||
import_processors("sglang.srt.multimodal.processors")
|
||||
_processor = None
|
||||
try:
|
||||
_processor = get_processor(
|
||||
server_args.tokenizer_path,
|
||||
tokenizer_mode=server_args.tokenizer_mode,
|
||||
trust_remote_code=server_args.trust_remote_code,
|
||||
revision=server_args.revision,
|
||||
use_fast=not server_args.disable_fast_image_processor,
|
||||
)
|
||||
except ValueError as e:
|
||||
error_message = str(e)
|
||||
if "does not have a slow version" in error_message:
|
||||
logger.info(
|
||||
f"Processor {server_args.tokenizer_path} does not have a slow version. Automatically use fast version"
|
||||
)
|
||||
_processor = get_processor(
|
||||
server_args.tokenizer_path,
|
||||
tokenizer_mode=server_args.tokenizer_mode,
|
||||
trust_remote_code=server_args.trust_remote_code,
|
||||
revision=server_args.revision,
|
||||
use_fast=True,
|
||||
)
|
||||
else:
|
||||
raise e
|
||||
self.mm_processor = get_mm_processor(
|
||||
hf_config, server_args, _processor, transport_mode
|
||||
)
|
||||
|
||||
# For zmq_to_scheduler
|
||||
def process_waiting_requests(self, recv_reqs):
|
||||
new_recv_reqs = []
|
||||
for recv_req in recv_reqs:
|
||||
# E Disaggregation
|
||||
if (
|
||||
isinstance(recv_req, TokenizedGenerateReqInput)
|
||||
and recv_req.need_wait_for_image is True
|
||||
):
|
||||
embedding_port = None
|
||||
if recv_req.embedding_ports is not None:
|
||||
embedding_port = recv_req.embedding_ports[
|
||||
self.tp_size * self.pp_rank + self.tp_rank
|
||||
]
|
||||
waiting_req = WaitingImageRequest(
|
||||
rid=recv_req.rid,
|
||||
recv_req=recv_req,
|
||||
mm_processor=self.mm_processor,
|
||||
encoder_urls=self.encode_urls,
|
||||
host_name=self.hostname,
|
||||
receive_count=self.world_size,
|
||||
embedding_port=embedding_port,
|
||||
)
|
||||
if recv_req.embedding_ports is None:
|
||||
waiting_req.send_encode_request()
|
||||
self.waiting_list.append(waiting_req)
|
||||
else:
|
||||
new_recv_reqs.append(recv_req)
|
||||
|
||||
if len(self.waiting_list) == 0:
|
||||
return new_recv_reqs
|
||||
|
||||
local_status = []
|
||||
for waiting_req in self.waiting_list:
|
||||
waiting_req._try_recv_mm_data()
|
||||
local_status.append(waiting_req.ready)
|
||||
|
||||
local_status = torch.tensor(local_status, device="cuda", dtype=torch.int32)
|
||||
|
||||
torch.distributed.all_reduce(local_status, op=torch.distributed.ReduceOp.MIN)
|
||||
|
||||
new_waiting = []
|
||||
for i, waiting_req in enumerate(self.waiting_list):
|
||||
if local_status[i].item():
|
||||
new_recv_reqs.append(waiting_req.recv_req)
|
||||
else:
|
||||
new_waiting.append(waiting_req)
|
||||
|
||||
self.waiting_list = new_waiting
|
||||
return new_recv_reqs
|
||||
|
||||
# For zmq_to_scheduler
|
||||
def _run_encode_in_thread(
|
||||
self, req_id, img_data, endpoint_encode, num_items_assigned, embedding_port
|
||||
):
|
||||
try:
|
||||
asyncio.run(
|
||||
self.encode(
|
||||
req_id=req_id,
|
||||
img_data=img_data,
|
||||
embedding_port=embedding_port,
|
||||
endpoint_encode=endpoint_encode,
|
||||
endpoint_send=None,
|
||||
num_items_assigned=num_items_assigned,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Encode failed for request {req_id}: {e}", exc_info=True)
|
||||
|
||||
async def encode(
|
||||
self,
|
||||
req_id,
|
||||
img_data,
|
||||
embedding_port,
|
||||
endpoint_encode,
|
||||
endpoint_send,
|
||||
num_items_assigned=None,
|
||||
):
|
||||
if len(img_data) == 0:
|
||||
return
|
||||
|
||||
# Split mm_items
|
||||
encode_requests = []
|
||||
if num_items_assigned is None:
|
||||
random.shuffle(self.encode_idx)
|
||||
num_items_assigned = [
|
||||
(idx + len(img_data)) // len(self.encode_urls)
|
||||
for idx in self.encode_idx
|
||||
]
|
||||
num_parts = sum(1 for x in num_items_assigned if x != 0)
|
||||
cum_num_items = 0
|
||||
cum_idx = 0
|
||||
for idx, assigned_num in enumerate(num_items_assigned):
|
||||
if assigned_num == 0:
|
||||
continue
|
||||
encode_requests.append(
|
||||
{
|
||||
"encoder_idx": idx,
|
||||
"mm_items": img_data[cum_num_items : cum_num_items + assigned_num],
|
||||
"num_parts": num_parts,
|
||||
"part_idx": cum_idx,
|
||||
"req_id": req_id,
|
||||
"prefill_host": self.host,
|
||||
"embedding_port": embedding_port,
|
||||
}
|
||||
)
|
||||
cum_idx += 1
|
||||
cum_num_items += assigned_num
|
||||
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(
|
||||
total=1800
|
||||
) # Add timeout for request reliability
|
||||
) as session:
|
||||
# Send encode requests
|
||||
|
||||
tasks = [
|
||||
session.post(
|
||||
f"{self.encode_urls[encode_request['encoder_idx']]}/{endpoint_encode}",
|
||||
json=encode_request,
|
||||
)
|
||||
for encode_request in encode_requests
|
||||
]
|
||||
|
||||
responses = await asyncio.gather(*tasks)
|
||||
response_json_list_unsort = [
|
||||
await response.json() for response in responses
|
||||
]
|
||||
|
||||
# zmq backend: return is None
|
||||
if None in response_json_list_unsort:
|
||||
return
|
||||
|
||||
# mooncake backend: send bootstrap info
|
||||
|
||||
embedding_size_list_sort = [None for _ in range(num_parts)]
|
||||
embedding_length_tot = 0
|
||||
response_json_list_sort = [None for _ in range(num_parts)]
|
||||
for response_json in response_json_list_unsort:
|
||||
idx = response_json["part_idx"]
|
||||
embedding_size_list_sort[idx] = response_json["embedding_size"]
|
||||
embedding_length_tot += response_json["embedding_len"]
|
||||
response_json_list_sort[idx] = response_json
|
||||
|
||||
offset = 0
|
||||
metadata_tasks = []
|
||||
buffer_address = await self.allocate_embedding_buffer(
|
||||
req_id,
|
||||
embedding_length_tot,
|
||||
response_json_list_sort[0]["embedding_dim"],
|
||||
)
|
||||
for idx in range(len(tasks)):
|
||||
response_json = response_json_list_sort[idx]
|
||||
buffer_address_adjust = offset + buffer_address
|
||||
response_json.update(
|
||||
{
|
||||
"session_id": self.embeddings_engine.session_id,
|
||||
"buffer_address": buffer_address_adjust,
|
||||
}
|
||||
)
|
||||
metadata_tasks.append(
|
||||
session.post(
|
||||
f"{self.encode_urls[response_json['encoder_idx']]}/{endpoint_send}",
|
||||
json=response_json,
|
||||
)
|
||||
)
|
||||
offset += embedding_size_list_sort[idx]
|
||||
await asyncio.gather(*metadata_tasks)
|
||||
|
||||
# For mooncake
|
||||
async def allocate_embedding_buffer(self, req_id, embedding_length, embedding_dim):
|
||||
embeddings = torch.zeros(
|
||||
(embedding_length, embedding_dim),
|
||||
dtype=self.dtype,
|
||||
)
|
||||
self.embeddings_engine.register(
|
||||
embeddings.data_ptr(),
|
||||
embeddings.nbytes,
|
||||
)
|
||||
self.embeddings_buffer[req_id] = embeddings
|
||||
return embeddings.data_ptr()
|
||||
|
||||
# For zmq_to_scheduler
|
||||
def send_encode_request(self, obj):
|
||||
if type(obj.image_data) != list:
|
||||
image_urls = [obj.image_data.url]
|
||||
else:
|
||||
image_urls = [img.url for img in obj.image_data]
|
||||
if obj.rid is None:
|
||||
obj.rid = uuid.uuid4().hex
|
||||
if image_urls and len(image_urls) > 0:
|
||||
logger.info(f"Processing {len(image_urls)} images for request {obj.rid}")
|
||||
obj.need_wait_for_image = True
|
||||
|
||||
encode_idx = list(range(len(self.encode_urls)))
|
||||
random.shuffle(encode_idx)
|
||||
obj.num_items_assigned = [
|
||||
(idx + len(image_urls)) // len(self.encode_urls) for idx in encode_idx
|
||||
]
|
||||
obj.embedding_ports = None
|
||||
encode_thread = threading.Thread(
|
||||
target=self._run_encode_in_thread,
|
||||
args=(
|
||||
obj.rid,
|
||||
image_urls,
|
||||
"encode",
|
||||
obj.num_items_assigned,
|
||||
obj.embedding_ports,
|
||||
),
|
||||
daemon=True,
|
||||
)
|
||||
encode_thread.start()
|
||||
|
||||
# For zmq_to_tokenizer and mooncake
|
||||
async def recv_mm_data(self, img_data, mm_processor, prompt):
|
||||
try:
|
||||
if len(self.encode_urls) == 0:
|
||||
return None
|
||||
req_id = uuid.uuid4().hex
|
||||
embedding_port, recv_socket = get_zmq_socket_on_host(self.context, zmq.PULL)
|
||||
if type(img_data) != list:
|
||||
img_data = [img_data.url]
|
||||
else:
|
||||
img_data = [img.url for img in img_data]
|
||||
asyncio.create_task(
|
||||
self.encode(req_id, img_data, embedding_port, "encode", "send")
|
||||
)
|
||||
return await asyncio.wait_for(
|
||||
self._recv_mm_data(req_id, recv_socket, mm_processor, prompt),
|
||||
timeout=20,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(f"Embedding recv timeout for request {req_id}")
|
||||
if hasattr(self, "embeddings_buffer") and req_id in self.embeddings_buffer:
|
||||
del self.embeddings_buffer[req_id]
|
||||
return None
|
||||
|
||||
# For zmq_to_tokenizer and mooncake
|
||||
async def _recv_mm_data(self, req_id, recv_socket, mm_processor, prompt):
|
||||
# Bypass MMReceiver
|
||||
if req_id is None:
|
||||
return None
|
||||
|
||||
recv_embedding = None
|
||||
|
||||
recv_embedding_data: EmbeddingData = None
|
||||
|
||||
while recv_embedding_data is None or not recv_embedding_data.ready:
|
||||
parts = await recv_socket.recv_multipart(copy=False)
|
||||
|
||||
recv_obj: EmbeddingData = pickle.loads(parts[0])
|
||||
logger.info(f"{recv_obj = }")
|
||||
if self.encoder_transfer_backend == "zmq_to_tokenizer":
|
||||
buffer = parts[1].buffer if hasattr(parts[1], "buffer") else parts[1]
|
||||
recv_obj.embedding = torch.frombuffer(
|
||||
buffer, dtype=recv_obj.dtype
|
||||
).reshape(recv_obj.shape)
|
||||
if recv_embedding_data is None:
|
||||
recv_obj.embedding_list[recv_obj.part_idx] = recv_obj.embedding
|
||||
recv_embedding_data = recv_obj
|
||||
else:
|
||||
recv_embedding_data.add(recv_obj)
|
||||
|
||||
if self.encoder_transfer_backend == "mooncake":
|
||||
recv_embedding = self.embeddings_buffer[req_id]
|
||||
del self.embeddings_buffer[req_id]
|
||||
self.embeddings_engine.deregister(recv_embedding.data_ptr())
|
||||
elif self.encoder_transfer_backend == "zmq_to_tokenizer":
|
||||
recv_embedding = recv_embedding_data.get_embedding(is_concat=True)
|
||||
|
||||
recv_socket.close()
|
||||
|
||||
img_grid_thw = recv_embedding_data.get_img_grid()
|
||||
|
||||
mm_inputs = mm_processor.get_mm_data(prompt, recv_embedding, img_grid_thw)
|
||||
return mm_inputs
|
||||
@@ -0,0 +1,527 @@
|
||||
import asyncio
|
||||
import ctypes
|
||||
import logging
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
import traceback
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import torch
|
||||
import uvicorn
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import ORJSONResponse, Response
|
||||
from transformers import AutoImageProcessor
|
||||
from transformers.image_utils import load_images
|
||||
|
||||
from sglang.srt.configs.device_config import DeviceConfig
|
||||
from sglang.srt.configs.load_config import LoadConfig
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.disaggregation.encode_receiver import EmbeddingData
|
||||
from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferEngine
|
||||
from sglang.srt.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention
|
||||
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import MultiModalStaticCache
|
||||
from sglang.srt.model_loader import get_model
|
||||
from sglang.srt.server_args import (
|
||||
PortArgs,
|
||||
ServerArgs,
|
||||
set_global_server_args_for_scheduler,
|
||||
)
|
||||
from sglang.srt.utils import get_local_ip_auto, get_zmq_socket, random_uuid
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
rid_lock = asyncio.Lock()
|
||||
rid_to_receive_endpoint: Dict[str, List[str]] = dict()
|
||||
rid_to_receive_count: Dict[str, int] = dict()
|
||||
|
||||
|
||||
class TensorWrapper:
|
||||
"""Wrapper to keep tensor alive while exposing buffer for zero-copy."""
|
||||
|
||||
def __init__(self, tensor):
|
||||
# Ensure tensor is on CPU and contiguous
|
||||
if tensor.is_cuda:
|
||||
tensor = tensor.cpu()
|
||||
if not tensor.is_contiguous():
|
||||
tensor = tensor.contiguous()
|
||||
|
||||
# Keep tensor reference
|
||||
self.tensor = tensor
|
||||
self.shape = list(tensor.shape)
|
||||
self.dtype = tensor.dtype
|
||||
|
||||
def __buffer__(self):
|
||||
data_ptr = self.tensor.data_ptr()
|
||||
total_bytes = self.tensor.numel() * self.tensor.element_size()
|
||||
c_obj = (ctypes.c_char * total_bytes).from_address(data_ptr)
|
||||
c_obj._keep_alive_ref = self
|
||||
return memoryview(c_obj)
|
||||
|
||||
|
||||
def _convert(data):
|
||||
if isinstance(data, torch.Tensor):
|
||||
return data
|
||||
elif isinstance(data, np.ndarray):
|
||||
return torch.tensor(data)
|
||||
elif isinstance(data, list) and isinstance(data[0], np.ndarray):
|
||||
return torch.tensor(np.array(data))
|
||||
elif isinstance(data, list) and isinstance(data[0], (int, float)):
|
||||
return torch.tensor(data)
|
||||
else:
|
||||
return data
|
||||
|
||||
|
||||
_image_grid_attrs = ["image_grid_thw", "image_grid_hws"]
|
||||
|
||||
|
||||
def _get_image_grid_dim(images_input):
|
||||
for attr in _image_grid_attrs:
|
||||
if attr in images_input:
|
||||
return images_input[attr]
|
||||
raise ValueError(
|
||||
f"Image grid dim ({_image_grid_attrs}) not found in {images_input}"
|
||||
)
|
||||
|
||||
|
||||
class MMEncoder:
|
||||
def __init__(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
schedule_path=None,
|
||||
dist_init_method=None,
|
||||
rank: int = 0,
|
||||
):
|
||||
logger.info(f"init MMEncoder {rank}/{server_args.tp_size}")
|
||||
self.server_args = server_args
|
||||
set_global_server_args_for_scheduler(server_args)
|
||||
self.rank = rank
|
||||
|
||||
self.image_processor = AutoImageProcessor.from_pretrained(
|
||||
server_args.model_path,
|
||||
trust_remote_code=server_args.trust_remote_code,
|
||||
use_fast=True,
|
||||
)
|
||||
|
||||
self.model_config = ModelConfig.from_server_args(
|
||||
server_args,
|
||||
)
|
||||
|
||||
self.load_config = LoadConfig(
|
||||
load_format=server_args.load_format,
|
||||
download_dir=server_args.download_dir,
|
||||
model_loader_extra_config=server_args.model_loader_extra_config,
|
||||
remote_instance_weight_loader_seed_instance_ip=server_args.remote_instance_weight_loader_seed_instance_ip,
|
||||
remote_instance_weight_loader_seed_instance_service_port=server_args.remote_instance_weight_loader_seed_instance_service_port,
|
||||
remote_instance_weight_loader_send_weights_group_ports=server_args.remote_instance_weight_loader_send_weights_group_ports,
|
||||
)
|
||||
|
||||
self.device = server_args.device
|
||||
self.gpu_id = server_args.base_gpu_id + rank
|
||||
|
||||
self.device_config = DeviceConfig(
|
||||
device=self.device,
|
||||
gpu_id=self.gpu_id,
|
||||
)
|
||||
|
||||
torch.get_device_module(self.device).set_device(self.gpu_id)
|
||||
|
||||
init_distributed_environment(
|
||||
world_size=server_args.tp_size,
|
||||
rank=rank,
|
||||
distributed_init_method=dist_init_method,
|
||||
local_rank=rank,
|
||||
)
|
||||
initialize_model_parallel(tensor_model_parallel_size=server_args.tp_size)
|
||||
initialize_dp_attention(server_args, self.model_config)
|
||||
|
||||
self.model = get_model(
|
||||
model_config=self.model_config,
|
||||
load_config=self.load_config,
|
||||
device_config=self.device_config,
|
||||
)
|
||||
|
||||
self.context = zmq.asyncio.Context(2)
|
||||
|
||||
embedding_cache_size = int(os.environ.get("SGLANG_VLM_CACHE_SIZE_MB", "4096"))
|
||||
self.mm_cache = MultiModalStaticCache(embedding_cache_size * 1024 * 1024)
|
||||
self.mm_cache_lock = asyncio.Lock()
|
||||
|
||||
if schedule_path is not None:
|
||||
self.schedule_socket = get_zmq_socket(
|
||||
self.context, zmq.PULL, schedule_path, True
|
||||
)
|
||||
|
||||
if self.rank == 0:
|
||||
logger.info(
|
||||
f"Using transfer backend: {self.server_args.encoder_transfer_backend}"
|
||||
)
|
||||
|
||||
if self.server_args.encoder_transfer_backend == "mooncake":
|
||||
self.local_ip = get_local_ip_auto()
|
||||
|
||||
self.engine = MooncakeTransferEngine(
|
||||
hostname=self.local_ip,
|
||||
gpu_id=None,
|
||||
ib_device=server_args.disaggregation_ib_device,
|
||||
)
|
||||
|
||||
self.embedding_to_send = dict()
|
||||
|
||||
logger.info(f"rank {rank} init finish ")
|
||||
|
||||
async def _encode(self, mm_items) -> torch.Tensor:
|
||||
images = load_images(mm_items)
|
||||
|
||||
images_input = self.image_processor(images=images)
|
||||
feature = images_input["pixel_values"]
|
||||
mm_item = MultimodalDataItem.from_dict(
|
||||
{
|
||||
"modality": Modality.IMAGE,
|
||||
"feature": _convert(feature),
|
||||
}
|
||||
)
|
||||
for k, v in images_input.items():
|
||||
if k == "pixel_values":
|
||||
continue
|
||||
mm_item.set(k, _convert(v))
|
||||
|
||||
# support mm_cache
|
||||
mm_embedding = None
|
||||
mm_hash = None
|
||||
|
||||
start_time = time.perf_counter()
|
||||
if self.server_args.enable_prefix_mm_cache:
|
||||
mm_item.set_pad_value()
|
||||
mm_hash = MultiModalStaticCache.combine_hashes([mm_item.hash])
|
||||
async with self.mm_cache_lock:
|
||||
mm_cache = self.mm_cache.get([mm_item.hash])
|
||||
if mm_cache is not None:
|
||||
mm_embedding = mm_cache
|
||||
|
||||
if mm_embedding is None:
|
||||
with torch.inference_mode():
|
||||
mm_embedding: torch.Tensor = self.model.get_image_feature([mm_item])
|
||||
mm_embedding = mm_embedding.cpu()
|
||||
if len(mm_embedding.shape) != 2:
|
||||
mm_embedding = mm_embedding.reshape(-1, mm_embedding.shape[-1])
|
||||
|
||||
if self.server_args.enable_prefix_mm_cache:
|
||||
async with self.mm_cache_lock:
|
||||
self.mm_cache.set(mm_hash, mm_embedding)
|
||||
end_time = time.perf_counter()
|
||||
logger.info(
|
||||
f"Vit time : {(end_time - start_time)*1000:.2f} ms {mm_embedding.shape = }"
|
||||
)
|
||||
|
||||
return _get_image_grid_dim(images_input), mm_embedding
|
||||
|
||||
async def _send(
|
||||
self,
|
||||
embedding: torch.Tensor,
|
||||
mm_data: EmbeddingData,
|
||||
session_id=None,
|
||||
buffer_address=None,
|
||||
prefill_host=None,
|
||||
embedding_port=None,
|
||||
url=None,
|
||||
):
|
||||
if self.server_args.encoder_transfer_backend == "mooncake":
|
||||
self.engine.register(embedding.data_ptr(), embedding.nbytes)
|
||||
self.engine.transfer_sync(
|
||||
session_id, embedding.data_ptr(), buffer_address, embedding.nbytes
|
||||
)
|
||||
self.engine.deregister(embedding.data_ptr())
|
||||
|
||||
mm_data.embedding = None
|
||||
mm_data.embedding_list[mm_data.part_idx] = None
|
||||
|
||||
# Send ack/data
|
||||
endpoint = (
|
||||
f"tcp://{url}"
|
||||
if url is not None
|
||||
else f"tcp://{prefill_host}:{embedding_port}"
|
||||
)
|
||||
logger.info(f"{endpoint = }")
|
||||
socket = get_zmq_socket(
|
||||
self.context,
|
||||
zmq.PUSH,
|
||||
endpoint,
|
||||
False,
|
||||
)
|
||||
|
||||
if self.server_args.encoder_transfer_backend == "mooncake":
|
||||
socket.send_multipart([pickle.dumps(mm_data)])
|
||||
else:
|
||||
new_mm_data = mm_data.copy_without_embedding()
|
||||
embedding_tensor = TensorWrapper(mm_data.embedding)
|
||||
socket.send_multipart(
|
||||
[pickle.dumps(new_mm_data), embedding_tensor.__buffer__()]
|
||||
)
|
||||
|
||||
async def encode(self, mm_items, req_id, num_parts, part_idx):
|
||||
start_time = time.time()
|
||||
image_grid_dim, mm_embedding = await self._encode(mm_items)
|
||||
end_time = time.time()
|
||||
logger.info(f"🕛 encode cost = {(end_time - start_time) * 1000:.2f}ms")
|
||||
if self.rank == 0:
|
||||
mm_data = EmbeddingData(
|
||||
req_id,
|
||||
num_parts,
|
||||
part_idx,
|
||||
image_grid_dim,
|
||||
mm_embedding,
|
||||
)
|
||||
self.embedding_to_send[mm_data.req_id] = mm_data
|
||||
return mm_embedding.nbytes, mm_embedding.shape[0], mm_embedding.shape[1]
|
||||
|
||||
# For zmq_to_tokenizer zmq_to_scheduler and mooncake
|
||||
async def send(
|
||||
self, req_id, prefill_host, embedding_port, session_id=None, buffer_address=None
|
||||
):
|
||||
mm_data: EmbeddingData = self.embedding_to_send[req_id]
|
||||
await self._send(
|
||||
mm_data.embedding,
|
||||
mm_data,
|
||||
session_id=session_id,
|
||||
buffer_address=buffer_address,
|
||||
prefill_host=prefill_host,
|
||||
embedding_port=embedding_port,
|
||||
)
|
||||
|
||||
# For zmq_to_scheduler
|
||||
async def send_with_url(
|
||||
self,
|
||||
req_id,
|
||||
):
|
||||
mm_data = self.embedding_to_send.get(req_id)
|
||||
if not mm_data:
|
||||
return
|
||||
sent_urls: Set[str] = set()
|
||||
all_tasks: List[Tuple[asyncio.Task, str]] = []
|
||||
start_time = asyncio.get_running_loop().time()
|
||||
timeout = 60.0
|
||||
|
||||
try:
|
||||
while True:
|
||||
async with rid_lock:
|
||||
current_targets = rid_to_receive_endpoint.get(req_id, set()).copy()
|
||||
expected_count = rid_to_receive_count.get(req_id)
|
||||
|
||||
new_targets = current_targets - sent_urls
|
||||
|
||||
if new_targets:
|
||||
logger.info(
|
||||
f"Found {len(new_targets)} new endpoints for {req_id}. Starting tasks..."
|
||||
)
|
||||
for url in new_targets:
|
||||
task = asyncio.create_task(
|
||||
self._send(
|
||||
mm_data.embedding,
|
||||
mm_data,
|
||||
url=url,
|
||||
)
|
||||
)
|
||||
all_tasks.append((task, url))
|
||||
sent_urls.add(url) # Mark as handled immediately
|
||||
if expected_count is not None and len(sent_urls) >= expected_count:
|
||||
logger.info(
|
||||
f"All {expected_count} endpoints initiated for {req_id}. Breaking loop."
|
||||
)
|
||||
break
|
||||
|
||||
if asyncio.get_running_loop().time() - start_time > timeout:
|
||||
logger.error(
|
||||
f"Timeout waiting for all endpoints for {req_id}. Initiated {len(sent_urls)}/{expected_count}"
|
||||
)
|
||||
break
|
||||
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
if all_tasks:
|
||||
logger.info(
|
||||
f"Loop finished. Awaiting completion of {len(all_tasks)} sending tasks..."
|
||||
)
|
||||
tasks_only = [t[0] for t in all_tasks]
|
||||
results = await asyncio.gather(*tasks_only, return_exceptions=True)
|
||||
|
||||
# Process results and log errors
|
||||
for i, result in enumerate(results):
|
||||
url = all_tasks[i][1] # Retrieve URL associated with the task
|
||||
if isinstance(result, Exception):
|
||||
logger.error(f"Failed to send to {url}: {result}")
|
||||
else:
|
||||
logger.debug(f"Successfully sent to {url}")
|
||||
|
||||
logger.info(f"All tasks completed for req_id: {req_id}")
|
||||
|
||||
finally:
|
||||
logger.info(f"Cleaning up resources for req_id {req_id}")
|
||||
async with rid_lock:
|
||||
rid_to_receive_endpoint.pop(req_id, None)
|
||||
rid_to_receive_count.pop(req_id, None)
|
||||
self.embedding_to_send.pop(req_id, None)
|
||||
|
||||
async def get_embedding_port(self, prefill_url):
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=1800)
|
||||
) as session:
|
||||
response = await session.post(
|
||||
f"{prefill_url}/embedding_bootstrap",
|
||||
json={"embedding_port": None},
|
||||
)
|
||||
response_json = await response.json()
|
||||
return response_json["embedding_port"]
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
encoder: Optional[MMEncoder] = None
|
||||
send_sockets: List[zmq.Socket] = []
|
||||
|
||||
|
||||
async def run_encoder(
|
||||
server_args: ServerArgs, schedule_path, dist_init_method, rank: int
|
||||
):
|
||||
encoder = MMEncoder(server_args, schedule_path, dist_init_method, rank)
|
||||
while True:
|
||||
request = await encoder.schedule_socket.recv_pyobj()
|
||||
await encoder.encode(
|
||||
mm_items=request["mm_items"],
|
||||
req_id=request["req_id"],
|
||||
num_parts=request["num_parts"],
|
||||
part_idx=request["part_idx"],
|
||||
)
|
||||
|
||||
|
||||
def launch_encoder(server_args, schedule_path, dist_init_method, rank):
|
||||
try:
|
||||
asyncio.run(run_encoder(server_args, schedule_path, dist_init_method, rank))
|
||||
except KeyboardInterrupt:
|
||||
logger.info(f"Exit rank {rank}")
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def launch_server(server_args: ServerArgs):
|
||||
global encoder
|
||||
ctx = mp.get_context("spawn")
|
||||
zmq_ctx = zmq.Context(10)
|
||||
ipc_path_prefix = random_uuid()
|
||||
port_args = PortArgs.init_new(server_args)
|
||||
if server_args.dist_init_addr:
|
||||
dist_init_method = f"tcp://{server_args.dist_init_addr}"
|
||||
else:
|
||||
dist_init_method = f"tcp://127.0.0.1:{port_args.nccl_port}"
|
||||
for rank in range(1, server_args.tp_size):
|
||||
schedule_path = f"ipc:///tmp/{ipc_path_prefix}_schedule_{rank}"
|
||||
send_sockets.append(
|
||||
get_zmq_socket(zmq_ctx, zmq.PUSH, schedule_path, bind=False)
|
||||
)
|
||||
ctx.Process(
|
||||
target=launch_encoder,
|
||||
args=(server_args, schedule_path, dist_init_method, rank),
|
||||
daemon=True,
|
||||
).start()
|
||||
encoder = MMEncoder(server_args, dist_init_method=dist_init_method)
|
||||
uvicorn.run(app, host=server_args.host, port=server_args.port)
|
||||
|
||||
|
||||
@app.post("/encode")
|
||||
async def handle_encode_request(request: dict):
|
||||
# broadcast request
|
||||
request.update({"enter_time": time.time()})
|
||||
for socket in send_sockets:
|
||||
socket.send_pyobj(request)
|
||||
|
||||
nbytes, embedding_len, embedding_dim = await encoder.encode(
|
||||
mm_items=request["mm_items"],
|
||||
req_id=request["req_id"],
|
||||
num_parts=request["num_parts"],
|
||||
part_idx=request["part_idx"],
|
||||
)
|
||||
if encoder.server_args.encoder_transfer_backend == "mooncake":
|
||||
del request["mm_items"]
|
||||
request.update(
|
||||
{
|
||||
"embedding_size": nbytes,
|
||||
"embedding_len": embedding_len,
|
||||
"embedding_dim": embedding_dim,
|
||||
}
|
||||
)
|
||||
return ORJSONResponse(content=request)
|
||||
elif encoder.server_args.encoder_transfer_backend == "zmq_to_scheduler":
|
||||
logger.info(f"{request['embedding_port'] = }")
|
||||
if request["embedding_port"] is None:
|
||||
await encoder.send_with_url(
|
||||
req_id=request["req_id"],
|
||||
)
|
||||
else:
|
||||
assert type(request["embedding_port"]) == list
|
||||
tasks = []
|
||||
for embedding_port in request["embedding_port"]:
|
||||
tasks.append(
|
||||
encoder.send(
|
||||
req_id=request["req_id"],
|
||||
prefill_host=request["prefill_host"],
|
||||
embedding_port=embedding_port,
|
||||
)
|
||||
)
|
||||
await asyncio.gather(*tasks)
|
||||
encoder.embedding_to_send.pop(request["req_id"], None)
|
||||
return ORJSONResponse(content=None)
|
||||
elif encoder.server_args.encoder_transfer_backend == "zmq_to_tokenizer":
|
||||
await encoder.send(
|
||||
req_id=request["req_id"],
|
||||
prefill_host=request["prefill_host"],
|
||||
embedding_port=request["embedding_port"],
|
||||
)
|
||||
encoder.embedding_to_send.pop(request["req_id"], None)
|
||||
return ORJSONResponse(content=None)
|
||||
|
||||
|
||||
@app.post("/send")
|
||||
async def handle_send_request(request: dict):
|
||||
# mooncake backend
|
||||
await encoder.send(
|
||||
req_id=request["req_id"],
|
||||
prefill_host=request["prefill_host"],
|
||||
embedding_port=request["embedding_port"],
|
||||
session_id=request["session_id"],
|
||||
buffer_address=request["buffer_address"],
|
||||
)
|
||||
encoder.embedding_to_send.pop(request["req_id"], None)
|
||||
return ORJSONResponse(content=None)
|
||||
|
||||
|
||||
@app.post("/scheduler_receive_url")
|
||||
async def handle_scheduler_receive_url_request(request: dict):
|
||||
rid = request["req_id"]
|
||||
async with rid_lock:
|
||||
global rid_to_receive_endpoint
|
||||
if rid not in rid_to_receive_endpoint:
|
||||
rid_to_receive_endpoint[rid] = set()
|
||||
rid_to_receive_count[rid] = request["receive_count"]
|
||||
assert rid_to_receive_count[rid] == request["receive_count"]
|
||||
rid_to_receive_endpoint[rid].add(request["receive_url"])
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
@app.get("/health_generate")
|
||||
async def health_generate():
|
||||
"""
|
||||
Health check endpoint for the encoder server.
|
||||
Returns 200 if the encoder is initialized and ready.
|
||||
"""
|
||||
if encoder is None:
|
||||
return Response(status_code=503)
|
||||
return Response(status_code=200)
|
||||
Reference in New Issue
Block a user