feat: support EPD disaggregation (#12263)

Co-authored-by: liusy58 <liusy58@linux.alibaba.com>
Co-authored-by: ZhengWG <zwg0606@gmail.com>
Co-authored-by: Nicholas <45984215+liusy58@users.noreply.github.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com>
This commit is contained in:
Tianyu Guo
2025-12-14 22:30:08 +08:00
committed by GitHub
co-authored by liusy58 ZhengWG Nicholas Shangming Cai Yuhao Yang
parent a9ce1623cd
commit 9acb21ae27
19 changed files with 1910 additions and 68 deletions
@@ -0,0 +1,544 @@
import asyncio
import logging
import pickle
import random
import threading
import uuid
from typing import List
import aiohttp
import torch
import zmq
import zmq.asyncio
from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferEngine
from sglang.srt.managers.io_struct import TokenizedGenerateReqInput
from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import get_local_ip_auto, get_zmq_socket_on_host
from sglang.srt.utils.hf_transformers_utils import get_processor
logger = logging.getLogger(__name__)
class EmbeddingData:
def __init__(self, req_id, num_parts, part_idx, image_grid_dim, embedding=None):
self.req_id = req_id
self.num_parts = num_parts
self.part_idx = part_idx
self.image_grid_dim = image_grid_dim
self.embedding = embedding
self.send_time = None
self.dtype = embedding.dtype if embedding is not None else None
self.shape = list(embedding.shape) if embedding is not None else None
# aggregated data
self.ready_list = [i == self.part_idx for i in range(self.num_parts)]
self.embedding_list = [
embedding if i == self.part_idx else None for i in range(self.num_parts)
]
self.image_grid_dim_list = [
self.image_grid_dim if i == self.part_idx else None
for i in range(self.num_parts)
]
def add(self, embedding_data):
assert self.req_id == embedding_data.req_id
assert not self.ready_list[embedding_data.part_idx]
self.ready_list[embedding_data.part_idx] = True
self.image_grid_dim_list[embedding_data.part_idx] = (
embedding_data.image_grid_dim
)
self.embedding_list[embedding_data.part_idx] = embedding_data.embedding
def get_embedding(self, is_concat=False):
if is_concat:
return torch.concat([embedding.cuda() for embedding in self.embedding_list])
else:
return self.embedding_list
def get_img_grid(self):
return torch.concatenate(self.image_grid_dim_list)
@property
def ready(self):
return sum(self.ready_list) == self.num_parts
def __repr__(self):
return f"EmbeddingData(req_id={self.req_id}, num_parts={self.num_parts}, part_idx={self.part_idx})"
def copy_without_embedding(self):
new_data = EmbeddingData(
req_id=self.req_id,
num_parts=self.num_parts,
part_idx=self.part_idx,
image_grid_dim=self.image_grid_dim,
)
new_data.send_time = self.send_time
new_data.dtype = self.dtype
new_data.shape = self.shape
return new_data
# For zmq_to_scheduler
class WaitingImageRequest:
def __init__(
self,
rid: str,
recv_req: TokenizedGenerateReqInput,
mm_processor,
encoder_urls,
host_name,
receive_count,
embedding_port=None,
):
self.rid = rid
self.recv_req = recv_req
self.mm_inputs = None
self.error = None
self.thread = None
self.mm_processor = mm_processor
self.encoder_urls = encoder_urls
self.host_name = host_name
self.receive_count = receive_count
self.num_items_assigned = recv_req.num_items_assigned
self.embedding_port, self.recv_socket = get_zmq_socket_on_host(
zmq.Context(), zmq.PULL
)
logger.info(f"Waiting for input {self.embedding_port = }")
self.recv_embedding_data = None
self.ready = False
def send_encode_request(self):
async def _send_single_request(session, url, payload):
try:
async with session.post(url, json=payload) as response:
response.raise_for_status()
return await response.text()
except Exception as e:
logger.error(f"Failed to send request to {url}: {e}")
raise
async def send_embedding_port(req_id, receive_count, host_name, embedding_port):
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=1800)
) as session:
tasks = []
logger.info(f"{self.num_items_assigned = } ")
for idx, assigned_num in enumerate(self.num_items_assigned):
if assigned_num == 0:
continue
encoder_url = self.encoder_urls[idx]
target_url = f"{encoder_url}/scheduler_receive_url"
payload = {
"req_id": req_id,
"receive_count": receive_count,
"receive_url": f"{host_name}:{embedding_port}",
}
logger.info(f"Preparing to send to {target_url}")
task = _send_single_request(session, target_url, payload)
tasks.append(task)
if not tasks:
logger.info("No tasks to send.")
return
logger.info(f"Concurrently sending {len(tasks)} requests...")
results = await asyncio.gather(*tasks, return_exceptions=True)
for i, result in enumerate(results):
if isinstance(result, Exception):
logger.error(f"Request {i} failed: {result}")
else:
logger.debug(f"Request {i} succeeded.")
asyncio.run(
send_embedding_port(
self.recv_req.rid,
self.receive_count,
self.host_name,
self.embedding_port,
)
)
def _try_recv_mm_data(self):
if self.ready:
return
while self.recv_embedding_data is None or not self.recv_embedding_data.ready:
try:
parts = self.recv_socket.recv_multipart(flags=zmq.NOBLOCK, copy=False)
except zmq.Again:
# No data available yet, wait a bit and retry
return
recv_obj: EmbeddingData = pickle.loads(parts[0])
buffer = parts[1].buffer if hasattr(parts[1], "buffer") else parts[1]
recv_obj.embedding = torch.frombuffer(buffer, dtype=recv_obj.dtype).reshape(
recv_obj.shape
)
recv_obj.embedding_list[recv_obj.part_idx] = recv_obj.embedding
if self.recv_embedding_data is None:
self.recv_embedding_data = recv_obj
else:
self.recv_embedding_data.add(recv_obj)
recv_embedding = self.recv_embedding_data.get_embedding(is_concat=True)
img_grid_thw = self.recv_embedding_data.get_img_grid()
mm_inputs = self.mm_processor.get_mm_data(
self.recv_req.input_text, recv_embedding, img_grid_thw
)
self.recv_req.mm_inputs = mm_inputs
self.recv_req.input_ids = mm_inputs["input_ids"]
self.ready = True
self.recv_socket.close()
def _determine_tensor_transport_mode(server_args):
is_cross_node = server_args.dist_init_addr
if is_cross_node:
# Fallback to default CPU transport for multi-node
return "default"
else:
return "cuda_ipc"
class MMReceiver:
def __init__(
self,
server_args: ServerArgs,
dtype=None,
hf_config=None,
pp_rank=None,
tp_rank=None,
):
self.context = zmq.asyncio.Context(20)
self.encoder_transfer_backend = server_args.encoder_transfer_backend
self.encode_urls = server_args.encoder_urls
self.encode_idx = list(range(len(self.encode_urls)))
self.host = server_args.host
if self.encoder_transfer_backend == "mooncake":
self.dtype = dtype
self.embeddings_engine = MooncakeTransferEngine(
hostname=get_local_ip_auto(),
gpu_id=None,
ib_device=server_args.disaggregation_ib_device,
)
self.embeddings_buffer = dict()
elif self.encoder_transfer_backend == "zmq_to_scheduler":
self.pp_rank = pp_rank
self.tp_rank = tp_rank
self.tp_size = server_args.tp_size
self.nnodes = server_args.nnodes
self.hostname = get_local_ip_auto()
self.world_size = server_args.pp_size * server_args.tp_size
self.waiting_list: List[WaitingImageRequest] = []
if hf_config is not None:
transport_mode = _determine_tensor_transport_mode(server_args)
import_processors("sglang.srt.multimodal.processors")
_processor = None
try:
_processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=not server_args.disable_fast_image_processor,
)
except ValueError as e:
error_message = str(e)
if "does not have a slow version" in error_message:
logger.info(
f"Processor {server_args.tokenizer_path} does not have a slow version. Automatically use fast version"
)
_processor = get_processor(
server_args.tokenizer_path,
tokenizer_mode=server_args.tokenizer_mode,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
use_fast=True,
)
else:
raise e
self.mm_processor = get_mm_processor(
hf_config, server_args, _processor, transport_mode
)
# For zmq_to_scheduler
def process_waiting_requests(self, recv_reqs):
new_recv_reqs = []
for recv_req in recv_reqs:
# E Disaggregation
if (
isinstance(recv_req, TokenizedGenerateReqInput)
and recv_req.need_wait_for_image is True
):
embedding_port = None
if recv_req.embedding_ports is not None:
embedding_port = recv_req.embedding_ports[
self.tp_size * self.pp_rank + self.tp_rank
]
waiting_req = WaitingImageRequest(
rid=recv_req.rid,
recv_req=recv_req,
mm_processor=self.mm_processor,
encoder_urls=self.encode_urls,
host_name=self.hostname,
receive_count=self.world_size,
embedding_port=embedding_port,
)
if recv_req.embedding_ports is None:
waiting_req.send_encode_request()
self.waiting_list.append(waiting_req)
else:
new_recv_reqs.append(recv_req)
if len(self.waiting_list) == 0:
return new_recv_reqs
local_status = []
for waiting_req in self.waiting_list:
waiting_req._try_recv_mm_data()
local_status.append(waiting_req.ready)
local_status = torch.tensor(local_status, device="cuda", dtype=torch.int32)
torch.distributed.all_reduce(local_status, op=torch.distributed.ReduceOp.MIN)
new_waiting = []
for i, waiting_req in enumerate(self.waiting_list):
if local_status[i].item():
new_recv_reqs.append(waiting_req.recv_req)
else:
new_waiting.append(waiting_req)
self.waiting_list = new_waiting
return new_recv_reqs
# For zmq_to_scheduler
def _run_encode_in_thread(
self, req_id, img_data, endpoint_encode, num_items_assigned, embedding_port
):
try:
asyncio.run(
self.encode(
req_id=req_id,
img_data=img_data,
embedding_port=embedding_port,
endpoint_encode=endpoint_encode,
endpoint_send=None,
num_items_assigned=num_items_assigned,
)
)
except Exception as e:
logger.error(f"Encode failed for request {req_id}: {e}", exc_info=True)
async def encode(
self,
req_id,
img_data,
embedding_port,
endpoint_encode,
endpoint_send,
num_items_assigned=None,
):
if len(img_data) == 0:
return
# Split mm_items
encode_requests = []
if num_items_assigned is None:
random.shuffle(self.encode_idx)
num_items_assigned = [
(idx + len(img_data)) // len(self.encode_urls)
for idx in self.encode_idx
]
num_parts = sum(1 for x in num_items_assigned if x != 0)
cum_num_items = 0
cum_idx = 0
for idx, assigned_num in enumerate(num_items_assigned):
if assigned_num == 0:
continue
encode_requests.append(
{
"encoder_idx": idx,
"mm_items": img_data[cum_num_items : cum_num_items + assigned_num],
"num_parts": num_parts,
"part_idx": cum_idx,
"req_id": req_id,
"prefill_host": self.host,
"embedding_port": embedding_port,
}
)
cum_idx += 1
cum_num_items += assigned_num
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(
total=1800
) # Add timeout for request reliability
) as session:
# Send encode requests
tasks = [
session.post(
f"{self.encode_urls[encode_request['encoder_idx']]}/{endpoint_encode}",
json=encode_request,
)
for encode_request in encode_requests
]
responses = await asyncio.gather(*tasks)
response_json_list_unsort = [
await response.json() for response in responses
]
# zmq backend: return is None
if None in response_json_list_unsort:
return
# mooncake backend: send bootstrap info
embedding_size_list_sort = [None for _ in range(num_parts)]
embedding_length_tot = 0
response_json_list_sort = [None for _ in range(num_parts)]
for response_json in response_json_list_unsort:
idx = response_json["part_idx"]
embedding_size_list_sort[idx] = response_json["embedding_size"]
embedding_length_tot += response_json["embedding_len"]
response_json_list_sort[idx] = response_json
offset = 0
metadata_tasks = []
buffer_address = await self.allocate_embedding_buffer(
req_id,
embedding_length_tot,
response_json_list_sort[0]["embedding_dim"],
)
for idx in range(len(tasks)):
response_json = response_json_list_sort[idx]
buffer_address_adjust = offset + buffer_address
response_json.update(
{
"session_id": self.embeddings_engine.session_id,
"buffer_address": buffer_address_adjust,
}
)
metadata_tasks.append(
session.post(
f"{self.encode_urls[response_json['encoder_idx']]}/{endpoint_send}",
json=response_json,
)
)
offset += embedding_size_list_sort[idx]
await asyncio.gather(*metadata_tasks)
# For mooncake
async def allocate_embedding_buffer(self, req_id, embedding_length, embedding_dim):
embeddings = torch.zeros(
(embedding_length, embedding_dim),
dtype=self.dtype,
)
self.embeddings_engine.register(
embeddings.data_ptr(),
embeddings.nbytes,
)
self.embeddings_buffer[req_id] = embeddings
return embeddings.data_ptr()
# For zmq_to_scheduler
def send_encode_request(self, obj):
if type(obj.image_data) != list:
image_urls = [obj.image_data.url]
else:
image_urls = [img.url for img in obj.image_data]
if obj.rid is None:
obj.rid = uuid.uuid4().hex
if image_urls and len(image_urls) > 0:
logger.info(f"Processing {len(image_urls)} images for request {obj.rid}")
obj.need_wait_for_image = True
encode_idx = list(range(len(self.encode_urls)))
random.shuffle(encode_idx)
obj.num_items_assigned = [
(idx + len(image_urls)) // len(self.encode_urls) for idx in encode_idx
]
obj.embedding_ports = None
encode_thread = threading.Thread(
target=self._run_encode_in_thread,
args=(
obj.rid,
image_urls,
"encode",
obj.num_items_assigned,
obj.embedding_ports,
),
daemon=True,
)
encode_thread.start()
# For zmq_to_tokenizer and mooncake
async def recv_mm_data(self, img_data, mm_processor, prompt):
try:
if len(self.encode_urls) == 0:
return None
req_id = uuid.uuid4().hex
embedding_port, recv_socket = get_zmq_socket_on_host(self.context, zmq.PULL)
if type(img_data) != list:
img_data = [img_data.url]
else:
img_data = [img.url for img in img_data]
asyncio.create_task(
self.encode(req_id, img_data, embedding_port, "encode", "send")
)
return await asyncio.wait_for(
self._recv_mm_data(req_id, recv_socket, mm_processor, prompt),
timeout=20,
)
except asyncio.TimeoutError:
logger.warning(f"Embedding recv timeout for request {req_id}")
if hasattr(self, "embeddings_buffer") and req_id in self.embeddings_buffer:
del self.embeddings_buffer[req_id]
return None
# For zmq_to_tokenizer and mooncake
async def _recv_mm_data(self, req_id, recv_socket, mm_processor, prompt):
# Bypass MMReceiver
if req_id is None:
return None
recv_embedding = None
recv_embedding_data: EmbeddingData = None
while recv_embedding_data is None or not recv_embedding_data.ready:
parts = await recv_socket.recv_multipart(copy=False)
recv_obj: EmbeddingData = pickle.loads(parts[0])
logger.info(f"{recv_obj = }")
if self.encoder_transfer_backend == "zmq_to_tokenizer":
buffer = parts[1].buffer if hasattr(parts[1], "buffer") else parts[1]
recv_obj.embedding = torch.frombuffer(
buffer, dtype=recv_obj.dtype
).reshape(recv_obj.shape)
if recv_embedding_data is None:
recv_obj.embedding_list[recv_obj.part_idx] = recv_obj.embedding
recv_embedding_data = recv_obj
else:
recv_embedding_data.add(recv_obj)
if self.encoder_transfer_backend == "mooncake":
recv_embedding = self.embeddings_buffer[req_id]
del self.embeddings_buffer[req_id]
self.embeddings_engine.deregister(recv_embedding.data_ptr())
elif self.encoder_transfer_backend == "zmq_to_tokenizer":
recv_embedding = recv_embedding_data.get_embedding(is_concat=True)
recv_socket.close()
img_grid_thw = recv_embedding_data.get_img_grid()
mm_inputs = mm_processor.get_mm_data(prompt, recv_embedding, img_grid_thw)
return mm_inputs
@@ -0,0 +1,527 @@
import asyncio
import ctypes
import logging
import multiprocessing as mp
import os
import pickle
import time
import traceback
from typing import Dict, List, Optional, Set, Tuple
import aiohttp
import numpy as np
import torch
import uvicorn
import zmq
import zmq.asyncio
from fastapi import FastAPI
from fastapi.responses import ORJSONResponse, Response
from transformers import AutoImageProcessor
from transformers.image_utils import load_images
from sglang.srt.configs.device_config import DeviceConfig
from sglang.srt.configs.load_config import LoadConfig
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.disaggregation.encode_receiver import EmbeddingData
from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferEngine
from sglang.srt.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from sglang.srt.layers.dp_attention import initialize_dp_attention
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import MultiModalStaticCache
from sglang.srt.model_loader import get_model
from sglang.srt.server_args import (
PortArgs,
ServerArgs,
set_global_server_args_for_scheduler,
)
from sglang.srt.utils import get_local_ip_auto, get_zmq_socket, random_uuid
logger = logging.getLogger(__name__)
rid_lock = asyncio.Lock()
rid_to_receive_endpoint: Dict[str, List[str]] = dict()
rid_to_receive_count: Dict[str, int] = dict()
class TensorWrapper:
"""Wrapper to keep tensor alive while exposing buffer for zero-copy."""
def __init__(self, tensor):
# Ensure tensor is on CPU and contiguous
if tensor.is_cuda:
tensor = tensor.cpu()
if not tensor.is_contiguous():
tensor = tensor.contiguous()
# Keep tensor reference
self.tensor = tensor
self.shape = list(tensor.shape)
self.dtype = tensor.dtype
def __buffer__(self):
data_ptr = self.tensor.data_ptr()
total_bytes = self.tensor.numel() * self.tensor.element_size()
c_obj = (ctypes.c_char * total_bytes).from_address(data_ptr)
c_obj._keep_alive_ref = self
return memoryview(c_obj)
def _convert(data):
if isinstance(data, torch.Tensor):
return data
elif isinstance(data, np.ndarray):
return torch.tensor(data)
elif isinstance(data, list) and isinstance(data[0], np.ndarray):
return torch.tensor(np.array(data))
elif isinstance(data, list) and isinstance(data[0], (int, float)):
return torch.tensor(data)
else:
return data
_image_grid_attrs = ["image_grid_thw", "image_grid_hws"]
def _get_image_grid_dim(images_input):
for attr in _image_grid_attrs:
if attr in images_input:
return images_input[attr]
raise ValueError(
f"Image grid dim ({_image_grid_attrs}) not found in {images_input}"
)
class MMEncoder:
def __init__(
self,
server_args: ServerArgs,
schedule_path=None,
dist_init_method=None,
rank: int = 0,
):
logger.info(f"init MMEncoder {rank}/{server_args.tp_size}")
self.server_args = server_args
set_global_server_args_for_scheduler(server_args)
self.rank = rank
self.image_processor = AutoImageProcessor.from_pretrained(
server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
use_fast=True,
)
self.model_config = ModelConfig.from_server_args(
server_args,
)
self.load_config = LoadConfig(
load_format=server_args.load_format,
download_dir=server_args.download_dir,
model_loader_extra_config=server_args.model_loader_extra_config,
remote_instance_weight_loader_seed_instance_ip=server_args.remote_instance_weight_loader_seed_instance_ip,
remote_instance_weight_loader_seed_instance_service_port=server_args.remote_instance_weight_loader_seed_instance_service_port,
remote_instance_weight_loader_send_weights_group_ports=server_args.remote_instance_weight_loader_send_weights_group_ports,
)
self.device = server_args.device
self.gpu_id = server_args.base_gpu_id + rank
self.device_config = DeviceConfig(
device=self.device,
gpu_id=self.gpu_id,
)
torch.get_device_module(self.device).set_device(self.gpu_id)
init_distributed_environment(
world_size=server_args.tp_size,
rank=rank,
distributed_init_method=dist_init_method,
local_rank=rank,
)
initialize_model_parallel(tensor_model_parallel_size=server_args.tp_size)
initialize_dp_attention(server_args, self.model_config)
self.model = get_model(
model_config=self.model_config,
load_config=self.load_config,
device_config=self.device_config,
)
self.context = zmq.asyncio.Context(2)
embedding_cache_size = int(os.environ.get("SGLANG_VLM_CACHE_SIZE_MB", "4096"))
self.mm_cache = MultiModalStaticCache(embedding_cache_size * 1024 * 1024)
self.mm_cache_lock = asyncio.Lock()
if schedule_path is not None:
self.schedule_socket = get_zmq_socket(
self.context, zmq.PULL, schedule_path, True
)
if self.rank == 0:
logger.info(
f"Using transfer backend: {self.server_args.encoder_transfer_backend}"
)
if self.server_args.encoder_transfer_backend == "mooncake":
self.local_ip = get_local_ip_auto()
self.engine = MooncakeTransferEngine(
hostname=self.local_ip,
gpu_id=None,
ib_device=server_args.disaggregation_ib_device,
)
self.embedding_to_send = dict()
logger.info(f"rank {rank} init finish ")
async def _encode(self, mm_items) -> torch.Tensor:
images = load_images(mm_items)
images_input = self.image_processor(images=images)
feature = images_input["pixel_values"]
mm_item = MultimodalDataItem.from_dict(
{
"modality": Modality.IMAGE,
"feature": _convert(feature),
}
)
for k, v in images_input.items():
if k == "pixel_values":
continue
mm_item.set(k, _convert(v))
# support mm_cache
mm_embedding = None
mm_hash = None
start_time = time.perf_counter()
if self.server_args.enable_prefix_mm_cache:
mm_item.set_pad_value()
mm_hash = MultiModalStaticCache.combine_hashes([mm_item.hash])
async with self.mm_cache_lock:
mm_cache = self.mm_cache.get([mm_item.hash])
if mm_cache is not None:
mm_embedding = mm_cache
if mm_embedding is None:
with torch.inference_mode():
mm_embedding: torch.Tensor = self.model.get_image_feature([mm_item])
mm_embedding = mm_embedding.cpu()
if len(mm_embedding.shape) != 2:
mm_embedding = mm_embedding.reshape(-1, mm_embedding.shape[-1])
if self.server_args.enable_prefix_mm_cache:
async with self.mm_cache_lock:
self.mm_cache.set(mm_hash, mm_embedding)
end_time = time.perf_counter()
logger.info(
f"Vit time : {(end_time - start_time)*1000:.2f} ms {mm_embedding.shape = }"
)
return _get_image_grid_dim(images_input), mm_embedding
async def _send(
self,
embedding: torch.Tensor,
mm_data: EmbeddingData,
session_id=None,
buffer_address=None,
prefill_host=None,
embedding_port=None,
url=None,
):
if self.server_args.encoder_transfer_backend == "mooncake":
self.engine.register(embedding.data_ptr(), embedding.nbytes)
self.engine.transfer_sync(
session_id, embedding.data_ptr(), buffer_address, embedding.nbytes
)
self.engine.deregister(embedding.data_ptr())
mm_data.embedding = None
mm_data.embedding_list[mm_data.part_idx] = None
# Send ack/data
endpoint = (
f"tcp://{url}"
if url is not None
else f"tcp://{prefill_host}:{embedding_port}"
)
logger.info(f"{endpoint = }")
socket = get_zmq_socket(
self.context,
zmq.PUSH,
endpoint,
False,
)
if self.server_args.encoder_transfer_backend == "mooncake":
socket.send_multipart([pickle.dumps(mm_data)])
else:
new_mm_data = mm_data.copy_without_embedding()
embedding_tensor = TensorWrapper(mm_data.embedding)
socket.send_multipart(
[pickle.dumps(new_mm_data), embedding_tensor.__buffer__()]
)
async def encode(self, mm_items, req_id, num_parts, part_idx):
start_time = time.time()
image_grid_dim, mm_embedding = await self._encode(mm_items)
end_time = time.time()
logger.info(f"🕛 encode cost = {(end_time - start_time) * 1000:.2f}ms")
if self.rank == 0:
mm_data = EmbeddingData(
req_id,
num_parts,
part_idx,
image_grid_dim,
mm_embedding,
)
self.embedding_to_send[mm_data.req_id] = mm_data
return mm_embedding.nbytes, mm_embedding.shape[0], mm_embedding.shape[1]
# For zmq_to_tokenizer zmq_to_scheduler and mooncake
async def send(
self, req_id, prefill_host, embedding_port, session_id=None, buffer_address=None
):
mm_data: EmbeddingData = self.embedding_to_send[req_id]
await self._send(
mm_data.embedding,
mm_data,
session_id=session_id,
buffer_address=buffer_address,
prefill_host=prefill_host,
embedding_port=embedding_port,
)
# For zmq_to_scheduler
async def send_with_url(
self,
req_id,
):
mm_data = self.embedding_to_send.get(req_id)
if not mm_data:
return
sent_urls: Set[str] = set()
all_tasks: List[Tuple[asyncio.Task, str]] = []
start_time = asyncio.get_running_loop().time()
timeout = 60.0
try:
while True:
async with rid_lock:
current_targets = rid_to_receive_endpoint.get(req_id, set()).copy()
expected_count = rid_to_receive_count.get(req_id)
new_targets = current_targets - sent_urls
if new_targets:
logger.info(
f"Found {len(new_targets)} new endpoints for {req_id}. Starting tasks..."
)
for url in new_targets:
task = asyncio.create_task(
self._send(
mm_data.embedding,
mm_data,
url=url,
)
)
all_tasks.append((task, url))
sent_urls.add(url) # Mark as handled immediately
if expected_count is not None and len(sent_urls) >= expected_count:
logger.info(
f"All {expected_count} endpoints initiated for {req_id}. Breaking loop."
)
break
if asyncio.get_running_loop().time() - start_time > timeout:
logger.error(
f"Timeout waiting for all endpoints for {req_id}. Initiated {len(sent_urls)}/{expected_count}"
)
break
await asyncio.sleep(0.001)
if all_tasks:
logger.info(
f"Loop finished. Awaiting completion of {len(all_tasks)} sending tasks..."
)
tasks_only = [t[0] for t in all_tasks]
results = await asyncio.gather(*tasks_only, return_exceptions=True)
# Process results and log errors
for i, result in enumerate(results):
url = all_tasks[i][1] # Retrieve URL associated with the task
if isinstance(result, Exception):
logger.error(f"Failed to send to {url}: {result}")
else:
logger.debug(f"Successfully sent to {url}")
logger.info(f"All tasks completed for req_id: {req_id}")
finally:
logger.info(f"Cleaning up resources for req_id {req_id}")
async with rid_lock:
rid_to_receive_endpoint.pop(req_id, None)
rid_to_receive_count.pop(req_id, None)
self.embedding_to_send.pop(req_id, None)
async def get_embedding_port(self, prefill_url):
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=1800)
) as session:
response = await session.post(
f"{prefill_url}/embedding_bootstrap",
json={"embedding_port": None},
)
response_json = await response.json()
return response_json["embedding_port"]
app = FastAPI()
encoder: Optional[MMEncoder] = None
send_sockets: List[zmq.Socket] = []
async def run_encoder(
server_args: ServerArgs, schedule_path, dist_init_method, rank: int
):
encoder = MMEncoder(server_args, schedule_path, dist_init_method, rank)
while True:
request = await encoder.schedule_socket.recv_pyobj()
await encoder.encode(
mm_items=request["mm_items"],
req_id=request["req_id"],
num_parts=request["num_parts"],
part_idx=request["part_idx"],
)
def launch_encoder(server_args, schedule_path, dist_init_method, rank):
try:
asyncio.run(run_encoder(server_args, schedule_path, dist_init_method, rank))
except KeyboardInterrupt:
logger.info(f"Exit rank {rank}")
except Exception:
traceback.print_exc()
def launch_server(server_args: ServerArgs):
global encoder
ctx = mp.get_context("spawn")
zmq_ctx = zmq.Context(10)
ipc_path_prefix = random_uuid()
port_args = PortArgs.init_new(server_args)
if server_args.dist_init_addr:
dist_init_method = f"tcp://{server_args.dist_init_addr}"
else:
dist_init_method = f"tcp://127.0.0.1:{port_args.nccl_port}"
for rank in range(1, server_args.tp_size):
schedule_path = f"ipc:///tmp/{ipc_path_prefix}_schedule_{rank}"
send_sockets.append(
get_zmq_socket(zmq_ctx, zmq.PUSH, schedule_path, bind=False)
)
ctx.Process(
target=launch_encoder,
args=(server_args, schedule_path, dist_init_method, rank),
daemon=True,
).start()
encoder = MMEncoder(server_args, dist_init_method=dist_init_method)
uvicorn.run(app, host=server_args.host, port=server_args.port)
@app.post("/encode")
async def handle_encode_request(request: dict):
# broadcast request
request.update({"enter_time": time.time()})
for socket in send_sockets:
socket.send_pyobj(request)
nbytes, embedding_len, embedding_dim = await encoder.encode(
mm_items=request["mm_items"],
req_id=request["req_id"],
num_parts=request["num_parts"],
part_idx=request["part_idx"],
)
if encoder.server_args.encoder_transfer_backend == "mooncake":
del request["mm_items"]
request.update(
{
"embedding_size": nbytes,
"embedding_len": embedding_len,
"embedding_dim": embedding_dim,
}
)
return ORJSONResponse(content=request)
elif encoder.server_args.encoder_transfer_backend == "zmq_to_scheduler":
logger.info(f"{request['embedding_port'] = }")
if request["embedding_port"] is None:
await encoder.send_with_url(
req_id=request["req_id"],
)
else:
assert type(request["embedding_port"]) == list
tasks = []
for embedding_port in request["embedding_port"]:
tasks.append(
encoder.send(
req_id=request["req_id"],
prefill_host=request["prefill_host"],
embedding_port=embedding_port,
)
)
await asyncio.gather(*tasks)
encoder.embedding_to_send.pop(request["req_id"], None)
return ORJSONResponse(content=None)
elif encoder.server_args.encoder_transfer_backend == "zmq_to_tokenizer":
await encoder.send(
req_id=request["req_id"],
prefill_host=request["prefill_host"],
embedding_port=request["embedding_port"],
)
encoder.embedding_to_send.pop(request["req_id"], None)
return ORJSONResponse(content=None)
@app.post("/send")
async def handle_send_request(request: dict):
# mooncake backend
await encoder.send(
req_id=request["req_id"],
prefill_host=request["prefill_host"],
embedding_port=request["embedding_port"],
session_id=request["session_id"],
buffer_address=request["buffer_address"],
)
encoder.embedding_to_send.pop(request["req_id"], None)
return ORJSONResponse(content=None)
@app.post("/scheduler_receive_url")
async def handle_scheduler_receive_url_request(request: dict):
rid = request["req_id"]
async with rid_lock:
global rid_to_receive_endpoint
if rid not in rid_to_receive_endpoint:
rid_to_receive_endpoint[rid] = set()
rid_to_receive_count[rid] = request["receive_count"]
assert rid_to_receive_count[rid] == request["receive_count"]
rid_to_receive_endpoint[rid].add(request["receive_url"])
@app.get("/health")
@app.get("/health_generate")
async def health_generate():
"""
Health check endpoint for the encoder server.
Returns 200 if the encoder is initialized and ready.
"""
if encoder is None:
return Response(status_code=503)
return Response(status_code=200)