chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Support Multi Process Tokenizer Manager (#6555)

Browse Source 
Signed-off-by: ybyang <ybyang7@iflytek.com>
Signed-off-by: huanglong <huanglong@linux.alibaba.com>
Co-authored-by: lw9527 <952799980@qq.com>
Co-authored-by: huanglong <huanglong@linux.alibaba.com>
Co-authored-by: Huang Long <121648372+LLLL114@users.noreply.github.com>
This commit is contained in:
ybyang
2025-08-08 16:45:50 +08:00
committed by GitHub
parent 6ee6619b7a
commit 7490e3f67d
9 changed files with 1133 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
python/sglang/srt/managers/io_struct.py
View File

@@ -782,12 +782,13 @@ class BatchEmbeddingOut:
@dataclass
class FlushCacheReqInput:
pass
rids: Optional[Union[List[str], str]] = None
@dataclass
class FlushCacheReqOutput:
success: bool
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -798,6 +799,7 @@ class UpdateWeightFromDiskReqInput:
load_format: Optional[str] = None
# Whether to abort all requests before updating weights
abort_all_requests: bool = False
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -806,6 +808,7 @@ class UpdateWeightFromDiskReqOutput:
message: str
# Number of paused requests during weight sync.
num_paused_requests: Optional[int] = 0
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -819,12 +822,14 @@ class UpdateWeightsFromDistributedReqInput:
flush_cache: bool = True
# Whether to abort all requests before updating weights
abort_all_requests: bool = False
rids: Optional[Union[List[str], str]] = None
@dataclass
class UpdateWeightsFromDistributedReqOutput:
success: bool
message: str
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -842,12 +847,14 @@ class UpdateWeightsFromTensorReqInput:
flush_cache: bool = True
# Whether to abort all requests before updating weights
abort_all_requests: bool = False
rids: Optional[Union[List[str], str]] = None
@dataclass
class UpdateWeightsFromTensorReqOutput:
success: bool
message: str
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -864,23 +871,27 @@ class InitWeightsUpdateGroupReqInput:
group_name: str = "weight_update_group"
# The backend
backend: str = "nccl"
rids: Optional[Union[List[str], str]] = None
@dataclass
class InitWeightsUpdateGroupReqOutput:
success: bool
message: str
rids: Optional[Union[List[str], str]] = None
@dataclass
class GetWeightsByNameReqInput:
name: str
truncate_size: int = 100
rids: Optional[Union[List[str], str]] = None
@dataclass
class GetWeightsByNameReqOutput:
parameter: list
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -888,11 +899,12 @@ class ReleaseMemoryOccupationReqInput:
# Optional tags to identify the memory region, which is primarily used for RL
# Currently we only support `weights` and `kv_cache`
tags: Optional[List[str]] = None
rids: Optional[Union[List[str], str]] = None
@dataclass
class ReleaseMemoryOccupationReqOutput:
pass
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -900,21 +912,23 @@ class ResumeMemoryOccupationReqInput:
# Optional tags to identify the memory region, which is primarily used for RL
# Currently we only support `weights` and `kv_cache`
tags: Optional[List[str]] = None
rids: Optional[Union[List[str], str]] = None
@dataclass
class ResumeMemoryOccupationReqOutput:
pass
rids: Optional[Union[List[str], str]] = None
@dataclass
class SlowDownReqInput:
forward_sleep_time: Optional[float]
rids: Optional[Union[List[str], str]] = None
@dataclass
class SlowDownReqOutput:
pass
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -923,29 +937,37 @@ class AbortReq:
rid: str = ""
# Whether to abort all requests
abort_all: bool = False
# The finished reason data
rids: Optional[Union[List[str], str]] = None
finished_reason: Optional[Dict[str, Any]] = None
def __post_init__(self):
self.rids = self.rid
@dataclass
class GetInternalStateReq:
pass
rids: Optional[Union[List[str], str]] = None
@dataclass
class GetInternalStateReqOutput:
internal_state: Dict[Any, Any]
rids: Optional[Union[List[str], str]] = None
@dataclass
class SetInternalStateReq:
server_args: Dict[str, Any]
rids: Optional[Union[List[str], str]] = None
@dataclass
class SetInternalStateReqOutput:
updated: bool
server_args: Dict[str, Any]
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -961,6 +983,7 @@ class ProfileReqInput:
profile_by_stage: bool = False
with_stack: Optional[bool] = None
record_shapes: Optional[bool] = None
rids: Optional[Union[List[str], str]] = None
class ProfileReqType(Enum):
@@ -979,12 +1002,14 @@ class ProfileReq:
with_stack: Optional[bool] = None
record_shapes: Optional[bool] = None
profile_id: Optional[str] = None
rids: Optional[Union[List[str], str]] = None
@dataclass
class ProfileReqOutput:
success: bool
message: str
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -993,27 +1018,32 @@ class ConfigureLoggingReq:
log_requests_level: Optional[int] = None
dump_requests_folder: Optional[str] = None
dump_requests_threshold: Optional[int] = None
rids: Optional[Union[List[str], str]] = None
@dataclass
class OpenSessionReqInput:
capacity_of_str_len: int
session_id: Optional[str] = None
rids: Optional[Union[List[str], str]] = None
@dataclass
class CloseSessionReqInput:
session_id: str
rids: Optional[Union[List[str], str]] = None
@dataclass
class OpenSessionReqOutput:
session_id: Optional[str]
success: bool
rids: Optional[Union[List[str], str]] = None
@dataclass
class HealthCheckOutput:
rids: Optional[Union[List[str], str]] = None
pass
@@ -1025,7 +1055,7 @@ class ExpertDistributionReq(Enum):
@dataclass
class ExpertDistributionReqOutput:
pass
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -1050,18 +1080,21 @@ class ParseFunctionCallReq:
tool_call_parser: Optional[str] = (
None # Specify the parser type, e.g. 'llama3', 'qwen25', or 'mistral'. If not specified, tries all.
)
rids: Optional[Union[List[str], str]] = None
@dataclass
class SeparateReasoningReqInput:
text: str # The text to parse.
reasoning_parser: str # Specify the parser type, e.g., "deepseek-r1".
rids: Optional[Union[List[str], str]] = None
@dataclass
class VertexGenerateReqInput:
instances: List[dict]
parameters: Optional[dict] = None
rids: Optional[Union[List[str], str]] = None
@dataclass
@@ -1086,6 +1119,7 @@ class LoadLoRAAdapterReqInput:
pinned: bool = False
# The unique identifier for the LoRA adapter, which automatically generated in the `TokenizerManager`.
lora_id: Optional[str] = None
rids: Optional[Union[List[str], str]] = None
def to_ref(self) -> LoRARef:
return LoRARef(
@@ -1102,6 +1136,7 @@ class UnloadLoRAAdapterReqInput:
lora_name: str
# The unique identifier for the LoRA adapter, which automatically generated in the `TokenizerManager`.
lora_id: Optional[str] = None
rids: Optional[Union[List[str], str]] = None
def to_ref(self) -> LoRARef:
return LoRARef(
@@ -1115,11 +1150,18 @@ class LoRAUpdateResult:
success: bool
error_message: Optional[str] = None
loaded_adapters: Optional[Dict[str, LoRARef]] = None
rids: Optional[Union[List[str], str]] = None
LoadLoRAAdapterReqOutput = UnloadLoRAAdapterReqOutput = LoRAUpdateResult
@dataclass
class MultiTokenizerRegisterReq:
rids: Optional[Union[List[str], str]] = None
ipc_name: Optional[str] = None
class BlockReqType(Enum):
BLOCK = 1
UNBLOCK = 2