chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[misc] add model arch and type to server info and use it for harmony (#14456)

Browse Source
This commit is contained in:
Simo Lin
2025-12-05 06:51:00 -08:00
committed by GitHub
parent 8fce9e7b2a
commit 7235a7fbe9
11 changed files with 205 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
python/sglang/srt/entrypoints/grpc_server.py
View File

@@ -22,6 +22,7 @@ from grpc_health.v1 import health_pb2_grpc
from grpc_reflection.v1alpha import reflection
import sglang
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
from sglang.srt.grpc import sglang_scheduler_pb2, sglang_scheduler_pb2_grpc
from sglang.srt.grpc.grpc_request_manager import GrpcRequestManager
@@ -321,7 +322,8 @@ class SGLangSchedulerServicer(sglang_scheduler_pb2_grpc.SglangSchedulerServicer)
max_context_length=self.model_info["max_context_length"],
vocab_size=self.model_info["vocab_size"],
supports_vision=self.model_info["supports_vision"],
model_type=self.model_info["model_type"],
model_type=self.model_info.get("model_type") or "",
architectures=self.model_info.get("architectures") or [],
eos_token_ids=self.model_info["eos_token_ids"],
pad_token_id=self.model_info["pad_token_id"],
bos_token_id=self.model_info["bos_token_id"],
@@ -718,7 +720,10 @@ async def serve_grpc(
server_args=server_args,
)
# Update model info from scheduler info
# Load model config to get HF config info (same as TokenizerManager does)
model_config = ModelConfig.from_server_args(server_args)
# Update model info from scheduler info and model config
if model_info is None:
model_info = {
"model_name": server_args.model_path,
@@ -727,7 +732,8 @@ async def serve_grpc(
),
"vocab_size": scheduler_info.get("vocab_size", 128256),
"supports_vision": scheduler_info.get("supports_vision", False),
"model_type": scheduler_info.get("model_type", "transformer"),
"model_type": getattr(model_config.hf_config, "model_type", None),
"architectures": getattr(model_config.hf_config, "architectures", None),
"max_req_input_len": scheduler_info.get("max_req_input_len", 8192),
"eos_token_ids": scheduler_info.get("eos_token_ids", []),
"pad_token_id": scheduler_info.get("pad_token_id", 0),

7
python/sglang/srt/entrypoints/http_server.py
View File

@@ -497,14 +497,17 @@ async def get_model_info():
@app.get("/model_info")
async def model_info():
"""Get the model information."""
model_config = _global_state.tokenizer_manager.model_config
result = {
"model_path": _global_state.tokenizer_manager.model_path,
"tokenizer_path": _global_state.tokenizer_manager.server_args.tokenizer_path,
"is_generation": _global_state.tokenizer_manager.is_generation,
"preferred_sampling_params": _global_state.tokenizer_manager.server_args.preferred_sampling_params,
"weight_version": _global_state.tokenizer_manager.server_args.weight_version,
"has_image_understanding": _global_state.tokenizer_manager.model_config.is_image_understandable_model,
"has_audio_understanding": _global_state.tokenizer_manager.model_config.is_audio_understandable_model,
"has_image_understanding": model_config.is_image_understandable_model,
"has_audio_understanding": model_config.is_audio_understandable_model,
"model_type": getattr(model_config.hf_config, "model_type", None),
"architectures": getattr(model_config.hf_config, "architectures", None),
}
return result

1
python/sglang/srt/grpc/sglang_scheduler.proto
View File

@@ -427,6 +427,7 @@ message GetModelInfoResponse {
int32 pad_token_id = 12;
int32 bos_token_id = 13;
int32 max_req_input_len = 14;
repeated string architectures = 15;
}
// Get server information

16
python/sglang/srt/grpc/sglang_scheduler_pb2.py
View File

File diff suppressed because one or more lines are too long

6
python/sglang/srt/grpc/sglang_scheduler_pb2.pyi
View File

@@ -432,7 +432,7 @@ class GetModelInfoRequest(_message.Message):
def __init__(self) -> None: ...
class GetModelInfoResponse(_message.Message):
__slots__ = ("model_path", "tokenizer_path", "is_generation", "preferred_sampling_params", "weight_version", "served_model_name", "max_context_length", "vocab_size", "supports_vision", "model_type", "eos_token_ids", "pad_token_id", "bos_token_id", "max_req_input_len")
__slots__ = ("model_path", "tokenizer_path", "is_generation", "preferred_sampling_params", "weight_version", "served_model_name", "max_context_length", "vocab_size", "supports_vision", "model_type", "eos_token_ids", "pad_token_id", "bos_token_id", "max_req_input_len", "architectures")
MODEL_PATH_FIELD_NUMBER: _ClassVar[int]
TOKENIZER_PATH_FIELD_NUMBER: _ClassVar[int]
IS_GENERATION_FIELD_NUMBER: _ClassVar[int]
@@ -447,6 +447,7 @@ class GetModelInfoResponse(_message.Message):
PAD_TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
BOS_TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
MAX_REQ_INPUT_LEN_FIELD_NUMBER: _ClassVar[int]
ARCHITECTURES_FIELD_NUMBER: _ClassVar[int]
model_path: str
tokenizer_path: str
is_generation: bool
@@ -461,7 +462,8 @@ class GetModelInfoResponse(_message.Message):
pad_token_id: int
bos_token_id: int
max_req_input_len: int
def __init__(self, model_path: _Optional[str] = ..., tokenizer_path: _Optional[str] = ..., is_generation: bool = ..., preferred_sampling_params: _Optional[str] = ..., weight_version: _Optional[str] = ..., served_model_name: _Optional[str] = ..., max_context_length: _Optional[int] = ..., vocab_size: _Optional[int] = ..., supports_vision: bool = ..., model_type: _Optional[str] = ..., eos_token_ids: _Optional[_Iterable[int]] = ..., pad_token_id: _Optional[int] = ..., bos_token_id: _Optional[int] = ..., max_req_input_len: _Optional[int] = ...) -> None: ...
architectures: _containers.RepeatedScalarFieldContainer[str]
def __init__(self, model_path: _Optional[str] = ..., tokenizer_path: _Optional[str] = ..., is_generation: bool = ..., preferred_sampling_params: _Optional[str] = ..., weight_version: _Optional[str] = ..., served_model_name: _Optional[str] = ..., max_context_length: _Optional[int] = ..., vocab_size: _Optional[int] = ..., supports_vision: bool = ..., model_type: _Optional[str] = ..., eos_token_ids: _Optional[_Iterable[int]] = ..., pad_token_id: _Optional[int] = ..., bos_token_id: _Optional[int] = ..., max_req_input_len: _Optional[int] = ..., architectures: _Optional[_Iterable[str]] = ...) -> None: ...
class GetServerInfoRequest(_message.Message):
__slots__ = ()