From b74a57a8d96ed47f9fa074536c9a3cb56be13884 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Wed, 21 Jan 2026 14:48:32 -0800 Subject: [PATCH] [Auto Sync] Update detokenizer_manager.py, io_struct.py, mu... (20260120) (#17442) Co-authored-by: github-actions[bot] Co-authored-by: Wangfan Fu --- docs/get_started/install.md | 34 ++++++-------- python/pyproject.toml | 1 - python/pyproject_cpu.toml | 1 - python/pyproject_other.toml | 1 - python/pyproject_xpu.toml | 1 - python/sglang/srt/entrypoints/http_server.py | 44 +++---------------- .../srt/managers/detokenizer_manager.py | 24 +++++----- python/sglang/srt/managers/io_struct.py | 14 ++++-- .../srt/managers/multi_tokenizer_mixin.py | 4 +- .../scheduler_output_processor_mixin.py | 10 ++--- .../sglang/srt/managers/tokenizer_manager.py | 4 +- .../openai_server/basic/test_openai_server.py | 7 ++- .../quant/test_gptqmodel_dynamic.py | 2 +- 13 files changed, 59 insertions(+), 88 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index e9686994f..990a8f7f5 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -15,25 +15,20 @@ pip install uv uv pip install "sglang" ``` -For different CUDA versions, you can try to add `--extra-index-url` to find the correct [PyTorch](https://pytorch.org/get-started/locally/) wheel. -For example, on GB200, you will need to do the following. Otherwise, it will install the CPU version of PyTorch. -``` -uv pip install "sglang" --extra-index-url https://download.pytorch.org/whl/cu129 -``` - -For CUDA 13, Docker is recommended (see Method 3 note on B300/GB300/CUDA 13). If you do not have Docker access, installing the matching `sgl_kernel` wheel from [the sgl-project whl releases](https://github.com/sgl-project/whl/releases) after installing SGLang also works. Replace `X.Y.Z` with the `sgl_kernel` version required by your SGLang install (you can find this by running `uv pip show sgl_kernel`). Examples: - -```bash -# x86_64 -uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl" - -# aarch64 -uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl" -``` - **Quick fixes to common problems** +- In some cases (e.g., GB200), the above command might install a wrong torch version (e.g., the CPU version) due to dependency resolution. To fix this, you can first run the above command and then force-reinstall the correct [PyTorch](https://pytorch.org/get-started/locally/) with the following: + ``` + uv pip install "torch" --extra-index-url https://download.pytorch.org/whl/cu129 --force-reinstall + ``` +- For CUDA 13, Docker is recommended (see the Method 3 note on B300/GB300/CUDA 13). If you do not have Docker access, installing the matching `sgl_kernel` wheel from [the sgl-project whl releases](https://github.com/sgl-project/whl/releases) after installing SGLang also works. Replace `X.Y.Z` with the `sgl_kernel` version required by your SGLang (you can find this by running `uv pip show sgl_kernel`). Examples: + ```bash + # x86_64 + uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_x86_64.whl" -- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: + # aarch64 + uv pip install "https://github.com/sgl-project/whl/releases/download/vX.Y.Z/sgl_kernel-X.Y.Z+cu130-cp310-abi3-manylinux2014_aarch64.whl" + ``` +- If you encounter `OSError: CUDA_HOME environment variable is not set`, set it to your CUDA install root with either of the following solutions: 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. 2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. @@ -84,9 +79,8 @@ docker run --gpus all \ You can also find the nightly docker images [here](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly). -On B300/GB300 (SM103) or CUDA 13 environment, we recommend using the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`. -Please, do not re-install the project as editable inside the docker image, since it will override the version of -libraries specified by the cu13 docker image. +Notes: +- On B300/GB300 (SM103) or CUDA 13 environment, we recommend using the nightly image at `lmsysorg/sglang:dev-cu13` or stable image at `lmsysorg/sglang:latest-cu130-runtime`. Please, do not re-install the project as editable inside the docker image, since it will override the version of libraries specified by the cu13 docker image. ## Method 4: Using Kubernetes diff --git a/python/pyproject.toml b/python/pyproject.toml index e4f00f0dc..fdf597405 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -69,7 +69,6 @@ dependencies = [ "torchaudio==2.9.1", "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. "torchvision", - "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 74e2b0c58..0957c74f5 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -57,7 +57,6 @@ dependencies = [ "soundfile==0.13.1", "tiktoken", "timm==1.0.16", - "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 7efce6dfa..f4a236d34 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -57,7 +57,6 @@ runtime_common = [ "soundfile==0.13.1", "tiktoken", "timm==1.0.16", - "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index f88480acf..2b3486e5d 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -61,7 +61,6 @@ dependencies = [ "soundfile==0.13.1", "tiktoken", "timm==1.0.16", - "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn", diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index d2526f09f..afac1d03d 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -46,7 +46,7 @@ import orjson import requests import uvicorn import uvloop -from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile +from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import ORJSONResponse, Response, StreamingResponse @@ -552,26 +552,20 @@ async def model_info(): "has_audio_understanding": model_config.is_audio_understandable_model, "model_type": getattr(model_config.hf_config, "model_type", None), "architectures": getattr(model_config.hf_config, "architectures", None), + "weight_version": _global_state.tokenizer_manager.server_args.weight_version, + # "hf_config": model_config.hf_config.to_dict(), } return result @app.get("/get_weight_version") -async def get_weight_version(): - """Get the current weight version (deprecated - use /weight_version instead).""" - logger.warning( - "Endpoint '/get_weight_version' is deprecated and will be removed in a future version. " - "Please use '/weight_version' instead." - ) - return await weight_version() - - @app.get("/weight_version") async def weight_version(): """Get the current weight version.""" - return { - "weight_version": _global_state.tokenizer_manager.server_args.weight_version - } + raise HTTPException( + status_code=404, + detail="Endpoint '/get_weight_version' or '/weight_version' is deprecated. Please use '/model_info' instead.", + ) @app.get("/get_server_info") @@ -661,30 +655,6 @@ async def generate_request(obj: GenerateReqInput, request: Request): return _create_error_response(e) -@app.api_route("/generate_from_file", methods=["POST"]) -async def generate_from_file_request(file: UploadFile, request: Request): - """Handle a generate request, this is purely to work with input_embeds.""" - content = await file.read() - input_embeds = orjson.loads(content.decode("utf-8")) - - obj = GenerateReqInput( - input_embeds=input_embeds, - sampling_params={ - "temperature": 0.0, - "max_new_tokens": 512, - }, - ) - - try: - ret = await _global_state.tokenizer_manager.generate_request( - obj, request - ).__anext__() - return ret - except ValueError as e: - logger.error(f"Error: {e}") - return _create_error_response(e) - - @app.api_route("/encode", methods=["POST", "PUT"]) async def encode_request(obj: EmbeddingReqInput, request: Request): """Handle an embedding request.""" diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py index 33cbacfa2..50b861e14 100644 --- a/python/sglang/srt/managers/detokenizer_manager.py +++ b/python/sglang/srt/managers/detokenizer_manager.py @@ -326,20 +326,20 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): return output_strs - def _extract_routed_experts(self, recv_obj: BatchTokenIDOutput) -> List[List[int]]: - output_routed_experts = None - if recv_obj.output_routed_experts is not None: - output_routed_experts = [ + def _extract_routed_experts( + self, recv_obj: BatchTokenIDOutput + ) -> List[List[int]] | None: + routed_experts = None + if recv_obj.routed_experts is not None: + routed_experts = [ ( - pybase64.b64encode(output_routed_experts.numpy().tobytes()).decode( - "utf-8" - ) - if output_routed_experts is not None + pybase64.b64encode(routed_experts.numpy().tobytes()).decode("utf-8") + if routed_experts is not None else [] ) - for output_routed_experts in recv_obj.output_routed_experts + for routed_experts in recv_obj.routed_experts ] - return output_routed_experts + return routed_experts def handle_batch_token_id_out(self, recv_obj: BatchTokenIDOutput): # If handling idle batch, set output_strs to []. @@ -348,7 +348,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): if len(recv_obj.rids) > 0 else [] ) - output_routed_experts = self._extract_routed_experts(recv_obj) + routed_experts = self._extract_routed_experts(recv_obj) return BatchStrOutput( rids=recv_obj.rids, @@ -375,7 +375,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): output_token_ids_logprobs_idx=recv_obj.output_token_ids_logprobs_idx, output_token_entropy_val=recv_obj.output_token_entropy_val, output_hidden_states=recv_obj.output_hidden_states, - output_routed_experts=output_routed_experts, + routed_experts=routed_experts, customized_info=recv_obj.customized_info, placeholder_tokens_idx=None, placeholder_tokens_val=None, diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 04e3b6c5b..fad02e0a0 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -202,6 +202,8 @@ class GenerateReqInput(BaseReq, APIServingTimingMixin): return_hidden_states: Union[List[bool], bool] = False # Whether to return captured routed experts return_routed_experts: bool = False + # The start location in the prompt for returning routed experts. + routed_experts_start_len: int = 0 # The modalities of the image data [image, multi-images, video] modalities: Optional[List[str]] = None @@ -709,6 +711,8 @@ class TokenizedGenerateReqInput(BaseReq): # Whether to return captured routed experts return_routed_experts: bool = False + # The start location in the prompt for returning routed experts. + routed_experts_start_len: int = 0 # The input embeds input_embeds: Optional[Union[List[List[List[float]]], List[List[float]]]] = None @@ -981,8 +985,9 @@ class BatchTokenIDOutput( # Hidden states output_hidden_states: List[List[float]] - # The routed experts for each output token - output_routed_experts: List[torch.Tensor] + # The routed experts for each token, including both input and output tokens + # routed_experts[i] is a tensor of shape (token, layer, top_k) for request i + routed_experts: List[Optional[torch.Tensor]] # The information of placeholder tokens (e.g., image token) # idx is the index of the token in the prompt after expansion. @@ -1068,8 +1073,9 @@ class BatchStrOutput( # Hidden states output_hidden_states: List[List[float]] - # The routed experts for each output token - output_routed_experts: List[List[int]] + # The routed experts for each token, including both input and output tokens + # routed_experts[i] is a tensor of shape (token, layer, top_k) for request i + routed_experts: List[Optional[torch.Tensor]] # The information of placeholder tokens (e.g., image token) # idx is the index of the token in the prompt after expansion. diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index 0e9d314e3..4f2e3fb19 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -278,8 +278,8 @@ def _handle_output_by_index(output, i): output_hidden_states=_extract_field_by_index( output, "output_hidden_states", i, check_length=False ), - output_routed_experts=_extract_field_by_index( - output, "output_routed_experts", i, check_length=False + routed_experts=_extract_field_by_index( + output, "routed_experts", i, check_length=False ), customized_info=_extract_field_by_index( output, "customized_info", i, check_length=False diff --git a/python/sglang/srt/managers/scheduler_output_processor_mixin.py b/python/sglang/srt/managers/scheduler_output_processor_mixin.py index bf74f81dc..c4728b714 100644 --- a/python/sglang/srt/managers/scheduler_output_processor_mixin.py +++ b/python/sglang/srt/managers/scheduler_output_processor_mixin.py @@ -854,7 +854,7 @@ class SchedulerOutputProcessorMixin: retraction_counts = [] output_hidden_states = None load = self.get_load() - output_routed_experts = None + routed_experts = None customized_info = {} queue_times = [] @@ -1051,9 +1051,9 @@ class SchedulerOutputProcessorMixin: output_hidden_states = [] output_hidden_states.append(req.hidden_states) if req.return_routed_experts: - if output_routed_experts is None: - output_routed_experts = [] - output_routed_experts.append(req.routed_experts) + if routed_experts is None: + routed_experts = [] + routed_experts.append(req.routed_experts) if req.customized_info is not None: for k, v in req.customized_info.items(): @@ -1108,7 +1108,7 @@ class SchedulerOutputProcessorMixin: output_token_ids_logprobs_idx=output_token_ids_logprobs_idx, output_token_entropy_val=None, output_hidden_states=output_hidden_states, - output_routed_experts=output_routed_experts, + routed_experts=routed_experts, customized_info=customized_info, placeholder_tokens_idx=None, placeholder_tokens_val=None, diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index acc02aebe..ae6211887 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1534,8 +1534,8 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi if getattr(recv_obj, "output_hidden_states", None): meta_info["hidden_states"] = recv_obj.output_hidden_states[i] - if getattr(recv_obj, "output_routed_experts", None): - meta_info["routed_experts"] = recv_obj.output_routed_experts[i] + if getattr(recv_obj, "routed_experts", None): + meta_info["routed_experts"] = recv_obj.routed_experts[i] if getattr(recv_obj, "customized_info", None): for k, v in recv_obj.customized_info.items(): meta_info[k] = v[i] diff --git a/test/registered/openai_server/basic/test_openai_server.py b/test/registered/openai_server/basic/test_openai_server.py index 04b6ec84e..235c92d73 100644 --- a/test/registered/openai_server/basic/test_openai_server.py +++ b/test/registered/openai_server/basic/test_openai_server.py @@ -89,7 +89,9 @@ class TestOpenAIServer(CustomTestCase): if logprobs: assert response.choices[0].logprobs - assert isinstance(response.choices[0].logprobs.tokens[0], str) + assert isinstance( + response.choices[0].logprobs.tokens[0], str + ), f"{response=}" assert isinstance(response.choices[0].logprobs.top_logprobs[1], dict) ret_num_top_logprobs = len(response.choices[0].logprobs.top_logprobs[1]) @@ -155,6 +157,9 @@ class TestOpenAIServer(CustomTestCase): if logprobs: assert response.choices[0].logprobs, f"no logprobs in response" + print( + f"{response=}, {response.choices[0]=}, {response.choices[0].logprobs=}" + ) assert isinstance( response.choices[0].logprobs.tokens[0], str ), f"{response.choices[0].logprobs.tokens[0]} is not a string" diff --git a/test/registered/quant/test_gptqmodel_dynamic.py b/test/registered/quant/test_gptqmodel_dynamic.py index dd8fd51c9..7a52b9028 100644 --- a/test/registered/quant/test_gptqmodel_dynamic.py +++ b/test/registered/quant/test_gptqmodel_dynamic.py @@ -54,11 +54,11 @@ def check_quant_method(model_path: str, use_marlin_kernel: bool): model_config=model_config, load_config=load_config, device_config=device_config ) - from sglang.srt.layers.linear import UnquantizedLinearMethod from sglang.srt.layers.quantization.gptq import ( GPTQLinearMethod, GPTQMarlinLinearMethod, ) + from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod linear_method_cls = ( GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod)