From 7f9a3d060932f7e5b015ca391012b000c36be2fd Mon Sep 17 00:00:00 2001 From: husf Date: Tue, 30 Dec 2025 20:05:40 +0800 Subject: [PATCH] [docs][NPU]Update model and feature docs support (#16124) --- docs/platforms/ascend_npu.md | 45 +++--- docs/platforms/ascend_npu_deepseek_example.md | 48 +++--- docs/platforms/ascend_npu_qwen3_examples.md | 12 +- docs/platforms/ascend_npu_support_features.md | 61 ++++---- docs/platforms/ascend_npu_support_models.md | 144 ++++++++++-------- 5 files changed, 165 insertions(+), 145 deletions(-) diff --git a/docs/platforms/ascend_npu.md b/docs/platforms/ascend_npu.md index 0a572d233..f1820a8e1 100644 --- a/docs/platforms/ascend_npu.md +++ b/docs/platforms/ascend_npu.md @@ -31,6 +31,7 @@ pip install mf-adapter==1.0.0 #### Pytorch and Pytorch Framework Adaptor on Ascend At the moment NPUGraph optimizations are supported only in `torch_npu==2.6.0.post3` that requires 'torch==2.6.0'. + _TODO: NPUGraph optimizations will be supported in future releases of 'torch_npu' 2.7.1, 2.8.0 and 2.9.0_ ```shell @@ -41,7 +42,7 @@ pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-ur pip install torch_npu==$TORCH_NPU_VERSION ``` -While there is no resleased versions of 'torch_npu' for 'torch==2.7.1' and 'torch==2.8.0' we provide custom builds of 'torch_npu'. PLATFORM can be 'aarch64' or 'x86_64' +While there is no released versions of 'torch_npu' for 'torch==2.7.1' and 'torch==2.8.0' we provide custom builds of 'torch_npu'. PLATFORM can be 'aarch64' or 'x86_64' ```shell PLATFORM="aarch64" @@ -52,7 +53,7 @@ wget https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/torch_npu/torc pip install torch_npu-${PYTORCH_VERSION}.post2.dev20251120-cp311-cp311-manylinux_2_28_${PLATFORM}.whl ``` -If you are using other versions of 'torch' install 'torch_npu' from sources, check [installation guide](https://github.com/Ascend/pytorch/blob/master/README.md) +If you are using other versions of `torch` and install `torch_npu`, check [installation guide](https://github.com/Ascend/pytorch/blob/master/README.md) #### Triton on Ascend @@ -69,7 +70,7 @@ pip install triton-ascend==3.2.0rc4 For installation of Triton on Ascend nightly builds or from sources, follow [installation guide](https://gitcode.com/Ascend/triton-ascend/blob/master/docs/sources/getting-started/installation.md) #### SGLang Kernels NPU -We provide our own set of SGL kernels, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md). +We provide SGL kernels for Ascend NPU, check [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/sgl_kernel_npu/README.md). #### DeepEP-compatible Library We provide a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md). @@ -97,7 +98,7 @@ mv python/pyproject_other.toml python/pyproject.toml pip install -e python[srt_npu] ``` -### Method 2: Using docker +### Method 2: Using Docker Image #### Obtain Image You can download the SGLang image or build an image based on Dockerfile to obtain the Ascend NPU image. 1. Download SGLang image @@ -136,10 +137,10 @@ alias drun='docker run -it --rm --privileged --network=host --ipc=host --shm-siz --volume /etc/ascend_install.info:/etc/ascend_install.info \ --volume /var/queue_schedule:/var/queue_schedule --volume ~/.cache/:/root/.cache/' -# Add HF_TOKEN env for download model by SGLang +# Add HF_TOKEN env for download model by SGLang. drun --env "HF_TOKEN=" \ \ - python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend --host 0.0.0.0 --port 30000 + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend ``` ## System Settings @@ -159,7 +160,6 @@ cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor # shows performance ```shell sudo sysctl -w kernel.numa_balancing=0 - # Check cat /proc/sys/kernel/numa_balancing # shows 0 ``` @@ -177,13 +177,20 @@ cat /proc/sys/vm/swappiness # shows 10 ### Running Service For Large Language Models #### PD Mixed Scene ```shell -python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend --host 0.0.0.0 --port 30000 -``` -#### PD Separation Scene -Launch prefill server -```shell +# Enabling CPU Affinity export SGLANG_SET_CPU_AFFINITY=1 -# PIP: recommended to config first Prefill Server IP, all server need to be config the same ip, PORT: one free port +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend +``` + +#### PD Separation Scene +1. Launch Prefill Server +```shell +# Enabling CPU Affinity +export SGLANG_SET_CPU_AFFINITY=1 + +# PIP: recommended to config first Prefill Server IP +# PORT: one free port +# all sglang servers need to be config the same PIP and PORT, export ASCEND_MF_STORE_URL="tcp://PIP:PORT" # if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" @@ -196,13 +203,13 @@ python3 -m sglang.launch_server \ --device npu \ --base-gpu-id 0 \ --tp-size 1 \ - --host 127.0.0.1 \ - --port 8000 ``` -Launch Decode server + +2. Launch Decode Server ```shell -export SGLANG_SET_CPU_AFFINITY=1 -# PIP: recommended to config first Prefill Server IP, all server need to be config the same ip, PORT: one free port +# PIP: recommended to config first Prefill Server IP +# PORT: one free port +# all sglang servers need to be config the same PIP and PORT, export ASCEND_MF_STORE_URL="tcp://PIP:PORT" # if you are Atlas 800I A2 hardware and use rdma for kv cache transfer, add this parameter export ASCEND_MF_TRANSFER_PROTOCOL="device_rdma" @@ -218,7 +225,7 @@ python3 -m sglang.launch_server \ --port 8001 ``` -Launch Router +3. Launch Router ```shell python3 -m sglang_router.launch_router \ --pd-disaggregation \ diff --git a/docs/platforms/ascend_npu_deepseek_example.md b/docs/platforms/ascend_npu_deepseek_example.md index acb864ef5..101b971da 100644 --- a/docs/platforms/ascend_npu_deepseek_example.md +++ b/docs/platforms/ascend_npu_deepseek_example.md @@ -2,7 +2,7 @@ ### Running DeepSeek-V3 -#### Running DeepSeek on 1 x Atlas 800I A3. +#### Running DeepSeek in PD mixed mode on 1 x Atlas 800I A3. W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). @@ -32,15 +32,13 @@ python3 -m sglang.launch_server \ --device npu \ --quantization modelslim \ --watchdog-timeout 9000 \ - --host 127.0.0.1 \ - --port 6688 \ --cuda-graph-bs 8 16 24 28 32 \ --mem-fraction-static 0.68 \ --max-running-requests 128 \ --context-length 8188 \ --disable-radix-cache \ --chunked-prefill-size -1 \ - --max-prefill-tokens 6000 \ + --max-prefill-tokens 16384 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --enable-dp-attention \ @@ -53,17 +51,17 @@ python3 -m sglang.launch_server \ --dtype bfloat16 ``` -#### Running DeepSeek with PD disaggregation on 2 x Atlas 800I A3. +#### Running DeepSeek with PD disaggregation mode on 2 x Atlas 800I A3. W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - -Prefill: +1. Prefill: ```shell export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 -#PD + +#memfabric config store export ASCEND_MF_STORE_URL="tcp://:" #Deepep communication settings @@ -78,10 +76,11 @@ export TASK_QUEUE_ENABLE=2 python -m sglang.launch_server \ --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ --host $PREFILL_HOST_IP \ --port 8000 \ + --disaggregation-mode prefill \ --disaggregation-bootstrap-port 8996 \ + --disaggregation-transfer-backend ascend \ --trust-remote-code \ --nnodes 1 \ --node-rank 0 \ @@ -90,7 +89,6 @@ python -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --quantization modelslim \ - --disaggregation-transfer-backend ascend \ --max-running-requests 8 \ --context-length 8192 \ --disable-radix-cache \ @@ -108,12 +106,13 @@ python -m sglang.launch_server \ --dtype bfloat16 ``` -Decode: +2. Decode: ```shell export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 -#PD + +#memfabric config store export ASCEND_MF_STORE_URL="tcp://:" #Deepep communication settings @@ -164,7 +163,7 @@ python -m sglang.launch_server \ --tokenizer-worker-num 4 ``` -sglang router: +3. SGLang Router ```shell python -m sglang_router.launch_router \ @@ -180,13 +179,13 @@ python -m sglang_router.launch_router \ W8A8 Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). -Prefill: +1. Prefill: ```shell export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 -#PD -P_HOST_IP=('xx,xx,xx,xx' 'xx,xx,xx,xx') + +#memfabric config store export ASCEND_MF_STORE_URL="tcp://:" #Deepep communication settings @@ -199,14 +198,18 @@ export SGLANG_USE_FIA_NZ=1 export ENABLE_MOE_NZ=1 export TASK_QUEUE_ENABLE=2 +#Please list all host ips of Prefill instance +P_HOST_IP=('xx,xx,xx,xx' 'xx,xx,xx,xx') + for i in "${!P_HOST_IP[@]}"; do python -m sglang.launch_server \ --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ --host ${P_HOST_IP[$i]} \ --port 8000 \ + --disaggregation-mode prefill \ --disaggregation-bootstrap-port $((8996+$i)) \ + --disaggregation-transfer-backend ascend \ --trust-remote-code \ --nnodes 1 \ --node-rank 0 \ @@ -215,7 +218,6 @@ do --attention-backend ascend \ --device npu \ --quantization modelslim \ - --disaggregation-transfer-backend ascend \ --max-running-requests 8 \ --context-length 8192 \ --disable-radix-cache \ @@ -234,12 +236,13 @@ do done ``` -Decode: +2. Decode: ```shell export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 -#PD + +#memfabric config store export ASCEND_MF_STORE_URL="tcp://:" #Deepep communication settings @@ -256,6 +259,7 @@ export SGLANG_NPU_USE_MLAPO=1 export SGLANG_USE_FIA_NZ=1 export ENABLE_MOE_NZ=1 +#please list all host ips of Prefill instance D_HOST_IP=('xx,xx,xx,xx' 'xx,xx,xx,xx') for i in "${!D_HOST_IP[@]}"; @@ -263,6 +267,7 @@ do python -m sglang.launch_server --model-path ${MODEL_PATH} \ --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ --host ${D_HOST_IP[$i]} \ --port 8001 \ --trust-remote-code \ @@ -281,7 +286,6 @@ do --deepep-mode low_latency \ --enable-dp-lm-head \ --cuda-graph-bs 8 10 12 14 16 18 20 22 24 26 \ - --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 8192 \ --speculative-algorithm NEXTN \ @@ -295,7 +299,7 @@ do done ``` -sglang router: +3. SGLang Router: ```shell python -m sglang_router.launch_router \ diff --git a/docs/platforms/ascend_npu_qwen3_examples.md b/docs/platforms/ascend_npu_qwen3_examples.md index 787dfb6a2..958ad8c97 100644 --- a/docs/platforms/ascend_npu_qwen3_examples.md +++ b/docs/platforms/ascend_npu_qwen3_examples.md @@ -13,13 +13,12 @@ export STREAMS_PER_DEVICE=32 export HCCL_BUFFSIZE=1536 export HCCL_OP_EXPANSION_MODE=AIV -ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 python -m sglang.launch_server \ +python -m sglang.launch_server \ --device npu \ --attention-backend ascend \ --trust-remote-code \ --tp-size 4 \ --model-path Qwen/Qwen3-32B \ - --port 30111 \ --mem-fraction-static 0.8 ``` @@ -43,7 +42,6 @@ python -m sglang.launch_server \ --trust-remote-code \ --tp-size 4 \ --model-path Qwen/Qwen3-32B \ - --port 30111 \ --mem-fraction-static 0.8 \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path Qwen/Qwen3-32B-Eagle3 \ @@ -66,13 +64,12 @@ export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DEEPEP_BF16_DISPATCH=1 export ENABLE_ASCEND_MOE_NZ=1 -ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 python -m sglang.launch_server \ +python -m sglang.launch_server \ --device npu \ --attention-backend ascend \ --trust-remote-code \ --tp-size 4 \ --model-path Qwen/Qwen3-30B-A3B \ - --port 30111 \ --mem-fraction-static 0.8 ``` @@ -96,7 +93,6 @@ python -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --watchdog-timeout 9000 \ - --port 30111 \ --mem-fraction-static 0.8 ``` @@ -111,14 +107,12 @@ export STREAMS_PER_DEVICE=32 export HCCL_BUFFSIZE=1536 export HCCL_OP_EXPANSION_MODE=AIV -ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 python -m sglang.launch_server \ - --device npu \ +python -m sglang.launch_server \ --enable-multimodal \ --attention-backend ascend \ --mm-attention-backend ascend_attn \ --trust-remote-code \ --tp-size 4 \ --model-path Qwen/Qwen3-VL-8B-Instruct \ - --port 30111 \ --mem-fraction-static 0.8 ``` diff --git a/docs/platforms/ascend_npu_support_features.md b/docs/platforms/ascend_npu_support_features.md index 6e13e105f..df1f8eae7 100644 --- a/docs/platforms/ascend_npu_support_features.md +++ b/docs/platforms/ascend_npu_support_features.md @@ -5,30 +5,31 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). ## Model and tokenizer -| Argument | Description | Defaults | Options | A2 Supported | A3 Supported | -|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|----------------------------------|:----------------------------------------:|:----------------------------------------:| -| `--model-path`
`--model` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | **√** | **√** | -| `--tokenizer-path` | The path of the tokenizer. | `None` | Type: str | **√** | **√** | -| `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | `auto` | `auto`, `slow` | **√** | **√** | -| `--tokenizer-worker-num` | The worker num of the tokenizer manager. | `1` | Type: int | **√** | **√** | -| `--skip-tokenizer-init` | If set, skip init tokenizer and pass input_ids in generate request. | `False` | bool flag (set to enable) | **√** | **√** | -| `--trust-remote-code` | Whether or not to allow for custom models defined on the Hub in their own modeling files. | `False` | bool flag (set to enable) | **√** | **√** | -| `--context-length` | The model's maximum context length. Defaults to None (will use the value from the model's config.json instead). | `None` | Type: int | **√** | **√** | -| `--is-embedding` | Whether to use a CausalLM as an embedding model. | `False` | bool flag (set to enable) | **√** | **√** | -| `--enable-multimodal` | Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen | `None` | bool flag (set to enable) | **√** | **√** | -| `--revision` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | **×** | **×** | -| `--model-impl` | Which implementation of the model to use. * “auto” will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available. * “sglang” will use the SGLang model implementation. * “transformers” will use the Transformers model implementation.* "mindspore" will use the MindSpore model implementation. | `None` | 'auto', 'sglang', 'transformers' | **×** | **√** | -| `--model-loader-extra-config` | Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format. | {} | Type: str' | **×** | **×** | +| Argument | Description | Defaults | Options | A2 Supported | A3 Supported | +|-------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|----------------------------------------------------------------------------------|:----------------------------------------:|:----------------------------------------:| +| `--model-path`
`--model` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | **√** | **√** | +| `--tokenizer-path` | The path of the tokenizer. | `None` | Type: str | **√** | **√** | +| `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | `auto` | `auto`, `slow` | **√** | **√** | +| `--tokenizer-worker-num` | The worker num of the tokenizer manager. | `1` | Type: int | **√** | **√** | +| `--skip-tokenizer-init` | If set, skip init tokenizer and pass input_ids in generate request. | `False` | bool flag (set to enable) | **√** | **√** | +| `--load-format` | The format of the model weights to load.
`auto` will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available.
`pt` will load the weights in the pytorch bin format.
`safetensors` will load the weights in the safetensors format.
`npcache` will load the weights in pytorch format and store a numpy cache to speed up the loading.
`dummy` will initialize the weights with random values, which is mainly for profiling.
`gguf` will load the weights in the gguf format.
`bitsandbytes` will load the weights using bitsandbytes quantization.
`layered` loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller. | `auto` | `auto`, `safetensors` | **√** | **√** | +| `--model-loader-extra-config` | Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format. | {} | Type: str
for example: {"enable_multithread_load": true,"num_threads": 64} | **√** | **√** | +| `--trust-remote-code` | Whether or not to allow for custom models defined on the Hub in their own modeling files. | `False` | bool flag (set to enable) | **√** | **√** | +| `--context-length` | The model's maximum context length. Defaults to None (will use the value from the model's config.json instead). | `None` | Type: int | **√** | **√** | +| `--is-embedding` | Whether to use a CausalLM as an embedding model. | `False` | bool flag (set to enable) | **√** | **√** | +| `--enable-multimodal` | Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen | `None` | bool flag (set to enable) | **√** | **√** | +| `--revision` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | **×** | **×** | +| `--model-impl` | Which implementation of the model to use.
`auto` will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available.
`sglang` will use the sglang model implementation.
`transformers` will use the transformers model implementation.
`mindspore` will use the MindSpore model implementation. | `None` | `auto`, `sglang`, `transformers` | **×** | **√** | ## HTTP server -| Argument | Description | Defaults | Options | A2 Supported | A3 Supported | -|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|---------------------------|:----------------------------------------:|:----------------------------------------:| -| `--host` | The host of the HTTP server. | `127.0.0.1` | Type: str | **√** | **√** | -| `--port` | The port of the HTTP server. | `30000` | Type: int | **√** | **√** | -| `--skip-server-warmup` | If set, skip warmup. | `False` | bool flag (set to enable) | **√** | **√** | -| `--warmups` | Specify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests | `None` | Type: str | **√** | **√** | -| `--nccl-port` | The port for NCCL distributed environment setup. Defaults to a random port. | `None` | Type: int | **√** | **√** | +| Argument | Description | Defaults | Options | A2 Supported | A3 Supported | +|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|---------------------------|:----------------------------------------:|:----------------------------------------:| +| `--host` | The host of the HTTP server. | `127.0.0.1` | Type: str | **√** | **√** | +| `--port` | The port of the HTTP server. | `30000` | Type: int | **√** | **√** | +| `--skip-server-warmup` | If set, skip warmup. | `False` | bool flag (set to enable) | **√** | **√** | +| `--warmups` | Specify custom warmup functions (csv) to run before server starts eg. `--warmups=warmup_name1,warmup_name2` will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests | `None` | Type: str | **√** | **√** | +| `--nccl-port` | The port for NCCL distributed environment setup. Defaults to a random port. | `None` | Type: int | **√** | **√** | ## Quantization and data type @@ -65,8 +66,8 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | Argument | Description | Defaults | Options | A2 Supported | A3 Supported | |---------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------------------|:----------------------------------------:|:----------------------------------------:| | `--device` | The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu'). Defaults to auto-detection if not specified. | `None` | Type: str | **√** | **√** | -| `--tensor-parallel-size`
`--tp-size` | The tensor parallelism size. | `1` | Type: int | **√** | **√** | -| `--pipeline-parallel-size`
`--pp-size` | The pipeline parallelism size. | `1` | Type: int | **×** | **×** | +| `--tensor-parallel-size`
`--tp-size` | The tensor parallelism size. | `1` | Type: int | **√** | **√** | +| `--pipeline-parallel-size`
`--pp-size` | The pipeline parallelism size. | `1` | Type: int | **×** | **×** | | `--pp-max-micro-batch-size` | The maximum micro batch size in pipeline parallelism. | `None` | Type: int | **×** | **×** | | `--stream-interval` | The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher | `1` | Type: int | **√** | **√** | | `--stream-output` | Whether to output as a sequence of disjoint segments. | `False` | bool flag (set to enable) | **√** | **√** | @@ -119,7 +120,7 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | `--completion-template` | The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | `None` | Type: str | **√** | **√** | | `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `True` | bool flag (set to enable) | **√** | **√** | | `--reasoning-parser` | Specify the parser for reasoning models. Supported parsers: [deepseek-r1, deepseek-v3, glm45, gpt-oss, kimi, qwen3, qwen3-thinking, step3]. | `None` | `deepseek-r1`, `deepseek-v3`, `glm45`, `gpt-oss`, `kimi`, `qwen3`, `qwen3-thinking`, `step3` | **√** | **√** | -| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Supported parsers: [ llama3,qwen]. | `None` | `llama3`,`qwen` | **√** | **√** | +| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Supported parsers: [ llama3, qwen]. | `None` | `llama3`,`qwen` | **√** | **√** | | `--sampling-defaults` | Where to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'. | `model` | `openai`, `model` | **×** | **×** | | `--tool-server` | Either 'demo' or a comma-separated list of tool server urls to use for the model. If not specified, no tool server will be used. | `None` | Type: str | **×** | **×** | @@ -127,7 +128,7 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | Argument | Description | Defaults | Options | A2 Supported | A3 Supported | |---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------|:----------------------------------------:|:----------------------------------------:| -| `--data-parallel-size`
`--dp-size` | The data parallelism size. | `1` | Type: int | **√** | **√** | +| `--data-parallel-size`
`--dp-size` | The data parallelism size. | `1` | Type: int | **√** | **√** | | `--load-balance-method` | The load balancing strategy for data parallelism. The Minimum Token algorithm can only be used when DP attention is applied. This algorithm performs load balancing based on the real-time token load of the DP workers. | `round_robin` | `round_robin`, `shortest_queue`, `minimum_tokens` | **√** | **√** | | `--prefill-round-robin-balance` | Prefill is round robin balanced. This is used to promise decode server can get the correct dp rank. | `False` | bool flag (set to enable) | **√** | **√** | @@ -135,7 +136,7 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | Argument | Description | Defaults | Options | A2 Supported | A3 Supported | |------------------------------------------|------------------------------------------------------------------------------------|----------|-----------|:----------------------------------------:|:----------------------------------------:| -| `--dist-init-addr`
`--nccl-init-addr` | The host address for initializing distributed backend (e.g., `192.168.0.2:25000`). | `None` | Type: str | **√** | **√** | +| `--dist-init-addr`
`--nccl-init-addr` | The host address for initializing distributed backend (e.g., `192.168.0.2:25000`). | `None` | Type: str | **√** | **√** | | `--nnodes` | The number of nodes. | `1` | Type: int | **√** | **√** | | `--node-rank` | The node rank. | `0` | Type: int | **√** | **√** | @@ -180,7 +181,7 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | Argument | Description | Defaults | Options | A2 Supported | A3 Supported | |-----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|-----------|---------------------------------------------------|:----------------------------------------:|:----------------------------------------:| | `--speculative-algorithm` | Speculative algorithm. | `None` | `EAGLE`, `EAGLE3`, `NEXTN`, `STANDALONE`, `NGRAM` | **√** | **√** | -| `--speculative-draft-model-path`
`--speculative-draft-model` | The path of the draft model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | **√** | **√** | +| `--speculative-draft-model-path`
`--speculative-draft-model` | The path of the draft model weights. This can be a local folder or a Hugging Face repo ID. | `None` | Type: str | **√** | **√** | | `--speculative-draft-model-revision` | The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | `None` | Type: str | **×** | **×** | | `--speculative-num-steps` | The number of steps sampled from draft model in Speculative Decoding. | `None` | Type: int | **√** | **√** | | `--speculative-eagle-topk` | The number of tokens sampled from the draft model in eagle2 each step. | `None` | Type: int | **√** | **√** | @@ -208,12 +209,13 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | Argument | Description | Defaults | Options | A2 Supported | A3 Supported | |---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|-----------------------------------|:----------------------------------------:|:----------------------------------------:| -| `--expert-parallel-size`
`--ep-size`
`--ep` | The expert parallelism size. Default equal to tp size. | `tp-size` | Type: int | **×** | **√** | +| `--expert-parallel-size`
`--ep-size`
`--ep` | The expert parallelism size. Default equal to tp size. | `tp-size` | Type: int | **×** | **√** | | `--moe-a2a-backend` | Select the backend for all-to-all communication for expert parallelism. | `none` | `none`, `deepep`, `ascend_fuseep` | **×** | **√** | | `--moe-runner-backend` | Choose the runner backend for MoE. | `auto` | `auto` | **×** | **√** | | `--flashinfer-mxfp4-moe-precision` | Choose the computation precision of flashinfer mxfp4 moe | `default` | `default`, `bf16` | **×** | **×** | | `--enable-flashinfer-allreduce-fusion` | Enable FlashInfer allreduce fusion with Residual RMSNorm. | `False` | bool flag (set to enable) | **×** | **×** | | `--deepep-mode` | Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch. | `auto` | `normal`, `low_latency`, `auto` | **×** | **√** | +| `--deepep-config` | Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path. | `None` | Type: str | **×** | **×** | | `--ep-num-redundant-experts` | Allocate this number of redundant experts in expert parallel. | `0` | Type: int | **×** | **×** | | `--ep-dispatch-algorithm` | The algorithm to choose ranks for redundant experts in expert parallel. | `None` | Type: str | **×** | **×** | | `--init-expert-location` | Initial location of EP experts. | `trivial` | Type: str | **×** | **×** | @@ -224,7 +226,6 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | `--expert-distribution-recorder-mode` | Mode of expert distribution recorder. | `None` | Type: str | **×** | **×** | | `--expert-distribution-recorder-buffer-size` | Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer. | `None` | Type: int | **×** | **×** | | `--enable-expert-distribution-metrics` | Enable logging metrics for expert balancedness | `False` | bool flag (set to enable) | **×** | **×** | -| `--deepep-config` | Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path. | `None` | Type: str | **×** | **×** | | `--moe-dense-tp-size` | TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports. | `None` | Type: int | **√** | **√** | | `--elastic-ep-backend` | Select the collective communication backend for elastic EP. Currently supports 'mooncake'. | None | N/A | **×** | **×** | | `--mooncake-ib-device` | The InfiniBand devices for Mooncake Backend, accepts multiple comma-separated devices. Default is None, which triggers automatic device detection when Mooncake Backend is enabled. | None | N/A | **×** | **×** | @@ -236,7 +237,7 @@ questions, please [open an issue](https://github.com/sgl-project/sglang/issues). | `--max-mamba-cache-size` | The maximum size of the mamba cache. | `None` | Type: int | **×** | **×** | | `--mamba-ssm-dtype` | The data type of the SSM states in mamba cache. | `float32` | `float32`, `bfloat16` | **×** | **×** | | `--mamba-full-memory-ratio` | The ratio of mamba state memory to full kv cache memory. | `0.2` | Type: float | **×** | **×** | -| `--mamba-scheduler-strategy` | The strategy to use for mamba scheduler. `auto` currently defaults to `no_buffer`. 1. `no_buffer` does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented. 2. `extra_buffer` supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes `2x` for non-spec; `1+(1/(2+speculative_num_draft_tokens))x` for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)). 2a. `extra_buffer` is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests. 2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support (currently only FLA backend), currently only extra_buffer supports branching. | `auto` | `auto`, `no_buffer`, `extra_buffer` | **×** | **×** | +| `--mamba-scheduler-strategy` | The strategy to use for mamba scheduler.
`auto` currently defaults to `no_buffer`.
1. `no_buffer` does not support overlap scheduler due to not allocating extra mamba state buffers. Branching point caching support is feasible but not implemented.
2. `extra_buffer` supports overlap schedule by allocating extra mamba state buffers to track mamba state for caching (mamba state usage per running req becomes `2x` for non-spec; `1+(1/(2+speculative_num_draft_tokens))x` for spec dec (e.g. 1.16x if speculative_num_draft_tokens==4)).
2a. `extra_buffer` is strictly better for non-KV-cache-bound cases; for KV-cache-bound cases, the tradeoff depends on whether enabling overlap outweighs reduced max running requests.
2b. mamba caching at radix cache branching point is strictly better than non-branch but requires kernel support (currently only FLA backend), currently only extra_buffer supports branching. | `auto` | `auto`, `no_buffer`, `extra_buffer` | **×** | **×** | | `--mamba-track-interval` | The interval (in tokens) to track the mamba state during decode. Only used when `--mamba-scheduler-strategy` is `extra_buffer`. Must be divisible by page_size if set, and must be >= speculative_num_draft_tokens when using speculative decoding. | `256` | Type: int | **×** | **×** | ## Args for multi-item scoring diff --git a/docs/platforms/ascend_npu_support_models.md b/docs/platforms/ascend_npu_support_models.md index 72be5e62d..42ad6aec7 100644 --- a/docs/platforms/ascend_npu_support_models.md +++ b/docs/platforms/ascend_npu_support_models.md @@ -1,83 +1,97 @@ # Support Models on Ascend NPU This section describes the models supported on the Ascend NPU, including Large Language Models, Multimodal Language -Models, Embedding Models, and Rerank Models. Mainstream DeepSeek/Qwen/GLM series are included. You are welcome to enable -various models based on your business requirements. +Models, Embedding Models, and Rerank Models. Mainstream DeepSeek/Qwen/GLM series are included. +You are welcome to enable various models based on your business requirements. ## Large Language Models -| Model Family | Recommend Models | A2 Supported | A3 Supported | -|--------------------------------|--------------------------------------------------------------------------------------------------------------------------|:----------------------------------------:|:----------------------------------------:| -| DeepSeek | DeepSeek V1, V2, V3(V3.1,V3.2), R1 | **√** | **√** | -| Qwen | Qwen 3, Qwen 3Moe | **√** | **√** | -| Llama | meta-llama/Llama-4-Scout-17B-16E-Instruct,
AI-ModelScope/Llama-3.1-8B-Instruct,
LLM-Research/Llama-3.2-1B-Instruct | **×** | **×** | -| Mistral | mistralai/Mistral-7B-Instruct-v0.2 | **√** | **√** | -| Gemma | google/gemma-3-4b-it | **√** | **√** | -| Phi | microsoft/Phi-4-multimodal-instruct | **√** | **√** | -| OLMoE | allenai/OLMoE-1B-7B-0924 | **×** | **×** | -| StableLM | stabilityai/stablelm-2-1_6b | **×** | **×** | -| Command-R | CohereForAI/c4ai-command-r-v01 | **×** | **×** | -| Grok | huihui-ai/grok-2 | **×** | **×** | -| ChatGLM | ZhipuAI/chatglm2-6b | **×** | **×** | -| InternLM 2 | Shanghai_AI_Laboratory/internlm2-7b | **√** | **√** | -| ExaONE 3 | LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct | **√** | **√** | -| XVERSE | xverse/XVERSE-MoE-A36B | **√** | **√** | -| SmolLM | HuggingFaceTB/SmolLM-1.7B | **√** | **√** | -| GLM-4 | ZhipuAI/glm-4-9b-chat | **×** | **×** | -| MiMo | XiaomiMiMo/MiMo-7B-RL | **√** | **√** | -| Arcee AFM-4.5B | arcee-ai/AFM-4.5B-Base | **√** | **√** | -| Persimmon | Howeee/persimmon-8b-chat | **√** | **√** | -| Ling | inclusionAI/Ling-lite | **√** | **√** | -| Granite | ibm-granite/granite-3.1-8b-instruct | **√** | **√** | -| Granite Moe | ibm-granite/granite-3.0-3b-a800m-instruct | **√** | **√** | -| DBRX (Databricks) | databricks/dbrx-instruct | **×** | **×** | -| Baichuan 2 (7B, 13B) | baichuan-inc/Baichuan2-13B-Chat | **×** | **×** | -| ERNIE-4.5 (4.5, 4.5MoE series) | baidu/ERNIE-4.5-21B-A3B-PT | **×** | **×** | -| MiniCPM (v3, 4B) | openbmb/MiniCPM3-4B | **×** | **×** | -| GPTOSS | openai/gpt-oss-120b | **×** | **×** | +| Models | Model Family | A2 Supported | A3 Supported | +|-------------------------------------------|--------------------------------|:----------------------------------------:|:----------------------------------------:| +| DeepSeek V3/V3.1 | DeepSeek | **√** | **√** | +| DeepSeek V3.2 | DeepSeek | **×** | **√** | +| DeepSeek R1 | DeepSeek | **√** | **√** | +| DeepSeek V2 | DeepSeek | **√** | **√** | +| Qwen3 | Qwen | **√** | **√** | +| Qwen3-MoE | Qwen | **√** | **√** | +| Qwen3-Next | Qwen | **√** | **√** | +| Qwen3-Coder | Qwen | **√** | **√** | +| Qwen2.5 | Qwen | **√** | **√** | +| QwQ-32B | Qwen | **√** | **√** | +| meta-llama/Llama-4-Scout-17B-16E-Instruct | Llama | **√** | **√** | +| AI-ModelScope/Llama-3.1-8B-Instruct | Llama | **√** | **√** | +| LLM-Research/Llama-3.2-1B-Instruct | Llama | **√** | **√** | +| mistralai/Mistral-7B-Instruct-v0.2 | Mistral | **√** | **√** | +| google/gemma-3-4b-it | Gemma | **√** | **√** | +| microsoft/Phi-4-multimodal-instruct | Phi | **√** | **√** | +| allenai/OLMoE-1B-7B-0924 | OLMoE | **√** | **√** | +| stabilityai/stablelm-2-1_6b | StableLM | **√** | **√** | +| CohereForAI/c4ai-command-r-v01 | Command-R | **×** | **√** | +| huihui-ai/grok-2 | Grok | **×** | **√** | +| ZhipuAI/chatglm2-6b | ChatGLM | **×** | **√** | +| Shanghai_AI_Laboratory/internlm2-7b | InternLM 2 | **√** | **√** | +| LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct | ExaONE 3 | **√** | **√** | +| xverse/XVERSE-MoE-A36B | XVERSE | **√** | **√** | +| HuggingFaceTB/SmolLM-1.7B | SmolLM | **√** | **√** | +| ZhipuAI/glm-4-9b-chat | GLM-4 | **√** | **√** | +| XiaomiMiMo/MiMo-7B-RL | MiMo | **√** | **√** | +| arcee-ai/AFM-4.5B-Base | Arcee AFM-4.5B | **√** | **√** | +| Howeee/persimmon-8b-chat | Persimmon | **√** | **√** | +| inclusionAI/Ling-lite | Ling | **√** | **√** | +| ibm-granite/granite-3.1-8b-instruct | Granite | **√** | **√** | +| ibm-granite/granite-3.0-3b-a800m-instruct | Granite MoE | **√** | **√** | +| databricks/dbrx-instruct | DBRX (Databricks) | **×** | **×** | +| baichuan-inc/Baichuan2-13B-Chat | Baichuan 2 (7B, 13B) | **×** | **×** | +| baidu/ERNIE-4.5-21B-A3B-PT | ERNIE-4.5 (4.5, 4.5MoE series) | **×** | **×** | +| openbmb/MiniCPM3-4B | MiniCPM (v3, 4B) | **×** | **×** | +| openai/gpt-oss-120b | GPTOSS | **×** | **×** | ## Multimodal Language Models -| Model Family | Recommend Models | A2 Supported | A3 Supported | -|--------------------------------|-----------------------------------------------|:----------------------------------------:|:----------------------------------------:| -| Qwen-VL (Qwen2 series) | Qwen/Qwen3-VL-235B-A22B-Instruct | **×** | **×** | -| DeepSeek-VL2 | deepseek-ai/deepseek-vl2 | **×** | **×** | -| Janus-Pro (1B, 7B) | deepseek-ai/Janus-Pro-7B | **√** | **√** | -| MiniCPM-V / MiniCPM-o | openbmb/MiniCPM-V-2_6 | **×** | **×** | -| Gemma 3 (Multimodal) | google/gemma-3-4b-it | **√** | **√** | -| Mistral-Small-3.1-24B | mistralai/Mistral-Small-3.1-24B-Instruct-2503 | **×** | **×** | -| Phi-4-multimodal-instruct | microsoft/Phi-4-multimodal-instruct | **×** | **×** | -| MiMo-VL (7B) | XiaomiMiMo/MiMo-VL-7B-RL | **×** | **×** | -| LLaVA (v1.5 & v1.6) | AI-ModelScope/llava-v1.6-34b | **√** | **√** | -| LLaVA-NeXT (8B, 72B) | lmms-lab/llava-next-72b | **√** | **√** | -| LLaVA-OneVision | lmms-lab/llava-onevision-qwen2-7b-ov | **×** | **×** | -| Kimi-VL (A3B) | Kimi/Kimi-VL-A3B-Instruct | **×** | **×** | -| GLM-4.5V (106B) / GLM-4.1V(9B) | ZhipuAI/GLM-4.5V | **×** | **√** | -| Llama 3.2 Vision (11B) | meta-llama/Llama-3.2-11B-Vision-Instruct | **×** | **×** | +| Models | Model Family (Variants) | A2 Supported | A3 Supported | +|-----------------------------------------------|---------------------------|------------------------------------------|:----------------------------------------:| +| Qwen2.5-VL-72B-Instruct-w8a8 | Qwen-VL | **×** | **√** | +| Qwen3-VL-30B-A3B-Instruct | Qwen-VL | **×** | **√** | +| Qwen3-VL-8B-Instruct | Qwen-VL | **×** | **√** | +| Qwen3-VL-4B-Instruct | Qwen-VL | **×** | **√** | +| Qwen3-VL-235B-A22B-Instruct | Qwen-VL | **×** | **√** | +| deepseek-ai/deepseek-vl2 | DeepSeek-VL2 | **×** | **√** | +| deepseek-ai/Janus-Pro-7B | Janus-Pro (1B, 7B) | **√** | **√** | +| openbmb/MiniCPM-V-2_6 | MiniCPM-V / MiniCPM-o | **×** | **√** | +| google/gemma-3-4b-it | Gemma 3 (Multimodal) | **√** | **√** | +| mistralai/Mistral-Small-3.1-24B-Instruct-2503 | Mistral-Small-3.1-24B | **×** | **√** | +| microsoft/Phi-4-multimodal-instruct | Phi-4-multimodal-instruct | **×** | **√** | +| XiaomiMiMo/MiMo-VL-7B-RL | MiMo-VL (7B) | **×** | **√** | +| AI-ModelScope/llava-v1.6-34b | LLaVA (v1.5 & v1.6) | **√** | **√** | +| lmms-lab/llava-next-72b | LLaVA-NeXT (8B, 72B) | **√** | **√** | +| lmms-lab/llava-onevision-qwen2-7b-ov | LLaVA-OneVision | **×** | **√** | +| Kimi/Kimi-VL-A3B-Instruct | Kimi-VL (A3B) | **×** | **√** | +| ZhipuAI/GLM-4.5V | GLM-4.5V (106B) | **×** | **√** | +| meta-llama/Llama-3.2-11B-Vision-Instruct | Llama 3.2 Vision (11B) | **×** | **×** | ## Embedding Models -| Model Family | Recommend Models | A2 Supported | A3 Supported | -|--------------------------|------------------------------------------|:--------------------------------------:|:----------------------------------------:| -| E5 (Llama/Mistral based) | intfloat/e5-mistral-7b-instruct | **×** | **×** | -| GTE-Qwen2 | iic/gte_Qwen2-1.5B-instruct | **×** | **×** | -| Qwen3-Embedding | Qwen/Qwen3-Embedding-8B | **×** | **×** | -| GME (Multimodal) | Alibaba-NLP/gme-Qwen2-VL-2B-Instruct | **×** | **×** | -| CLIP | AI-ModelScope/clip-vit-large-patch14-336 | **×** | **√** | -| BGE | BAAI/bge-large-en-v1.5 | **×** | **×** | +| Models | Model Family | A2 Supported | A3 Supported | +|-------------------------------------------|--------------------------|------------------------------------------|:----------------------------------------:| +| intfloat/e5-mistral-7b-instruct | E5 (Llama/Mistral based) | **×** | **√** | +| iic/gte_Qwen2-1.5B-instruct | GTE-Qwen2 | **×** | **√** | +| Qwen/Qwen3-Embedding-8B | Qwen3-Embedding | **×** | **×** | +| Alibaba-NLP/gme-Qwen2-VL-2B-Instruct | GME (Multimodal) | **×** | **√** | +| AI-ModelScope/clip-vit-large-patch14-336 | CLIP | **√** | **√** | +| BAAI/bge-large-en-v1.5 | BGE | **×** | **×** | ## Reward Models -| Model Family | Recommend Models | A2 Supported | A3 Supported | -|---------------------------|--------------------------------------------|:--------------------------------------:|:----------------------------------------:| -| Llama3.1 Reward | Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 | **×** | **√** | -| InternLM 2 Reward | Shanghai_AI_Laboratory/internlm2-7b-reward | **×** | **√** | -| Qwen2.5 Reward - Math | Qwen/Qwen2.5-Math-RM-72B | **×** | **√** | -| Qwen2.5 Reward - Sequence | jason9693/Qwen2.5-1.5B-apeach | **×** | **√** | -| Gemma 2-27B Reward | Skywork/Skywork-Reward-Gemma-2-27B-v0.2 | **×** | **×** | +| Models | Model Family | A2 Supported | A3 Supported | +|---------------------------------------------|---------------------------|----------------------------------------|:----------------------------------------:| +| Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 | Llama3.1 Reward | **×** | **√** | +| Shanghai_AI_Laboratory/internlm2-7b-reward | InternLM 2 Reward | **×** | **√** | +| Qwen/Qwen2.5-Math-RM-72B | Qwen2.5 Reward - Math | **×** | **√** | +| jason9693/Qwen2.5-1.5B-apeach | Qwen2.5 Reward - Sequence | **×** | **√** | +| Skywork/Skywork-Reward-Gemma-2-27B-v0.2 | Gemma 2-27B Reward | **×** | **×** | ## Rerank Models -| Model Family | Recommend Models | A2 Supported | A3 Supported | -|--------------|-------------------------|:--------------------------------------:|:--------------------------------------:| -| BGE-Reranker | BAAI/bge-reranker-v2-m3 | **×** | **×** | +| Models | Model Family | A2 Supported | A3 Supported | +|-------------------------|--------------|:--------------------------------------:|:--------------------------------------:| +| BAAI/bge-reranker-v2-m3 | BGE-Reranker | **×** | **×** |