chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: Handle multiple named chat templates in HuggingFace tokenizers (#17236)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Xinyuan Tong
2026-01-18 09:20:04 +00:00
committed by GitHub
parent 1fe0c82f67
commit 2069050d3f
3 changed files with 72 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
docs/advanced_features/server_arguments.md
View File

@@ -56,7 +56,7 @@ You can find all arguments by `python3 -m sglang.launch_server --help`
- To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments.
- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`.
- To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](../advanced_features/deterministic_inference.md).
- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md).
- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). If the tokenizer has multiple named templates (e.g., 'default', 'tool_use'), you can select one using `--hf-chat-template-name tool_use`.
- To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph`
```bash
@@ -216,6 +216,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
| `--served-model-name` | Override the model name returned by the v1/models endpoint in OpenAI API server. | `None` | Type: str |
| `--weight-version` | Version identifier for the model weights. Defaults to 'default' if not specified. | `default` | Type: str |
| `--chat-template` | The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. | `None` | Type: str |
| `--hf-chat-template-name` | When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used. | `None` | Type: str |
| `--completion-template` | The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | `None` | Type: str |
| `--file-storage-path` | The path of the file storage in backend. | `sglang_storage` | Type: str |
| `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `False` | bool flag (set to enable) |

83
python/sglang/srt/managers/template_manager.py
View File

@@ -22,8 +22,9 @@ import json
import logging
import os
import re
from typing import Optional
from typing import Dict, Optional
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.parser.code_completion_parser import (
CompletionTemplate,
FimPosition,
@@ -99,7 +100,10 @@ class TemplateManager:
return has_reasoning
def load_chat_template(
self, tokenizer_manager, chat_template_arg: Optional[str], model_path: str
self,
tokenizer_manager: TokenizerManager,
chat_template_arg: Optional[str],
model_path: str,
) -> None:
"""
Load a chat template from various sources.
@@ -143,7 +147,7 @@ class TemplateManager:
)
def _load_explicit_chat_template(
self, tokenizer_manager, chat_template_arg: str
self, tokenizer_manager: TokenizerManager, chat_template_arg: str
) -> None:
"""Load explicitly specified chat template."""
logger.info(f"Loading chat template from argument: {chat_template_arg}")
@@ -197,7 +201,7 @@ class TemplateManager:
def initialize_templates(
self,
tokenizer_manager,
tokenizer_manager: TokenizerManager,
model_path: str,
chat_template: Optional[str] = None,
completion_template: Optional[str] = None,
@@ -218,7 +222,9 @@ class TemplateManager:
if completion_template:
self.load_completion_template(completion_template)
def _load_jinja_template(self, tokenizer_manager, template_path: str) -> None:
def _load_jinja_template(
self, tokenizer_manager: TokenizerManager, template_path: str
) -> None:
"""Load a Jinja template file."""
with open(template_path, "r") as f:
chat_template = "".join(f.readlines()).strip("\n")
@@ -288,21 +294,56 @@ class TemplateManager:
)
self._completion_template_name = template["name"]
def _resolve_hf_chat_template(self, tokenizer_manager) -> Optional[str]:
"""
Resolve HuggingFace chat template.
Returns the chat template string if found, None otherwise.
"""
def _resolve_hf_chat_template(
self, tokenizer_manager: TokenizerManager
) -> Optional[str]:
try:
if processor := tokenizer_manager.processor:
if hasattr(processor, "chat_template") and processor.chat_template:
return processor.chat_template
if tokenizer := tokenizer_manager.tokenizer:
if hasattr(tokenizer, "chat_template") and tokenizer.chat_template:
return tokenizer.chat_template
except Exception as e:
logger.debug(f"Error getting chat template: {e}")
# Try (mm-)processor first, then tokenizer
template = (
getattr(tokenizer_manager.processor, "chat_template", None)
if tokenizer_manager.processor
else None
) or (
getattr(tokenizer_manager.tokenizer, "chat_template", None)
if tokenizer_manager.tokenizer
else None
)
logger.debug("No HuggingFace chat template found")
return None
if template is None:
logger.warning("No HuggingFace chat template found")
return None
# Handle dict templates (multiple named templates)
if isinstance(template, dict):
return self._select_named_template(template, tokenizer_manager)
# Single string template
return template
except Exception as e:
logger.warning(f"Error getting chat template: {e}")
return None
def _select_named_template(
self, templates: Dict[str, str], tokenizer_manager: TokenizerManager
) -> str:
if not templates:
raise ValueError("Empty templates dict provided")
available_names = list(templates.keys())
logger.info(f"Multiple HuggingFace chat templates available: {available_names}")
# Use specified template if provided
if preferred_name := tokenizer_manager.server_args.hf_chat_template_name:
if preferred_name not in templates:
raise ValueError(
f"Specified template '{preferred_name}' not found. "
f"Available templates: {available_names}"
)
logger.info(f"Using specified chat template: '{preferred_name}'")
return templates[preferred_name]
# Fallback: Use first available template
first_name = available_names[0]
logger.info(f"Using first available template: '{first_name}'")
return templates[first_name]

8
python/sglang/srt/server_args.py
View File

@@ -380,6 +380,7 @@ class ServerArgs:
served_model_name: Optional[str] = None
weight_version: str = "default"
chat_template: Optional[str] = None
hf_chat_template_name: Optional[str] = None
completion_template: Optional[str] = None
file_storage_path: str = "sglang_storage"
enable_cache_report: bool = False
@@ -3291,6 +3292,13 @@ class ServerArgs:
default=ServerArgs.chat_template,
help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.",
)
parser.add_argument(
"--hf-chat-template-name",
type=str,
default=ServerArgs.hf_chat_template_name,
help="When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), "
"specify which named template to use. If not set, the first available template is used.",
)
parser.add_argument(
"--completion-template",
type=str,