diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md index efd73a4e4..857e8e306 100644 --- a/docs/advanced_features/server_arguments.md +++ b/docs/advanced_features/server_arguments.md @@ -56,7 +56,7 @@ You can find all arguments by `python3 -m sglang.launch_server --help` - To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments. - To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`. - To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](../advanced_features/deterministic_inference.md). -- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). +- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). If the tokenizer has multiple named templates (e.g., 'default', 'tool_use'), you can select one using `--hf-chat-template-name tool_use`. - To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph` ```bash @@ -216,6 +216,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | `--served-model-name` | Override the model name returned by the v1/models endpoint in OpenAI API server. | `None` | Type: str | | `--weight-version` | Version identifier for the model weights. Defaults to 'default' if not specified. | `default` | Type: str | | `--chat-template` | The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. | `None` | Type: str | +| `--hf-chat-template-name` | When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used. | `None` | Type: str | | `--completion-template` | The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | `None` | Type: str | | `--file-storage-path` | The path of the file storage in backend. | `sglang_storage` | Type: str | | `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `False` | bool flag (set to enable) | diff --git a/python/sglang/srt/managers/template_manager.py b/python/sglang/srt/managers/template_manager.py index 1d9bbea81..dd9bbcc55 100644 --- a/python/sglang/srt/managers/template_manager.py +++ b/python/sglang/srt/managers/template_manager.py @@ -22,8 +22,9 @@ import json import logging import os import re -from typing import Optional +from typing import Dict, Optional +from sglang.srt.managers.tokenizer_manager import TokenizerManager from sglang.srt.parser.code_completion_parser import ( CompletionTemplate, FimPosition, @@ -99,7 +100,10 @@ class TemplateManager: return has_reasoning def load_chat_template( - self, tokenizer_manager, chat_template_arg: Optional[str], model_path: str + self, + tokenizer_manager: TokenizerManager, + chat_template_arg: Optional[str], + model_path: str, ) -> None: """ Load a chat template from various sources. @@ -143,7 +147,7 @@ class TemplateManager: ) def _load_explicit_chat_template( - self, tokenizer_manager, chat_template_arg: str + self, tokenizer_manager: TokenizerManager, chat_template_arg: str ) -> None: """Load explicitly specified chat template.""" logger.info(f"Loading chat template from argument: {chat_template_arg}") @@ -197,7 +201,7 @@ class TemplateManager: def initialize_templates( self, - tokenizer_manager, + tokenizer_manager: TokenizerManager, model_path: str, chat_template: Optional[str] = None, completion_template: Optional[str] = None, @@ -218,7 +222,9 @@ class TemplateManager: if completion_template: self.load_completion_template(completion_template) - def _load_jinja_template(self, tokenizer_manager, template_path: str) -> None: + def _load_jinja_template( + self, tokenizer_manager: TokenizerManager, template_path: str + ) -> None: """Load a Jinja template file.""" with open(template_path, "r") as f: chat_template = "".join(f.readlines()).strip("\n") @@ -288,21 +294,56 @@ class TemplateManager: ) self._completion_template_name = template["name"] - def _resolve_hf_chat_template(self, tokenizer_manager) -> Optional[str]: - """ - Resolve HuggingFace chat template. - - Returns the chat template string if found, None otherwise. - """ + def _resolve_hf_chat_template( + self, tokenizer_manager: TokenizerManager + ) -> Optional[str]: try: - if processor := tokenizer_manager.processor: - if hasattr(processor, "chat_template") and processor.chat_template: - return processor.chat_template - if tokenizer := tokenizer_manager.tokenizer: - if hasattr(tokenizer, "chat_template") and tokenizer.chat_template: - return tokenizer.chat_template - except Exception as e: - logger.debug(f"Error getting chat template: {e}") + # Try (mm-)processor first, then tokenizer + template = ( + getattr(tokenizer_manager.processor, "chat_template", None) + if tokenizer_manager.processor + else None + ) or ( + getattr(tokenizer_manager.tokenizer, "chat_template", None) + if tokenizer_manager.tokenizer + else None + ) - logger.debug("No HuggingFace chat template found") - return None + if template is None: + logger.warning("No HuggingFace chat template found") + return None + + # Handle dict templates (multiple named templates) + if isinstance(template, dict): + return self._select_named_template(template, tokenizer_manager) + + # Single string template + return template + + except Exception as e: + logger.warning(f"Error getting chat template: {e}") + return None + + def _select_named_template( + self, templates: Dict[str, str], tokenizer_manager: TokenizerManager + ) -> str: + if not templates: + raise ValueError("Empty templates dict provided") + + available_names = list(templates.keys()) + logger.info(f"Multiple HuggingFace chat templates available: {available_names}") + + # Use specified template if provided + if preferred_name := tokenizer_manager.server_args.hf_chat_template_name: + if preferred_name not in templates: + raise ValueError( + f"Specified template '{preferred_name}' not found. " + f"Available templates: {available_names}" + ) + logger.info(f"Using specified chat template: '{preferred_name}'") + return templates[preferred_name] + + # Fallback: Use first available template + first_name = available_names[0] + logger.info(f"Using first available template: '{first_name}'") + return templates[first_name] diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 6fd803654..aee922368 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -380,6 +380,7 @@ class ServerArgs: served_model_name: Optional[str] = None weight_version: str = "default" chat_template: Optional[str] = None + hf_chat_template_name: Optional[str] = None completion_template: Optional[str] = None file_storage_path: str = "sglang_storage" enable_cache_report: bool = False @@ -3291,6 +3292,13 @@ class ServerArgs: default=ServerArgs.chat_template, help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.", ) + parser.add_argument( + "--hf-chat-template-name", + type=str, + default=ServerArgs.hf_chat_template_name, + help="When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), " + "specify which named template to use. If not set, the first available template is used.", + ) parser.add_argument( "--completion-template", type=str,