diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 9681a1d70..8f3441b1b 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -43,6 +43,7 @@ import logging import re import time from enum import Enum, auto +from functools import lru_cache from http import HTTPStatus from itertools import chain from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple, Union @@ -95,10 +96,28 @@ if TYPE_CHECKING: INIT_INCREMENTAL_DETOKENIZATION_OFFSET = 5 +# Constant used as the base offset for MM (multimodal) pad values. +# This ensures pad_values don't overlap with valid text token IDs. +MM_PAD_SHIFT_VALUE = 1_000_000 logger = logging.getLogger(__name__) +@lru_cache(maxsize=1) +def sanity_check_mm_pad_shift_value(vocab_size: int) -> None: + if vocab_size > MM_PAD_SHIFT_VALUE: + raise ValueError( + f"Model vocab_size ({vocab_size}) exceeds MM_PAD_SHIFT_VALUE ({MM_PAD_SHIFT_VALUE}). " + f"MM pad_values may overlap with valid token IDs. " + f"Please increase MM_PAD_SHIFT_VALUE in schedule_batch.py." + ) + + +def _compute_pad_value(hash: int) -> int: + """Compute pad value from hash.""" + return MM_PAD_SHIFT_VALUE + (hash % (1 << 30)) + + class BaseFinishReason: def __init__(self, is_error: bool = False): self.is_error = is_error @@ -262,7 +281,7 @@ class MultimodalDataItem: import uuid self.hash = uuid.uuid4().int - self.pad_value = self.hash % (1 << 30) + self.pad_value = _compute_pad_value(self.hash) return if self.hash is None: if self.feature is not None: @@ -271,7 +290,7 @@ class MultimodalDataItem: hashed_feature = self.precomputed_embeddings self.hash = hash_feature(hashed_feature) assert self.hash is not None - self.pad_value = self.hash % (1 << 30) + self.pad_value = _compute_pad_value(self.hash) def is_modality(self, modality: Modality) -> bool: return self.modality == modality diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 11d60729d..01d6c8866 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -106,6 +106,7 @@ from sglang.srt.layers.sampler import create_sampler from sglang.srt.layers.torchao_utils import apply_torchao_config_to_model from sglang.srt.lora.lora_manager import LoRAManager from sglang.srt.lora.lora_registry import LoRARef +from sglang.srt.managers.schedule_batch import sanity_check_mm_pad_shift_value from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.model_executor.cpu_graph_runner import CPUGraphRunner @@ -390,6 +391,9 @@ class ModelRunner(ModelRunnerKVCacheMixin): self.initialize(min_per_gpu_memory) self.check_quantized_moe_compatibility() + if self.is_multimodal: + sanity_check_mm_pad_shift_value(self.model_config.vocab_size) + # Temporary cached values self.support_pp = ( "pp_proxy_tensors" in inspect.signature(self.model.forward).parameters