From 479ab7a4e7e40d4227d892f9d3da871424a316bb Mon Sep 17 00:00:00 2001 From: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:57:00 +0800 Subject: [PATCH] model: support Kimi-K2.5 (#17789) Co-authored-by: Mick --- python/sglang/srt/configs/__init__.py | 2 + python/sglang/srt/configs/kimi_k25.py | 171 ++++ python/sglang/srt/configs/model_config.py | 20 +- .../srt/entrypoints/openai/serving_chat.py | 2 +- python/sglang/srt/layers/attention/vision.py | 11 +- python/sglang/srt/models/kimi_k25.py | 744 ++++++++++++++++++ .../multimodal/processors/base_processor.py | 1 + .../srt/multimodal/processors/kimi_k25.py | 88 +++ python/sglang/srt/parser/reasoning_parser.py | 22 +- python/sglang/srt/server_args.py | 2 + .../sglang/srt/utils/hf_transformers_utils.py | 2 + 11 files changed, 1053 insertions(+), 12 deletions(-) create mode 100644 python/sglang/srt/configs/kimi_k25.py create mode 100644 python/sglang/srt/models/kimi_k25.py create mode 100644 python/sglang/srt/multimodal/processors/kimi_k25.py diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py index 671ee1af2..043de321b 100644 --- a/python/sglang/srt/configs/__init__.py +++ b/python/sglang/srt/configs/__init__.py @@ -9,6 +9,7 @@ from sglang.srt.configs.falcon_h1 import FalconH1Config from sglang.srt.configs.janus_pro import MultiModalityConfig from sglang.srt.configs.jet_nemotron import JetNemotronConfig from sglang.srt.configs.jet_vlm import JetVLMConfig +from sglang.srt.configs.kimi_k25 import KimiK25Config from sglang.srt.configs.kimi_linear import KimiLinearConfig from sglang.srt.configs.kimi_vl import KimiVLConfig from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig @@ -39,6 +40,7 @@ __all__ = [ "Step3VisionEncoderConfig", "Olmo3Config", "KimiLinearConfig", + "KimiK25Config", "Qwen3NextConfig", "DotsVLMConfig", "DotsOCRConfig", diff --git a/python/sglang/srt/configs/kimi_k25.py b/python/sglang/srt/configs/kimi_k25.py new file mode 100644 index 000000000..1ea8e7d89 --- /dev/null +++ b/python/sglang/srt/configs/kimi_k25.py @@ -0,0 +1,171 @@ +""" +Kimi K25 Model Configuration. +""" + +from transformers import DeepseekV3Config +from transformers.configuration_utils import PretrainedConfig + + +class KimiK25VisionConfig(PretrainedConfig): + """Vision configuration for K2-VL (vision tower + mm projector). + + Args: + Vision Tower Parameters: + patch_size: Patch size for vision tower. + init_pos_emb_height: Initial position embedding height. + init_pos_emb_width: Initial position embedding width. + init_pos_emb_time: Initial position embedding time dimension. + pos_emb_type: Type of position embedding. + num_attention_heads: Number of attention heads in vision tower. + num_hidden_layers: Number of hidden layers in vision tower. + hidden_size: Hidden size of vision tower. + intermediate_size: Intermediate size in vision tower FFN. + merge_kernel_size: Kernel size for spatial patch merging. + video_attn_type: Type of video attention. + merge_type: Type of merge operation. + + MM Projector Parameters: + mm_projector_type: Type of multimodal projector. + mm_hidden_size: Hidden size for projector (defaults to hidden_size). + projector_hidden_act: Activation function for projector. + projector_ln_eps: Layer norm epsilon for projector. + """ + + model_type = "kimi_k25" + + def __init__( + self, + # Vision Tower + patch_size: int = 14, + init_pos_emb_height: int = 64, + init_pos_emb_width: int = 64, + init_pos_emb_time: int = 4, + pos_emb_type: str = "divided_fixed", + num_attention_heads: int = 16, + num_hidden_layers: int = 27, + hidden_size: int = 1152, + intermediate_size: int = 4304, + merge_kernel_size: tuple[int, int] = (2, 2), + video_attn_type: str = "spatial_temporal", + merge_type: str = "sd2_tpool", + # MM Projector + mm_projector_type: str = "patchmerger", + mm_hidden_size: int | None = None, + projector_hidden_act: str = "gelu", + projector_ln_eps: float = 1e-5, + text_hidden_size: int = 7168, + **kwargs, + ): + super().__init__(**kwargs) + # Vision Tower + self.patch_size = patch_size + self.init_pos_emb_height = init_pos_emb_height + self.init_pos_emb_width = init_pos_emb_width + self.init_pos_emb_time = init_pos_emb_time + self.pos_emb_type = pos_emb_type + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_hidden_layers + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.merge_kernel_size = merge_kernel_size + self.video_attn_type = video_attn_type + self.merge_type = merge_type + # MM Projector + self.mm_projector_type = mm_projector_type + if mm_hidden_size is not None: + self.mm_hidden_size = mm_hidden_size + else: + self.mm_hidden_size = hidden_size + self.projector_hidden_act = projector_hidden_act + self.projector_ln_eps = projector_ln_eps + self.text_hidden_size = text_hidden_size + + +class KimiK25Config(PretrainedConfig): + """K2-VL model configuration. + + K2-VL extends Kimi-VL with video support using video-chunks. + A video-chunk consists of multiple consecutive frames (default: 4) + that are processed together with temporal pooling. + + Args: + text_config: Configuration for the text model (DeepseekV3). + + Vision Tower Parameters: + patch_size: Patch size for vision tower. + init_pos_emb_height: Initial position embedding height. + init_pos_emb_width: Initial position embedding width. + init_pos_emb_time: Initial position embedding time dimension. + pos_emb_type: Type of position embedding. + vt_num_attention_heads: Number of attention heads in vision tower. + vt_num_hidden_layers: Number of hidden layers in vision tower. + vt_hidden_size: Hidden size of vision tower. + vt_intermediate_size: Intermediate size in vision tower FFN. + merge_kernel_size: Kernel size for spatial patch merging. + video_attn_type: Type of video attention. + merge_type: Type of merge operation. + + Video-Chunk Parameters: + temporal_merge_kernel_size: Number of frames per video chunk. + Default is 4, meaning 4 frames are merged into 1 chunk. + sample_fps: Video sampling frame rate. + timestamp_mode: Format for chunk timestamps. + + MM Projector Parameters: + mm_projector_type: Type of multimodal projector. + mm_hidden_size: Hidden size from vision tower. + projector_hidden_act: Activation function for projector. + projector_ln_eps: Layer norm epsilon for projector. + + Other Parameters: + ignore_index: The ignore index for the loss function. + media_placeholder_token_id: The token ID for media placeholders. + pad_token_id: The token ID for padding. + """ + + model_type = "kimi_k25" + + def __init__( + self, + text_config: dict | DeepseekV3Config | None = None, + vision_config: dict | KimiK25VisionConfig | None = None, + # Other parameters + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + use_unified_vision_chunk: bool = False, + video_placeholder: str = "<|kimi_k25_video_placeholder|>", + **kwargs, + ): + if text_config is None: + text_config = DeepseekV3Config() + elif isinstance(text_config, dict): + text_config = DeepseekV3Config(**text_config) + + if vision_config is None: + vision_config = KimiK25VisionConfig() + elif isinstance(vision_config, dict): + vision_config = KimiK25VisionConfig(**vision_config) + self.vision_config = vision_config + self.text_config = text_config + # Other config + self.ignore_index = ignore_index + self.media_placeholder_token_id = media_placeholder_token_id + self.use_unified_vision_chunk = use_unified_vision_chunk + self.video_placeholder = video_placeholder + + # Propagate quantization config from text model + if getattr(self.text_config, "quantization_config", None) is not None: + self.quantization_config = self.text_config.quantization_config + + super().__init__(pad_token_id=pad_token_id, **kwargs) + + @property + def hidden_size(self) -> int: + """Get hidden size from text config for compatibility.""" + return self.text_config.hidden_size + + @property + def vocab_size(self) -> int: + """Get vocab size from text config for compatibility.""" + return self.text_config.vocab_size diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index b3a49f8a4..fbcb9754c 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -391,16 +391,17 @@ class ModelConfig: or "MistralLarge3ForCausalLM" in self.hf_config.architectures or "PixtralForConditionalGeneration" in self.hf_config.architectures or "MistralLarge3ForCausalLMEagle" in self.hf_config.architectures + or "KimiK25ForConditionalGeneration" in self.hf_config.architectures ): self.head_dim = 256 self.attention_arch = AttentionArch.MLA - self.kv_lora_rank = self.hf_config.kv_lora_rank - self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim - self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim - self.v_head_dim = self.hf_config.v_head_dim + self.kv_lora_rank = self.hf_text_config.kv_lora_rank + self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim + self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim + self.v_head_dim = self.hf_text_config.v_head_dim self.index_head_dim = ( - get_nsa_index_head_dim(self.hf_config) - if is_deepseek_nsa(self.hf_config) + get_nsa_index_head_dim(self.hf_text_config) + if is_deepseek_nsa(self.hf_text_config) else None ) @@ -412,11 +413,11 @@ class ModelConfig: self.scaling = 1 / math.sqrt( self.qk_nope_head_dim + self.qk_rope_head_dim ) - if self.hf_config.rope_scaling: - mscale_all_dim = self.hf_config.rope_scaling.get( + if self.hf_text_config.rope_scaling: + mscale_all_dim = self.hf_text_config.rope_scaling.get( "mscale_all_dim", False ) - scaling_factor = self.hf_config.rope_scaling["factor"] + scaling_factor = self.hf_text_config.rope_scaling["factor"] mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale @@ -1169,6 +1170,7 @@ multimodal_model_archs = [ "PaddleOCRVLForConditionalGeneration", "MiDashengLMModel", "StepVLForConditionalGeneration", + "KimiK25ForConditionalGeneration", ] if external_mm_model_arch := envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.get(): diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 3b8773a55..c59f4b443 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1194,7 +1194,7 @@ class OpenAIServingChat(OpenAIServingBase): """Judge whether the request needs reasoning""" if not self.reasoning_parser: return False - if self.reasoning_parser in ["deepseek-v3"]: + if self.reasoning_parser in ["deepseek-v3", "kimi_k2"]: return ( request.chat_template_kwargs is not None and request.chat_template_kwargs.get("thinking") is True diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index 79d9cd3c5..68593ab08 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -13,7 +13,11 @@ from einops import rearrange from sglang.jit_kernel.norm import can_use_fused_inplace_qknorm as can_use_jit_qk_norm from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import get_attention_tp_rank, get_attention_tp_size +from sglang.srt.layers.dp_attention import ( + get_attention_tp_group, + get_attention_tp_rank, + get_attention_tp_size, +) from sglang.srt.models.utils import apply_qk_norm from sglang.srt.utils import ( get_bool_env_var, @@ -692,6 +696,7 @@ class VisionAttention(nn.Module): quant_config=quant_config, tp_rank=self.tp_rank, tp_size=self.tp_size, + reduce_results=False, prefix=add_prefix("proj", prefix), ) self.aux_stream = aux_stream @@ -914,6 +919,8 @@ class VisionAttention(nn.Module): # [b, s, h * head_size] --> [b, s, h * head_size] output, _ = self.proj(output) + if self.tp_size > 1: + output = get_attention_tp_group().all_reduce(output) else: # [b * s, h, head_size] --> [s, b, h * head_size] context_layer = rearrange( @@ -922,6 +929,8 @@ class VisionAttention(nn.Module): # [s, b, h * head_size] --> [s, b, h * head_size] output, _ = self.proj(context_layer) + if self.tp_size > 1: + output = get_attention_tp_group().all_reduce(output) # [s, b, h * head_size] --> [b, s, h * head_size] output = output.view(bsz, s, -1) diff --git a/python/sglang/srt/models/kimi_k25.py b/python/sglang/srt/models/kimi_k25.py new file mode 100644 index 000000000..048ef0578 --- /dev/null +++ b/python/sglang/srt/models/kimi_k25.py @@ -0,0 +1,744 @@ +import logging +from copy import deepcopy +from typing import Iterable, List, Optional, Sequence, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from transformers import activations + +from sglang.srt.configs.kimi_k25 import KimiK25Config, KimiK25VisionConfig +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, + general_mm_embed_routine, +) + +try: + from transformers.activations import PytorchGELUTanh +except ImportError: + from transformers.activations import GELUTanh + + activations.PytorchGELUTanh = GELUTanh + PytorchGELUTanh = GELUTanh + +from sglang.srt.layers.attention.vision import VisionAttention +from sglang.srt.layers.linear import ReplicatedLinear +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.models.deepseek_v2 import DeepseekV3ForCausalLM +from sglang.srt.models.kimi_vl_moonvit import MLP2 +from sglang.srt.utils import add_prefix + +KIMIV_VT_INFER_MAX_PATCH_NUM = 16328 +logger = logging.getLogger(__name__) + + +def apply_rope( + xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor, x_shape=None +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Args: (The leading dimensions of all inputs should be the same) + xq: query, tensor of shape (..., num_heads, head_dim) + xk: key, tensor of shape (..., num_heads, head_dim) + freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64. It contains the precomputed cis(freqs) for each position in the 2D grid. + Returns: + xq_out, xk_out: tensors of shape (..., num_heads, head_dim) + """ + + freqs_cis = freqs_cis.unsqueeze(-2) # ..., 1, head_dim/2 + # ..., num_heads, head_dim/2 + xq_ = torch.view_as_complex(xq.float().view(*xq.shape[:-1], -1, 2)) + xk_ = torch.view_as_complex(xk.float().view(*xq.shape[:-1], -1, 2)) + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim + return xq_out.type_as(xq), xk_out.type_as(xk) + + +def tpool_patch_merger( + x: torch.Tensor, + grid_thws: torch.Tensor, + merge_kernel_size: tuple[int, int] = (2, 2), +) -> list[torch.Tensor]: + d_model = x.size(-1) + + outputs = [] + pre_sum = 0 + for t, h, w in grid_thws.tolist(): + # Get the current sequence + seq = x[pre_sum : pre_sum + t * h * w] + # Reshape along self.merge_kernel_size and concat to the last dimension + kernel_height, kernel_width = merge_kernel_size + new_height, new_width = h // kernel_height, w // kernel_width + reshaped_seq = seq.view( + t, new_height, kernel_height, new_width, kernel_width, d_model + ) + reshaped_seq = ( + reshaped_seq.permute(0, 1, 3, 2, 4, 5).contiguous().mean(dim=0) + ) # temporal pooling + padded_seq = reshaped_seq.view( + new_height * new_width, kernel_height * kernel_width, -1 + ) + outputs.append(padded_seq) + pre_sum += t * h * w + + return outputs + + +class MoonViTEncoderLayer(nn.Module): + + def __init__( + self, + num_heads: int, + hidden_dim: int, + mlp_dim: int, + *, + activation=F.gelu, + attn_bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + use_data_parallel: bool = False, + ): + super().__init__() + self.num_heads = num_heads + self.hidden_dim = hidden_dim + self.hidden_size_per_attention_head = self.hidden_dim // self.num_heads + + self.norm0 = nn.LayerNorm(hidden_dim) + self.norm1 = nn.LayerNorm(hidden_dim) + self.mlp = MLP2([hidden_dim, mlp_dim, hidden_dim], activation) + + self.attn = VisionAttention( + embed_dim=hidden_dim, + num_heads=num_heads, + projection_size=hidden_dim, + use_qkv_parallel=True, + qkv_bias=attn_bias, + proj_bias=attn_bias, + flatten_batch=True, + quant_config=quant_config, + prefix=add_prefix("attn", prefix), + use_data_parallel=use_data_parallel, + customized_position_embedding_applier=apply_rope, + ) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: int, + rope_freqs_cis: torch.Tensor | None = None, + ): + residual = hidden_states + hidden_states = self.norm0(hidden_states) + + hidden_states = self.attn( + hidden_states, + cu_seqlens=cu_seqlens, + position_embeddings=rope_freqs_cis, + ) + + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + return hidden_states + + +def get_rope_shape_decorate(func): + _get_rope_shape_first_call_flag = set() + + def wrapper(org, interpolation_mode, shape): + key = (org.requires_grad, torch.is_grad_enabled(), interpolation_mode) + if key not in _get_rope_shape_first_call_flag: + _get_rope_shape_first_call_flag.add(key) + _ = func(org, interpolation_mode, shape=(64, 64)) + return func(org, interpolation_mode, shape) + + return wrapper + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + From: + https://github.com/OpenGVLab/InternVideo/blob/421f6d2361fc8f61a3394244571f2601a4e99e29/InternVideo2/multi_modality/models/backbones/internvideo2/pos_embed.py#L86 + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float32) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +@get_rope_shape_decorate +@torch.compile(dynamic=True) +def get_rope_shape(org, interpolation_mode, shape): + return ( + F.interpolate( + org.permute((2, 0, 1)).unsqueeze(0), + size=shape, + mode=interpolation_mode, + ) + .squeeze(0) + .permute((1, 2, 0)) + .flatten(end_dim=1) + ) + + +def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False): + """ + t_size: int of the temporal size + return: + pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token) + """ + grid_t = np.arange(t_size, dtype=np.float32) + pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t) + if cls_token: + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0) + return pos_embed + + +class Learnable2DInterpPosEmbDivided_fixed(nn.Module): + + def __init__( + self, + height: int, + width: int, + num_frames: int, + dim: int, + interpolation_mode: str = "bicubic", + ) -> None: + super().__init__() + self.height = height + self.width = width + self.num_frames = num_frames + self.dim = dim + self.interpolation_mode = interpolation_mode + self.weight = nn.Parameter(torch.empty(height, width, dim)) + self.register_buffer( + "time_weight", + torch.from_numpy(get_1d_sincos_pos_embed(self.dim, self.num_frames)) + .float() + .unsqueeze(1), + persistent=False, + ) + + self.reset_parameters() + + def reset_parameters(self): + nn.init.normal_(self.weight) + + def forward(self, x: torch.Tensor, grid_thws: torch.Tensor) -> torch.Tensor: + pos_embs = [] + for t, h, w in grid_thws.tolist(): + assert t <= self.num_frames, f"t:{t} > self.num_frames:{self.num_frames}" + if (h, w) == self.weight.shape[:-1]: + pos_emb_2d = self.weight.flatten(end_dim=1) + else: + pos_emb_2d = get_rope_shape( + self.weight, + interpolation_mode=self.interpolation_mode, + shape=(h, w), + ) + + if t == 1: + pos_emb_3d = pos_emb_2d + else: + pos_emb_3d = ( + pos_emb_2d.unsqueeze(0).repeat(t, 1, 1) + self.time_weight[0:t] + ) + + pos_embs.append(pos_emb_3d.reshape(-1, pos_emb_3d.shape[-1])) + + out = x + torch.cat(pos_embs) + return out + + +class Rope2DPosEmbRepeated(nn.Module): + """2D rotary position embedding with multi-resolution support. + This class is intended to be used in the following way: + 1. Before training, create an instance of Rope2DPosEmb. This instance will hold the precomputed cis. + 2. Before each forward pass, call `get_freqs_cis_by_*` to get the `freqs_cis` tensor for this iteration. + 3. During the forward pass, pass the `freqs_cis` tensor to each attention layer, and call `apply` just before each attention operation. + The rope is shared across all attention layers and all heads. + Refs: + - RoFormer: https://arxiv.org/abs/2104.09864 + - VisionLLaMA: https://arxiv.org/abs/2403.00522 + - https://github.com/Meituan-AutoML/VisionLLaMA/blob/main/dit/models.py + Args: + dim (int): usually the multi-head attention dimension, should be divisible by 4 (TODO: relax this constraint if needed) + max_height (int): the maximum height of the 2D grid + max_width (int): the maximum width of the 2D grid + theta_base (float): the base of the theta + """ + + def __init__(self, dim: int, max_height: int, max_width: int, theta_base=10000): + super().__init__() + self.dim = dim + assert self.dim % 4 == 0, "dim must be divisible by 4" + self.max_height = max_height + self.max_width = max_width + self.theta_base = theta_base + + def extra_repr(self): + return f"dim={self.dim}, max_height={self.max_height}, max_width={self.max_width}, theta_base={self.theta_base}" + + def _precompute_freqs_cis(self, device: torch.device) -> torch.Tensor: + """Calculate the cis(freqs) for each position in the 2D grid. + Return: complex tensor of shape (max_height, max_width, dim//2) and value: + height axis: ret[h, w, 2*i] = cis(h * theta_base**(-4*i/dim)) + weight axis: ret[h, w, 2*i+1] = cis(w * theta_base**(-4*i/dim)) with (i in [0, dim//4)) + note: `cis` is a mathematical notation defined by cis x = cos x + i sin x, + """ + N = self.max_height * self.max_width + flat_pos = torch.arange(0, N).float().to(device) + x_pos = flat_pos % self.max_width + y_pos = flat_pos // self.max_width + dim_range = ( + torch.arange(0, self.dim, 4)[: (self.dim // 4)].float().to(device) + ) # C/4 + freqs = 1.0 / (self.theta_base ** (dim_range / self.dim)) + x_freqs = torch.outer(x_pos, freqs).float() # N, C/4 + y_freqs = torch.outer(y_pos, freqs).float() # N, C/4 + x_cis = torch.polar(torch.ones_like(x_freqs), x_freqs) # N, C/4 + y_cis = torch.polar(torch.ones_like(y_freqs), y_freqs) # N, C/4 + # N, C/4, 2 + freqs_cis = torch.cat( + [x_cis.unsqueeze(dim=-1), y_cis.unsqueeze(dim=-1)], dim=-1 + ) + # max_height, max_width, C/2 + freqs_cis = freqs_cis.reshape(self.max_height, self.max_width, -1) + return freqs_cis + + def get_freqs_cis( + self, grid_thws: torch.Tensor, device: torch.device + ) -> torch.Tensor: + """ + Args: + grid_thws (torch.Tensor): grid time, height and width + Returns: + freqs_cis: tensor of shape (sum(t * height * width), dim//2) + """ + if not hasattr(self, "freqs_cis"): + self.register_buffer( + "freqs_cis", self._precompute_freqs_cis(device), persistent=False + ) + + shapes = grid_thws.tolist() + assert all( + 1 <= h <= self.max_height and 1 <= w <= self.max_width for t, h, w in shapes + ), ( + shapes, + self.max_height, + self.max_width, + ) + freqs_cis = torch.cat( + [ + self.freqs_cis[:h, :w].reshape(-1, self.dim // 2).repeat(t, 1) + for t, h, w in shapes + ], + dim=0, + ) + return freqs_cis + + +class MoonVision3dPatchEmbed(nn.Module): + + def __init__( + self, + out_dim: int, + in_dim: int = 3, + patch_size: int | tuple[int, int] = (14, 14), + pos_emb_height: int = 14, + pos_emb_width: int = 14, + pos_emb_time: int = 4, + pos_emb_type: str = "divided_fixed", + ): + super().__init__() + assert isinstance( + patch_size, int | Sequence + ), f"Invalid patch_size type: {type(patch_size)}" + if isinstance(patch_size, int): + patch_size = (patch_size, patch_size) + assert ( + len(patch_size) == 2 + ), f"Expected patch_size to be a tuple of 2, got {patch_size}" + self.patch_size = patch_size + + self.proj = nn.Conv2d( + in_dim, out_dim, kernel_size=patch_size, stride=patch_size + ) + + if pos_emb_type == "divided_fixed": + self.pos_emb = Learnable2DInterpPosEmbDivided_fixed( + height=pos_emb_height, + width=pos_emb_width, + num_frames=pos_emb_time, + dim=out_dim, + ) + else: + raise NotImplementedError(f"Not support pos_emb_type: {pos_emb_type}") + + def forward(self, x: torch.Tensor, grid_thws: torch.Tensor) -> torch.Tensor: + """ + Args: + x (L, Channels): input tensor + grid_hws (N, 3): temporal, height and width + Returns: + (L, Cout) tensor + """ + x = self.proj(x).view(x.size(0), -1) + # apply positional embedding + x = self.pos_emb(x, grid_thws) + return x + + +class MoonViT3dEncoder(nn.Module): + + def __init__( + self, + hidden_dim: int, + num_layers: int, + block_cfg: dict, + video_attn_type: str = "spatial_temporal", + ) -> None: + super().__init__() + + assert ( + video_attn_type == "spatial_temporal" + ), f'video_attn_type must be "spatial_temporal", got {video_attn_type}' + self.video_attn_type = video_attn_type + self.rope_2d = Rope2DPosEmbRepeated( + block_cfg["hidden_dim"] // block_cfg["num_heads"], 512, 512 + ) + self.blocks = nn.ModuleList( + [MoonViTEncoderLayer(**block_cfg) for _ in range(num_layers)] + ) + self.final_layernorm = nn.LayerNorm(hidden_dim) + + def forward( + self, + hidden_states: torch.Tensor, + grid_thws: torch.Tensor, + ) -> torch.Tensor: + rope_freqs_cis = self.rope_2d.get_freqs_cis( + grid_thws=grid_thws, device=hidden_states.device + ) + + lengths = torch.cat( + ( + torch.zeros(1, dtype=grid_thws.dtype, device=grid_thws.device), + grid_thws[:, 0] * grid_thws[:, 1] * grid_thws[:, 2], + ) + ) + + max_seqlen = lengths.max() + cu_seqlens = lengths.to(hidden_states.device).cumsum(dim=0, dtype=torch.int32) + + for block in self.blocks: + hidden_states = block( + hidden_states, cu_seqlens, max_seqlen, rope_freqs_cis=rope_freqs_cis + ) + + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +class MoonViT3dPretrainedModel(nn.Module): + model_type = "moonvit3d" + _no_split_modules = ["PackingTransformer"] + _supports_flash_attn_2 = True + _supports_sdpa = True + + def __init__(self, config, *inputs, **kwargs): + super().__init__() + config = deepcopy(config) + self.merge_kernel_size = config.merge_kernel_size + self.patch_size = config.patch_size + self.merge_type = config.merge_type + + self.patch_embed = MoonVision3dPatchEmbed( + out_dim=config.hidden_size, + patch_size=config.patch_size, + pos_emb_height=config.init_pos_emb_height, + pos_emb_width=config.init_pos_emb_width, + pos_emb_time=config.init_pos_emb_time, + pos_emb_type=config.pos_emb_type, + ) + + self.encoder = MoonViT3dEncoder( + hidden_dim=config.hidden_size, + num_layers=config.num_hidden_layers, + block_cfg={ + "num_heads": config.num_attention_heads, + "hidden_dim": config.hidden_size, + "mlp_dim": config.intermediate_size, + "activation": PytorchGELUTanh(), + "attn_bias": True, + }, + video_attn_type=config.video_attn_type, + ) + + @property + def dtype(self) -> torch.dtype: + return self.patch_embed.proj.weight.dtype + + @property + def device(self) -> torch.device: + return self.patch_embed.proj.weight.device + + def forward( + self, pixel_values: torch.Tensor, grid_thws: torch.Tensor + ) -> torch.Tensor: + """ + Args: + pixel_values (torch.Tensor): The input pixel values. + grid_thws (torch.Tensor): Temporal, height and width. + Returns: + torch.Tensor: The output tokens. + """ + assert grid_thws.ndim == 2, f"grid_thws should be 2D, got {grid_thws.ndim}" + assert grid_thws.size(1) == 3, f"No support for _thw: {grid_thws}" + hidden_states = self.patch_embed(pixel_values, grid_thws) + hidden_states = self.encoder(hidden_states, grid_thws) + hidden_states = hidden_states.squeeze(0) + # spatial downsampling 2x with temporal pooling all + hidden_states = tpool_patch_merger( + hidden_states, grid_thws, merge_kernel_size=self.merge_kernel_size + ) + + return hidden_states + + +class K2VLMultiModalProjector(nn.Module): + """Multi-modal projector with patch merging for K2-VL.""" + + def __init__( + self, + config: KimiK25VisionConfig, + use_data_parallel: bool = False, + prefix: str = "", + ): + super().__init__() + self.use_data_parallel = use_data_parallel + + # Hidden size after patch merging + merge_h, merge_w = config.merge_kernel_size + self.hidden_size = config.vt_hidden_size * merge_h * merge_w + + self.pre_norm = torch.nn.LayerNorm(config.vt_hidden_size, eps=1e-5) + self.linear_1 = ReplicatedLinear( + self.hidden_size, + self.hidden_size, + bias=True, + prefix=add_prefix(prefix, "linear_1"), + ) + self.linear_2 = ReplicatedLinear( + self.hidden_size, + config.text_hidden_size, + bias=True, + prefix=add_prefix(prefix, "linear_2"), + ) + self.act = nn.GELU() + + def forward(self, image_features: torch.Tensor) -> torch.Tensor: + hidden_states = self.pre_norm(image_features).view(-1, self.hidden_size) + hidden_states, _ = self.linear_1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states, _ = self.linear_2(hidden_states) + return hidden_states + + +@torch.inference_mode() +def mm_projection_auto( + mm_projector: torch.nn.Module | None, vt_output: list[torch.Tensor] +): + """Apply MM projector to vision tower outputs.""" + if mm_projector is None: + return vt_output + + num_embedding_list = [x.shape[0] for x in vt_output] + batched = torch.cat(vt_output, dim=0) + proj_out = mm_projector(batched) if mm_projector else batched + proj_out = proj_out.reshape(-1, proj_out.shape[-1]) + proj_out = torch.split(proj_out, num_embedding_list) + return proj_out + + +@torch.inference_mode() +def vision_tower_forward_auto( + vision_tower: torch.nn.Module, + pixel_values: torch.Tensor, + grid_thw: torch.Tensor, + mm_projector: torch.nn.Module | None = None, +) -> list[torch.Tensor]: + """Auto-batched vision tower forward.""" + assert isinstance( + pixel_values, torch.Tensor + ), "expect pixel_values to be a tensor, get {}".format(type(pixel_values)) + n = grid_thw.shape[0] + n_patches_each_media = grid_thw.prod(-1) + max_infer_batch = max(n_patches_each_media.max(), KIMIV_VT_INFER_MAX_PATCH_NUM) + logger.debug( + "vt max_infer_batch: %s, KIMIV_VT_INFER_MAX_PATCH_NUM: %s", + max_infer_batch, + KIMIV_VT_INFER_MAX_PATCH_NUM, + ) + tensors = [] + pre_sum = 0 + current_group_start = 0 + current_group_patches = 0 + + for i in range(n): + current_media_patches = n_patches_each_media[i].item() + if current_group_patches + current_media_patches <= max_infer_batch: + current_group_patches += current_media_patches + else: + if current_group_start < i: + group_grid_thw = grid_thw[current_group_start:i] + group_n_patches = n_patches_each_media[current_group_start:i].sum() + group_input = pixel_values[pre_sum : pre_sum + group_n_patches] + group_output = vision_tower(group_input, group_grid_thw) + proj_out = mm_projection_auto(mm_projector, group_output) + tensors.extend(proj_out) + pre_sum += group_n_patches + + current_group_start = i + current_group_patches = current_media_patches + + # Process the last group + if current_group_start < n: + group_grid_thw = grid_thw[current_group_start:n] + group_n_patches = n_patches_each_media[current_group_start:n].sum() + group_input = pixel_values[pre_sum : pre_sum + group_n_patches] + group_output = vision_tower(group_input, group_grid_thw) + proj_out = mm_projection_auto(mm_projector, group_output) + tensors.extend(proj_out) + + return tensors + + +class KimiK25ForConditionalGeneration(nn.Module): + def __init__( + self, + config: KimiK25Config, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + **kwargs, # fix init_tts argument error + ) -> None: + super().__init__() + self.config = config + # Create vision tower + self.vision_tower = MoonViT3dPretrainedModel(config.vision_config) + # Create mm projector + self.mm_projector = K2VLMultiModalProjector(config.vision_config) + + self.language_model = DeepseekV3ForCausalLM(config.text_config, quant_config) + + # Ensure that the dtype of the vision_tower and mm_projector matches that of the language_model. + # This solves the dtype mismatch issue when using device_map="auto" and torch_dtype. + if hasattr(self.language_model, "dtype"): + target_dtype = self.language_model.dtype + self.vision_tower = self.vision_tower.to(dtype=target_dtype) + self.mm_projector = self.mm_projector.to(dtype=target_dtype) + + def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + pixel_values = torch.cat([item.feature for item in items], dim=0).type( + self.vision_tower.dtype + ) + grid_thws = torch.concat([item.grid_thws for item in items], dim=0).to( + self.vision_tower.device + ) + + target_dtype = self.vision_tower.patch_embed.proj.weight.dtype + pixel_values = pixel_values.to(target_dtype) + image_features = vision_tower_forward_auto( + self.vision_tower, + pixel_values, + grid_thws, + mm_projector=self.mm_projector, + ) + image_features = torch.cat(image_features, dim=0) + return image_features + + def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): + pattern = MultiModalityDataPaddingPatternMultimodalTokens() + return pattern.pad_input_tokens(input_ids, mm_inputs) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + get_embedding: bool = False, + ): + hidden_states = general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self.language_model, + data_embedding_funcs={ + Modality.IMAGE: self.get_image_feature, + }, + positions=positions, + ) + + return hidden_states + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + """Load weights for the model, separating vision and language weights""" + weights = list(weights) + + # Separate vision tower weights and language model weights + vision_weights = [] + language_weights = [] + + for name, loaded_weight in weights: + if "vision_tower" in name or "mm_projector" in name: + name = name.replace(r"wqkv.", r"attn.qkv_proj.") + name = name.replace(r"wo.", r"attn.proj.") + name = name.replace("mm_projector.proj.0", "mm_projector.linear_1") + name = name.replace("mm_projector.proj.2", "mm_projector.linear_2") + vision_weights.append((name, loaded_weight)) + else: + name = name.replace("language_model.", "") + # All other weights go to language model + language_weights.append((name, loaded_weight)) + + # Load vision tower weights + vision_state_dict = dict(vision_weights) + params_dict = dict(self.named_parameters(remove_duplicate=False)) + for name, loaded_weight in vision_state_dict.items(): + if name not in params_dict: + raise ValueError(f"Weight {name} not found in params_dict") + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + # loaded_weight = self._pad_vit_attn_dummy_heads(name, loaded_weight) + weight_loader(param, loaded_weight) + + # Load language model weights + if language_weights: + self.language_model.load_weights(language_weights) + + +EntryClass = [KimiK25ForConditionalGeneration] diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index f29344909..c21339ef8 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -210,6 +210,7 @@ class BaseMultimodalProcessor(ABC): "num_patches": Modality.IMAGE, "patch_pixel_values": Modality.IMAGE, "block_sizes": Modality.IMAGE, + "grid_thws": Modality.IMAGE, # for kimi k2.5 # Audio-related attributes "audio_features": Modality.AUDIO, "audio_feature_lens": Modality.AUDIO, diff --git a/python/sglang/srt/multimodal/processors/kimi_k25.py b/python/sglang/srt/multimodal/processors/kimi_k25.py new file mode 100644 index 000000000..bab6b28a6 --- /dev/null +++ b/python/sglang/srt/multimodal/processors/kimi_k25.py @@ -0,0 +1,88 @@ +import re +from typing import Dict, List, Tuple, Union + +import torch + +from sglang.srt.managers.schedule_batch import MultimodalDataItem +from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration +from sglang.srt.multimodal.processors.base_processor import ( + BaseMultimodalProcessor as SGLangBaseProcessor, +) +from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens + + +# Compatible with KimiVLForConditionalGeneration +class KimiK2_5VLImageProcessor(SGLangBaseProcessor): + models = [KimiK25ForConditionalGeneration] + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + self.mm_tokens = MultimodalSpecialTokens( + image_token="<|media_pad|>", + # TODO: could we convert in MultimodalSpecialTokens? + image_token_id=hf_config.media_placeholder_token_id, + image_token_regex=re.compile(r"(?:<\|media_pad\|>)+"), + ).build(_processor) + + async def process_mm_data_async( + self, + image_data: List[Union[str, bytes, Dict]], + input_text, + request_obj, + *args, + **kwargs, + ): + base_output = self.load_mm_data( + prompt=input_text, + image_data=image_data, + multimodal_tokens=self.mm_tokens, + ) + prompt = base_output.input_text + + mm_items, input_ids, _ = self.process_and_combine_mm_data( + base_output, self.mm_tokens + ) + + return { + "input_ids": input_ids.tolist(), + "mm_items": mm_items, + "im_token_id": self.mm_tokens.image_token_id, + } + + def _process_and_collect_mm_items( + self, input_text: str, images=None, audios=None, videos=None, **kwargs + ) -> Tuple[List[MultimodalDataItem], torch.Tensor, dict]: + """ + Helper method to process multimodal data and create mm_items in one step. + + Returns: + Tuple of (created mm_items, input_ids) + """ + + parts = input_text.split(self.mm_tokens.image_token) + + result = [parts[0]] + for image, part in zip(images, parts[1:]): + num_tokens = self._processor.media_processor.media_tokens_calculator( + {"type": "image", "image": image} + ) + result.append(self.mm_tokens.image_token * num_tokens + part) + + input_text = "".join(result) + + if images: # for kimi k2 vl + mediums = [] + for image in images: + mediums.append({"type": "image", "image": image}) + key = "_medias"[1:] # bypass lint + kwargs[key] = mediums + images = None + + ret = self.process_mm_data( + input_text=input_text, images=images, audios=audios, videos=videos, **kwargs + ) + + input_ids = ret["input_ids"].flatten() + collected_items = self.collect_mm_items_from_processor_output(ret) + + return collected_items, input_ids, ret diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 8949ba5d7..72477a4b6 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -148,6 +148,26 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector): # https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599 +class KimiK2Detector(BaseReasoningFormatDetector): + """ + Detector for Kimi K2 model. + + It uses the DeepSeek-R1 reasoning format: ()*(.*). + Defaults to thinking mode (force_reasoning=True), but allows disabling it + if the model is configured to not think. + """ + + def __init__(self, stream_reasoning: bool = True, force_reasoning: bool = True): + super().__init__( + "", + "", + # Allow force_reasoning to be controlled by arguments, defaulting to True + # to match vLLM's default `thinking=True` behavior. + force_reasoning=force_reasoning, + stream_reasoning=stream_reasoning, + ) + + class Qwen3Detector(BaseReasoningFormatDetector): """ Detector for Qwen3 models (e.g., Qwen/Qwen3-235B-A22B). @@ -307,7 +327,7 @@ class ReasoningParser: "glm45": Qwen3Detector, "gpt-oss": GptOssDetector, "kimi": KimiDetector, - "kimi_k2": DeepSeekR1Detector, + "kimi_k2": KimiK2Detector, "qwen3": Qwen3Detector, "qwen3-thinking": Qwen3Detector, "minimax": Qwen3Detector, diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 1a049ced4..0026eea61 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1174,6 +1174,7 @@ class ServerArgs: if model_arch in [ "DeepseekV3ForCausalLM", + "KimiK25ForConditionalGeneration", "MistralLarge3ForCausalLM", "PixtralForConditionalGeneration", ]: @@ -1592,6 +1593,7 @@ class ServerArgs: "Glm4MoeForCausalLM", "Glm4MoeLiteForCausalLM", "Qwen3MoeForCausalLM", + "KimiK25ForConditionalGeneration", ] and (is_sm90_supported() or is_sm100_supported()) and not self.enable_dp_attention diff --git a/python/sglang/srt/utils/hf_transformers_utils.py b/python/sglang/srt/utils/hf_transformers_utils.py index f8743b416..c78d75c6a 100644 --- a/python/sglang/srt/utils/hf_transformers_utils.py +++ b/python/sglang/srt/utils/hf_transformers_utils.py @@ -54,6 +54,7 @@ from sglang.srt.configs import ( FalconH1Config, JetNemotronConfig, JetVLMConfig, + KimiK25Config, KimiLinearConfig, KimiVLConfig, LongcatFlashConfig, @@ -93,6 +94,7 @@ _CONFIG_REGISTRY: List[Type[PretrainedConfig]] = [ DeepseekVLV2Config, JetNemotronConfig, JetVLMConfig, + KimiK25Config, ] _CONFIG_REGISTRY = {