sglang/python/sglang/srt/multimodal/evs/evs_module.py

# Copyright 2025 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================


import dataclasses
import typing
from abc import ABC, abstractmethod
from dataclasses import dataclass

import torch
from transformers import PretrainedConfig

from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult
from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor
from sglang.utils import logger

from .evs_core import compute_retention_mask, replace_offsets_with_tokens_per_frame


@dataclasses.dataclass(kw_only=True)
class EVSDataItem(MultimodalDataItem):
    thw_grids: list[tuple[int, int, int]]


@dataclasses.dataclass(kw_only=True)
class VideoEVSDataItem(EVSDataItem):
    pre_chunked_input_ids: torch.Tensor

    def __post_init__(self):
        assert self.is_video()


@dataclass(kw_only=True)
class EVSEmbeddingResult(EmbeddingResult):
    """
    Embedding result that includes per-frame token counts after EVS pruning.

    After pruning, each frame retains a different number of tokens based on its
    dissimilarity to the previous frame. This metadata is needed downstream to
    adjust the input_ids placeholder spans to match the actual embedding sizes.

    Attributes:
        embedding: The pruned video embeddings tensor.
        num_tokens_per_frame: Actual retained token count for each frame.
            For example, [256, 180, 195, 256] means frame 0 kept all 256 tokens
            (first frame is never pruned), while frames 1-2 were pruned.
    """

    num_tokens_per_frame: list[int]

    def redistribute_pruned_frames_placeholders(
        self,
        input_ids: torch.Tensor,
        offsets: list[tuple[int, int]],
        *,
        item: VideoEVSDataItem,
        extend_prefix_len: int,
        extend_seq_len: int,
    ) -> tuple[torch.Tensor, list[tuple[int, int]]]:
        assert len(input_ids) == extend_seq_len
        assert isinstance(
            item, VideoEVSDataItem
        ), f"Expected VideoEVSDataItem, got {type(item)}"
        pre_chunked_input_ids = item.pre_chunked_input_ids
        filler_token_id = item.pad_value
        input_ids_list = replace_offsets_with_tokens_per_frame(
            pre_chunked_input_ids=pre_chunked_input_ids,
            num_tokens_per_frame=self.num_tokens_per_frame,
            frame_offsets_inclusive=offsets,
            filler_token_id=filler_token_id,
        )
        input_ids = torch.tensor(
            input_ids_list, dtype=input_ids.dtype, device=input_ids.device
        )
        offsets = BaseMultimodalProcessor.get_mm_items_offset(
            input_ids, filler_token_id
        )
        input_ids = input_ids[extend_prefix_len : extend_prefix_len + extend_seq_len]
        assert (
            len(input_ids) == extend_seq_len
        ), f"Input ids length changed after redistribution, got {len(input_ids)} != {extend_seq_len}"
        return input_ids, offsets


@dataclass(frozen=True, kw_only=True)
class EVSConfig:
    video_pruning_rate: float
    spatial_merge_size: int = 1

    def __post_init__(self):
        assert (
            self.video_pruning_rate >= 0.0 and self.video_pruning_rate < 1.0
        ), f"Video pruning rate must be between 0.0 and 1.0, got {self.video_pruning_rate=}"


class EVS(torch.nn.Module, ABC):
    """
    Base class for video models that support EVS pruning.

    Subclass this alongside your model class and implement the static `create_evs_config`.
    On initialization, if video_pruning_rate > 0, this mixin replaces the model's
    get_video_feature() method with a wrapper that applies EVS pruning.

    Example: See `NemotronH_Nano_VL_V2`
    """

    @staticmethod
    @abstractmethod
    def create_evs_config(config: PretrainedConfig) -> EVSConfig:
        """Extract EVS parameters from model config. Must be implemented by subclass."""
        raise NotImplementedError

    @abstractmethod
    def get_video_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
        """Extract EVS parameters from model config. Must be implemented by subclass."""
        raise NotImplementedError

    def __init__(
        self,
        config: PretrainedConfig,
        *args: typing.Any,
        **kwargs: typing.Any,
    ) -> None:
        super().__init__()
        model_name = self.__class__.__name__
        self.original_get_video_feature = self.get_video_feature
        self.evs_config = self.create_evs_config(config)
        self.evs_enabled = self.evs_config.video_pruning_rate > 0.0
        if self.evs_enabled:
            logger.info(f"[EVS] enabled for {model_name} [{self.evs_config}]")
            self.get_video_feature = self.evs_video
        else:
            logger.info(
                f"[EVS] requested on model {model_name} but is disabled for pruning_rate == 0.0."
            )

    def evs_video(self, items: list[MultimodalDataItem]) -> EVSEmbeddingResult:
        """
        Apply EVS pruning to video embeddings.

        Args:
            items: List containing a single VideoEVSDataItem with video features.

        Returns:
            EVSEmbeddingResult with pruned embeddings and actual token counts per frame.
        """
        logger.debug(
            f"[EVS] beginning for model {self.__class__.__name__} [evs_config={self.evs_config=}]"
        )
        assert len(items) == 1, f"Expected 1 item, got {len(items)}"
        item = items[0]
        assert isinstance(
            item, VideoEVSDataItem
        ), f"Expected VideoEVSDataItem with modality VIDEO, got {item}"

        q = self.evs_config.video_pruning_rate
        merge = self.evs_config.spatial_merge_size
        videos_features = self.original_get_video_feature([item])
        if videos_features.ndim == 3:
            videos_features = videos_features.flatten(0, 1)
        assert videos_features.ndim == 2, videos_features.ndim

        final_embeddings: list[torch.Tensor] = []
        num_tokens_per_frame: list[int] = []

        sizes = [(t * h * w // merge**2) for t, h, w in item.thw_grids]
        for single_video, video_size_thw in zip(
            videos_features.split(sizes),
            item.thw_grids,
            strict=True,
        ):
            retention_mask = compute_retention_mask(
                single_video,
                video_size_thw=video_size_thw,
                spatial_merge_size=merge,
                q=q,
            )
            preserved = single_video[retention_mask]
            final_embeddings.append(preserved)
            num_frames = video_size_thw[0]
            tokens_per_frame = (
                retention_mask.reshape(num_frames, -1).sum(dim=-1).tolist()
            )
            num_tokens_per_frame.extend(tokens_per_frame)
        final_embeddings_tensor = torch.cat(final_embeddings)
        return EVSEmbeddingResult(
            embedding=final_embeddings_tensor,
            num_tokens_per_frame=num_tokens_per_frame,
        )