[VLM] Replace decord with torchcodec for video decoding (#20055)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: BakerBunker <17872844+BakerBunker@users.noreply.github.com>

examples/runtime/multimodal/llava_onevision_server.py

@@ -15,11 +15,12 @@ import numpy as np
 import openai
 import pybase64
 import requests
-from decord import VideoReader, cpu
 from PIL import Image
 
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
+
 # pip install httpx==0.23.3
-# pip install decord
+# pip install torchcodec
 # pip install protobuf==3.20.0
 
 
@@ -200,13 +201,13 @@ def video_speed_test(client, video_path):
 
 def prepare_video_messages(video_path):
     max_frames_num = 32
-    vr = VideoReader(video_path, ctx=cpu(0))
+    decoder = VideoDecoderWrapper(video_path)
-    total_frame_num = len(vr)
+    total_frame_num = len(decoder)
     uniform_sampled_frames = np.linspace(
         0, total_frame_num - 1, max_frames_num, dtype=int
     )
     frame_idx = uniform_sampled_frames.tolist()
-    frames = vr.get_batch(frame_idx).asnumpy()
+    frames = decoder.get_frames_at(frame_idx)
 
     base64_frames = []
     for frame in frames:

python/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
   "build",
   "compressed-tensors",
   "cuda-python==12.9",
-  "decord2",
+  "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
   "datasets",
   "einops",
   "fastapi",
@@ -68,7 +68,8 @@ dependencies = [
   "torch==2.9.1",
   "torchao==0.9.0",
   "torchaudio==2.9.1",
-  "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
+  "torchcodec==0.9.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec 0.9.1 for torch 2.9.x. Not available on Linux ARM.
+  "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
   "torchvision",
   "tqdm",
   "transformers==4.57.1",

python/pyproject_cpu.toml

@@ -23,7 +23,6 @@ dependencies = [
   "build",
   "compressed-tensors",
   "datasets",
-  "decord; platform_machine == 'x86_64'",
   "einops",
   "fastapi",
   "gguf",

python/pyproject_npu.toml

@@ -22,7 +22,6 @@ dependencies = [
   "av",
   "build",
   "compressed-tensors",
-  "decord2",
   "datasets",
   "einops",
   "fastapi",

python/pyproject_other.toml

@@ -24,7 +24,6 @@ runtime_common = [
   "av",
   "build",
   "compressed-tensors",
-  "decord2",
   "datasets",
   "einops",
   "fastapi",

python/pyproject_xpu.toml

@@ -16,7 +16,7 @@ classifiers = [
 
 dependencies = [
   "torch==2.9.0",
-  "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
+  "torchcodec==0.9.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. torch==2.9.0 on XPU uses 0.9.1
   "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
   "torchaudio==2.9.0",
   "torchvision",
@@ -28,7 +28,6 @@ dependencies = [
   "build",
   "compressed-tensors",
   "datasets",
-  "decord",
   "einops",
   "fastapi",
   "gguf",

python/sglang/check_env.py

@@ -50,7 +50,7 @@ PACKAGE_LIST = [
     "tiktoken",
     "anthropic",
     "litellm",
-    "decord2",
+    "torchcodec",
 ]
 
 

python/sglang/srt/multimodal/processors/base_processor.py

@@ -385,8 +385,7 @@ class BaseMultimodalProcessor(ABC):
         """
         estimate the total frame count from all visual input
         """
-        # Lazy import because decord is not available on some arm platforms.
+        from sglang.srt.utils.video_decoder import VideoDecoderWrapper
-        from decord import VideoReader, cpu
 
         # Before processing inputs
         if not image_data or len(image_data) == 0:
@@ -395,9 +394,8 @@ class BaseMultimodalProcessor(ABC):
         for image in image_data:
             if isinstance(image, str) and image.startswith("video:"):
                 path = image[len("video:") :]
-                # Estimate frames for the video
+                decoder = VideoDecoderWrapper(path)
-                vr = VideoReader(path, ctx=cpu(0))
+                num_frames = len(decoder)
-                num_frames = len(vr)
             else:
                 # For images, each contributes one frame
                 num_frames = 1

python/sglang/srt/multimodal/processors/internvl.py

@@ -6,7 +6,6 @@ from typing import List
 
 import numpy as np
 import torch
-from decord import VideoReader, cpu, gpu
 from PIL import Image
 
 from sglang.srt.managers.schedule_batch import (
@@ -20,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import (
     BaseMultiModalProcessorOutput,
     MultimodalSpecialTokens,
 )
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
 
 logger = logging.getLogger(__name__)
 
@@ -205,14 +205,8 @@ class InternVLProcessor(BaseMultimodalProcessor):
         return torch.stack(tiles).to(torch.bfloat16)
 
     @staticmethod
-    def _open_video_reader(path: str) -> VideoReader:
+    def _open_video_reader(path: str):
-        try:
+        return VideoDecoderWrapper(path)
-            return VideoReader(path, ctx=gpu(0), num_threads=1)
-        except (RuntimeError, OSError) as e:
-            logger.warning(
-                "[internvl] VideoReader gpu decode failed (%s), fallback CPU", e
-            )
-            return VideoReader(path, ctx=cpu(0), num_threads=1)
 
     def _ensure_placeholders_before_assistant(
         self, prompt: str, placeholder: str, want: int
@@ -488,11 +482,8 @@ class InternVLProcessor(BaseMultimodalProcessor):
 
         if base_output.videos and num_frames > 0 and self.video_token_id is not None:
             for video in base_output.videos:
-                vr = (
+                is_video_obj = isinstance(video, VideoDecoderWrapper)
-                    video
+                vr = video if is_video_obj else self._open_video_reader(str(video))
-                    if isinstance(video, VideoReader)
-                    else self._open_video_reader(str(video))
-                )
                 max_frame = len(vr) - 1
                 frame_indices = (
                     [0]
@@ -503,12 +494,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
                 per_video_tiles = []
                 per_video_patch_cnt = []
                 for fi in frame_indices:
-                    frame = vr[int(fi)]
+                    img_np = vr[int(fi)]
-                    img_np = (
-                        frame.asnumpy()
-                        if hasattr(frame, "asnumpy")
-                        else np.array(frame)
-                    )
                     frame_t = (
                         torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
                     )

python/sglang/srt/multimodal/processors/nano_nemotron_vl.py

@@ -12,7 +12,6 @@
 # limitations under the License.
 
 from math import sqrt
-from typing import TYPE_CHECKING
 
 import numpy as np
 import torch
@@ -28,9 +27,6 @@ from sglang.srt.multimodal.processors.base_processor import (
 )
 from sglang.srt.utils.common import sample_video_frames
 
-if TYPE_CHECKING:
-    from decord import VideoReader
-
 DEFAULT_NUM_TILES = 12
 NUM_VIDEO_TILES = 1
 DESIRED_FPS = 2  # TODO: allow desired fps/num frames to be configurable
@@ -99,13 +95,16 @@ class NanoNemotronVLImageProcessor(BaseMultimodalProcessor):
         return f"Frame {frame_index + 1} sampled at {timestamp:.2f} seconds: {self.PLACEHOLDER}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}"
 
     @staticmethod
-    def parse_video(video: "VideoReader") -> tuple[np.ndarray, list[float]]:
+    def parse_video(video) -> tuple[np.ndarray, list[float]]:
         frames = sample_video_frames(
             video, desired_fps=DESIRED_FPS, max_frames=MAX_FRAMES
         )
-        video_array = video.get_batch(frames).asnumpy()
+        video_array = video.get_frames_at(frames)
-        # doing the `1000 /` and then `/ 1000` is to match vllm's timestamping *exactly*, for reference.
+        avg_fps = video.avg_fps
-        frame_duration_ms = int(1000 / video.get_avg_fps())
+        if avg_fps > 0:
+            frame_duration_ms = int(1000 / avg_fps)
+        else:
+            frame_duration_ms = 0
         timestamps = [i * frame_duration_ms / 1000.0 for i in frames]
         return video_array, timestamps
 

python/sglang/srt/multimodal/processors/qwen_vl.py

@@ -7,7 +7,6 @@ from typing import List, Union
 import numpy as np
 import torch
 import torchvision
-from decord import VideoReader
 from PIL import Image
 from torchvision.transforms import InterpolationMode
 
@@ -29,6 +28,7 @@ from sglang.srt.multimodal.processors.base_processor import (
 from sglang.srt.multimodal.processors.base_processor import (
     MultimodalSpecialTokens,
 )
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
 from sglang.utils import logger
 
 IMAGE_FACTOR = 28
@@ -156,19 +156,22 @@ async def preprocess_video(
     video_config: dict = {},
 ) -> torch.Tensor:
     # preprocessed video
-    if not isinstance(vr, VideoReader):
+    is_video_obj = isinstance(vr, VideoDecoderWrapper)
+    if not is_video_obj:
         return vr
     entry_time = time.perf_counter()
 
-    total_frames, video_fps = len(vr), vr.get_avg_fps()
+    total_frames, video_fps = len(vr), vr.avg_fps
+
     nframes = smart_nframes(
         video_config, total_frames=total_frames, video_fps=video_fps
     )
     idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
     idx = np.unique(idx)
-    video_np = vr.get_batch(idx).asnumpy()
+
-    video = torch.from_numpy(video_np).pin_memory()
+    video = vr.get_frames_as_tensor(idx.tolist())
-    video = video.permute(0, 3, 1, 2)  # Convert to TCHW format
+
+    video = video.permute(0, 3, 1, 2)  # NHWC -> TCHW
 
     nframes, _, height, width = video.shape
     min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS)

python/sglang/srt/utils/common.py

@@ -94,11 +94,9 @@ from typing_extensions import Literal
 
 from sglang.srt.environ import envs
 from sglang.srt.observability.func_timer import enable_func_timer
+from sglang.srt.utils.video_decoder import VideoDecoderWrapper
 
 if TYPE_CHECKING:
-    # Apparently importing this here is necessary to avoid a segfault, see comment in load_video below
-    from decord import VideoReader
-
     from sglang.srt.server_args import ServerArgs
 
 logger = logging.getLogger(__name__)
@@ -956,75 +954,55 @@ def get_image_bytes(image_file: Union[str, bytes]):
         raise NotImplementedError(f"Invalid image: {image_file}")
 
 
-def load_video(video_file: Union[str, bytes], use_gpu: bool = True):
+def _normalize_video_input(
-    # We import decord here to avoid a strange Segmentation fault (core dumped) issue.
+    video_file: Union[str, bytes],
-    from decord import VideoReader, cpu, gpu
+) -> Union[str, bytes, None]:
+    """Normalize video input (URL, base64, file://, etc.) to a file path or bytes.
 
-    try:
+    Returns a file path or bytes suitable for a decoder, or None on failure.
-        from decord.bridge import decord_bridge
+    URLs and base64 are returned as bytes (no temp files needed since both
-
+    torchcodec and VideoDecoderWrapper accept bytes natively).
-        ctx = gpu(0)
+    """
-        _ = decord_bridge.get_ctx_device(ctx)
+    if isinstance(video_file, bytes):
-    except Exception:
+        return video_file
-        ctx = cpu(0)
+    elif isinstance(video_file, str):
-
+        if video_file.startswith(("http://", "https://")):
-    tmp_file = None
+            timeout = int(os.getenv("REQUEST_TIMEOUT", "10"))
-    vr = None
+            response = requests.get(video_file, stream=True, timeout=timeout)
-    try:
+            response.raise_for_status()
-        if isinstance(video_file, bytes):
+            return response.content
-            tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
+        elif video_file.startswith("data:"):
-            tmp_file.write(video_file)
+            _, encoded = video_file.split(",", 1)
-            tmp_file.close()
+            return pybase64.b64decode(encoded, validate=True)
-            vr = VideoReader(tmp_file.name, ctx=ctx)
+        elif video_file.startswith("file://"):
-        elif isinstance(video_file, str):
+            return unquote(urlparse(video_file).path)
-            if video_file.startswith(("http://", "https://")):
+        elif os.path.isfile(unquote(urlparse(video_file).path)):
-                timeout = int(os.getenv("REQUEST_TIMEOUT", "10"))
+            return video_file
-                response = requests.get(video_file, stream=True, timeout=timeout)
-                response.raise_for_status()
-                tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
-                for chunk in response.iter_content(chunk_size=8192):
-                    tmp_file.write(chunk)
-                tmp_file.close()
-                vr = VideoReader(tmp_file.name, ctx=ctx)
-            elif video_file.startswith("data:"):
-                _, encoded = video_file.split(",", 1)
-                video_bytes = pybase64.b64decode(encoded, validate=True)
-                tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
-                tmp_file.write(video_bytes)
-                tmp_file.close()
-                vr = VideoReader(tmp_file.name, ctx=ctx)
-            elif video_file.startswith("file://"):
-                video_file = unquote(urlparse(video_file).path)
-                vr = VideoReader(video_file, ctx=ctx)
-            # `urlparse` supports file:// paths, and so does VideoReader
-            elif os.path.isfile(unquote(urlparse(video_file).path)):
-                vr = VideoReader(video_file, ctx=ctx)
-            else:
-                video_bytes = pybase64.b64decode(video_file, validate=True)
-                tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
-                tmp_file.write(video_bytes)
-                tmp_file.close()
-                vr = VideoReader(tmp_file.name, ctx=ctx)
-        elif isinstance(video_file, (list, tuple, torch.Tensor, np.ndarray)):
-            vr = video_file
         else:
-            raise ValueError(f"Unsupported video input type: {type(video_file)}")
+            return pybase64.b64decode(video_file, validate=True)
-
+    else:
-        return vr
+        return None
-
-    finally:
-        if tmp_file and os.path.exists(tmp_file.name):
-            os.unlink(tmp_file.name)
 
 
-def sample_video_frames(
+def load_video(video_file: Union[str, bytes], use_gpu: bool = True):
-    video: "VideoReader", *, desired_fps: int, max_frames: int
+    if isinstance(video_file, (list, tuple, torch.Tensor, np.ndarray)):
-) -> list[int]:
+        return video_file
+
+    source = _normalize_video_input(video_file)
+    if source is None:
+        raise ValueError(f"Unsupported video input type: {type(video_file)}")
+
+    device = "cuda" if use_gpu else "cpu"
+    return VideoDecoderWrapper(source, device=device)
+
+
+def sample_video_frames(video, *, desired_fps: int, max_frames: int) -> list[int]:
     total_frames = len(video)
     assert total_frames > 0, "Video must have at least one frame"
 
-    duration = total_frames / video.get_avg_fps()
+    avg_fps = video.avg_fps
-    fps = min(desired_fps, video.get_avg_fps())
+    duration = total_frames / avg_fps if avg_fps > 0 else 0
+    fps = min(desired_fps, avg_fps)
 
     num_frames = math.floor(duration * fps)
     num_frames = min(max_frames, num_frames, total_frames)
@@ -1036,9 +1014,6 @@ def sample_video_frames(
 
 
 def encode_video(video_path, frame_count_limit=None):
-    # Lazy import because decord is not available on some arm platforms.
-    from decord import VideoReader, cpu
-
     if not os.path.exists(video_path):
         logger.error(f"Video {video_path} does not exist")
         return []
@@ -1051,14 +1026,23 @@ def encode_video(video_path, frame_count_limit=None):
         idxs = [int(i * gap + gap / 2) for i in range(n)]
         return [l[i] for i in idxs]
 
-    vr = VideoReader(video_path, ctx=cpu(0))
+    decoder = VideoDecoderWrapper(video_path)
-    sample_fps = round(vr.get_avg_fps() / 1)  # FPS
+    avg_fps = decoder.avg_fps
-    frame_indices = [i for i in range(0, len(vr), sample_fps)]
+    total_frames = len(decoder)
+
+    sample_fps = round(avg_fps / 1)
+    if sample_fps == 0:
+        sample_fps = 1
+    frame_indices = [i for i in range(0, total_frames, sample_fps)]
     if frame_count_limit is not None and len(frame_indices) > frame_count_limit:
         frame_indices = uniform_sample(frame_indices, frame_count_limit)
 
-    frames = vr.get_batch(frame_indices).asnumpy()
+    if not frame_indices:
-    frames = [Image.fromarray(v.astype("uint8")) for v in frames]
+        return []
+
+    frames_data = decoder.get_frames_at(frame_indices)
+    frames = [Image.fromarray(v.astype("uint8")) for v in frames_data]
+
     return frames
 
 

python/sglang/srt/utils/video_decoder.py

@@ -0,0 +1,129 @@
+"""Unified video decoder: torchcodec preferred, decord as fallback."""
+
+import logging
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+try:
+    from torchcodec.decoders import VideoDecoder
+
+    _BACKEND = "torchcodec"
+except (ImportError, RuntimeError):
+    _BACKEND = "decord"
+
+
+_cuda_backend_enabled: bool | None = None
+
+
+def _try_cuda_backend() -> bool:
+    """Try to enable torchcodec CUDA backend. Caches result after first call."""
+    global _cuda_backend_enabled
+    if _cuda_backend_enabled is not None:
+        return _cuda_backend_enabled
+    try:
+        from torchcodec.decoders import set_cuda_backend
+
+        set_cuda_backend("beta")
+        _cuda_backend_enabled = True
+    except Exception:
+        _cuda_backend_enabled = False
+    return _cuda_backend_enabled
+
+
+class VideoDecoderWrapper:
+    """Unified video decoder that uses torchcodec when available, decord as fallback.
+
+    All frames are returned in NHWC uint8 numpy format for consistency.
+    """
+
+    def __init__(self, source, device: str = "cpu"):
+        """source: file path (str) or video bytes.
+        device: "cpu" or "cuda". GPU decoding only supported with torchcodec.
+        """
+        self._tmp_path = None
+        if _BACKEND == "torchcodec":
+            kwargs = {"dimension_order": "NHWC"}
+            if device == "cuda" and _try_cuda_backend():
+                kwargs["device"] = "cuda"
+            try:
+                self._decoder = VideoDecoder(source, **kwargs)
+            except RuntimeError:
+                if "device" in kwargs:
+                    logger.warning("CUDA video decoding failed, falling back to CPU.")
+                    kwargs.pop("device")
+                    self._decoder = VideoDecoder(source, **kwargs)
+                else:
+                    raise
+        else:
+            from decord import VideoReader, cpu
+
+            if isinstance(source, bytes):
+                import os
+                import tempfile
+
+                fd, tmp_path = tempfile.mkstemp(suffix=".mp4")
+                try:
+                    os.write(fd, source)
+                finally:
+                    os.close(fd)
+                self._tmp_path = tmp_path
+                self._decoder = VideoReader(tmp_path, ctx=cpu(0))
+            else:
+                self._decoder = VideoReader(source, ctx=cpu(0))
+
+    def __len__(self):
+        return len(self._decoder)
+
+    def __getitem__(self, idx):
+        """Return single frame as numpy NHWC uint8."""
+        if _BACKEND == "torchcodec":
+            return self._decoder[idx].numpy()
+        else:
+            frame = self._decoder[idx]
+            return frame.asnumpy() if hasattr(frame, "asnumpy") else np.array(frame)
+
+    @property
+    def avg_fps(self) -> float:
+        if _BACKEND == "torchcodec":
+            return self._decoder.metadata.average_fps
+        else:
+            return self._decoder.get_avg_fps()
+
+    def get_frames_at(self, indices: list) -> np.ndarray:
+        """Return frames at given indices as numpy array with shape (N, H, W, C)."""
+        if _BACKEND == "torchcodec":
+            batch = self._decoder.get_frames_at(indices)
+            return batch.data.numpy()
+        else:
+            return self._decoder.get_batch(indices).asnumpy()
+
+    def get_frames_as_tensor(self, indices: list):
+        """Return frames at given indices as a torch tensor (NHWC, uint8, pinned memory)."""
+        import torch
+
+        if _BACKEND == "torchcodec":
+            batch = self._decoder.get_frames_at(indices)
+            return batch.data.pin_memory()
+        else:
+            arr = self._decoder.get_batch(indices).asnumpy()
+            return torch.from_numpy(arr).pin_memory()
+
+    def close(self):
+        """Explicitly clean up temporary files."""
+        if self._tmp_path is not None:
+            import os
+
+            if os.path.exists(self._tmp_path):
+                os.unlink(self._tmp_path)
+            self._tmp_path = None
+
+    def __del__(self):
+        self.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

python/sglang/test/vlm_utils.py

@@ -378,20 +378,16 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase):
         # the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa
         # the size of the video embeds differs from the `modality` argument when preprocessed
 
-        # We import decord here to avoid a strange Segmentation fault (core dumped) issue.
+        from sglang.srt.utils.video_decoder import VideoDecoderWrapper
-        # The following import order will cause Segmentation fault.
-        # import decord
-        # from transformers import AutoTokenizer
-        from decord import VideoReader, cpu
 
         max_frames_num = 10
-        vr = VideoReader(video_path, ctx=cpu(0))
+        decoder = VideoDecoderWrapper(video_path)
-        total_frame_num = len(vr)
+        total_frame_num = len(decoder)
         uniform_sampled_frames = np.linspace(
             0, total_frame_num - 1, max_frames_num, dtype=int
         )
         frame_idx = uniform_sampled_frames.tolist()
-        frames = vr.get_batch(frame_idx).asnumpy()
+        frames = decoder.get_frames_at(frame_idx)
 
         base64_frames = []
         for frame in frames:

scripts/ci/cuda/ci_install_dependency.sh

@@ -28,10 +28,10 @@ echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}"
 # The NVIDIA driver packages may have broken dependencies that are unrelated to these packages
 # Run apt-get update first to refresh package index (stale index causes 404 on security.ubuntu.com)
 apt-get update || true
-apt-get install -y --no-install-recommends python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils || {
+apt-get install -y --no-install-recommends python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils ffmpeg libavcodec-dev libavformat-dev libavutil-dev libswscale-dev || {
     echo "Warning: apt-get install failed, checking if required packages are available..."
     # Verify the packages we need are actually installed
-    for pkg in python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils; do
+    for pkg in python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils ffmpeg; do
         if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then
             echo "ERROR: Required package $pkg is not installed and apt-get failed"
             exit 1

test/registered/vlm/test_video_utils.py

@@ -16,7 +16,8 @@ class DummyVideo:
     def __len__(self):
         return self._frames
 
-    def get_avg_fps(self):
+    @property
+    def avg_fps(self):
         return self._fps
 
 

test/registered/vlm/test_vision_chunked_prefill.py

@@ -40,19 +40,15 @@ logger = logging.getLogger(__name__)
 class TestVisionChunkedPrefill(CustomTestCase):
 
     def prepare_video_messages(self, video_path, max_frames_num=8):
-        # We import decord here to avoid a strange Segmentation fault (core dumped) issue.
+        from sglang.srt.utils.video_decoder import VideoDecoderWrapper
-        # The following import order will cause Segmentation fault.
-        # import decord
-        # from transformers import AutoTokenizer
-        from decord import VideoReader, cpu
 
-        vr = VideoReader(video_path, ctx=cpu(0))
+        decoder = VideoDecoderWrapper(video_path)
-        total_frame_num = len(vr)
+        total_frame_num = len(decoder)
         uniform_sampled_frames = np.linspace(
             0, total_frame_num - 1, max_frames_num, dtype=int
         )
         frame_idx = uniform_sampled_frames.tolist()
-        frames = vr.get_batch(frame_idx).asnumpy()
+        frames = decoder.get_frames_at(frame_idx)
 
         base64_frames = []
         for frame in frames: