diff --git a/examples/runtime/multimodal/llava_onevision_server.py b/examples/runtime/multimodal/llava_onevision_server.py index 2cf16e3bd..3908ab9a3 100644 --- a/examples/runtime/multimodal/llava_onevision_server.py +++ b/examples/runtime/multimodal/llava_onevision_server.py @@ -15,11 +15,12 @@ import numpy as np import openai import pybase64 import requests -from decord import VideoReader, cpu from PIL import Image +from sglang.srt.utils.video_decoder import VideoDecoderWrapper + # pip install httpx==0.23.3 -# pip install decord +# pip install torchcodec # pip install protobuf==3.20.0 @@ -200,13 +201,13 @@ def video_speed_test(client, video_path): def prepare_video_messages(video_path): max_frames_num = 32 - vr = VideoReader(video_path, ctx=cpu(0)) - total_frame_num = len(vr) + decoder = VideoDecoderWrapper(video_path) + total_frame_num = len(decoder) uniform_sampled_frames = np.linspace( 0, total_frame_num - 1, max_frames_num, dtype=int ) frame_idx = uniform_sampled_frames.tolist() - frames = vr.get_batch(frame_idx).asnumpy() + frames = decoder.get_frames_at(frame_idx) base64_frames = [] for frame in frames: diff --git a/python/pyproject.toml b/python/pyproject.toml index d0feacb86..3b2bc9b93 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "build", "compressed-tensors", "cuda-python==12.9", - "decord2", + "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "datasets", "einops", "fastapi", @@ -68,7 +68,8 @@ dependencies = [ "torch==2.9.1", "torchao==0.9.0", "torchaudio==2.9.1", - "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. + "torchcodec==0.9.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec 0.9.1 for torch 2.9.x. Not available on Linux ARM. + "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "torchvision", "tqdm", "transformers==4.57.1", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index e28e6d99b..0087f823a 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -23,7 +23,6 @@ dependencies = [ "build", "compressed-tensors", "datasets", - "decord; platform_machine == 'x86_64'", "einops", "fastapi", "gguf", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index da87a936b..26044a6f9 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -22,7 +22,6 @@ dependencies = [ "av", "build", "compressed-tensors", - "decord2", "datasets", "einops", "fastapi", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 6dccf38a3..83cfbba04 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -24,7 +24,6 @@ runtime_common = [ "av", "build", "compressed-tensors", - "decord2", "datasets", "einops", "fastapi", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index c9c56e1c2..6c843b57d 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -16,7 +16,7 @@ classifiers = [ dependencies = [ "torch==2.9.0", - "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. + "torchcodec==0.9.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. torch==2.9.0 on XPU uses 0.9.1 "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "torchaudio==2.9.0", "torchvision", @@ -28,7 +28,6 @@ dependencies = [ "build", "compressed-tensors", "datasets", - "decord", "einops", "fastapi", "gguf", diff --git a/python/sglang/check_env.py b/python/sglang/check_env.py index 8a312c560..4bff85d2e 100644 --- a/python/sglang/check_env.py +++ b/python/sglang/check_env.py @@ -50,7 +50,7 @@ PACKAGE_LIST = [ "tiktoken", "anthropic", "litellm", - "decord2", + "torchcodec", ] diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index 22adf71e2..4cc4cfb50 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -385,8 +385,7 @@ class BaseMultimodalProcessor(ABC): """ estimate the total frame count from all visual input """ - # Lazy import because decord is not available on some arm platforms. - from decord import VideoReader, cpu + from sglang.srt.utils.video_decoder import VideoDecoderWrapper # Before processing inputs if not image_data or len(image_data) == 0: @@ -395,9 +394,8 @@ class BaseMultimodalProcessor(ABC): for image in image_data: if isinstance(image, str) and image.startswith("video:"): path = image[len("video:") :] - # Estimate frames for the video - vr = VideoReader(path, ctx=cpu(0)) - num_frames = len(vr) + decoder = VideoDecoderWrapper(path) + num_frames = len(decoder) else: # For images, each contributes one frame num_frames = 1 diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index 2c05608d9..e9a0753e4 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -6,7 +6,6 @@ from typing import List import numpy as np import torch -from decord import VideoReader, cpu, gpu from PIL import Image from sglang.srt.managers.schedule_batch import ( @@ -20,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import ( BaseMultiModalProcessorOutput, MultimodalSpecialTokens, ) +from sglang.srt.utils.video_decoder import VideoDecoderWrapper logger = logging.getLogger(__name__) @@ -205,14 +205,8 @@ class InternVLProcessor(BaseMultimodalProcessor): return torch.stack(tiles).to(torch.bfloat16) @staticmethod - def _open_video_reader(path: str) -> VideoReader: - try: - return VideoReader(path, ctx=gpu(0), num_threads=1) - except (RuntimeError, OSError) as e: - logger.warning( - "[internvl] VideoReader gpu decode failed (%s), fallback CPU", e - ) - return VideoReader(path, ctx=cpu(0), num_threads=1) + def _open_video_reader(path: str): + return VideoDecoderWrapper(path) def _ensure_placeholders_before_assistant( self, prompt: str, placeholder: str, want: int @@ -488,11 +482,8 @@ class InternVLProcessor(BaseMultimodalProcessor): if base_output.videos and num_frames > 0 and self.video_token_id is not None: for video in base_output.videos: - vr = ( - video - if isinstance(video, VideoReader) - else self._open_video_reader(str(video)) - ) + is_video_obj = isinstance(video, VideoDecoderWrapper) + vr = video if is_video_obj else self._open_video_reader(str(video)) max_frame = len(vr) - 1 frame_indices = ( [0] @@ -503,12 +494,7 @@ class InternVLProcessor(BaseMultimodalProcessor): per_video_tiles = [] per_video_patch_cnt = [] for fi in frame_indices: - frame = vr[int(fi)] - img_np = ( - frame.asnumpy() - if hasattr(frame, "asnumpy") - else np.array(frame) - ) + img_np = vr[int(fi)] frame_t = ( torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0 ) diff --git a/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py b/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py index 7464bd341..83d72441f 100644 --- a/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py +++ b/python/sglang/srt/multimodal/processors/nano_nemotron_vl.py @@ -12,7 +12,6 @@ # limitations under the License. from math import sqrt -from typing import TYPE_CHECKING import numpy as np import torch @@ -28,9 +27,6 @@ from sglang.srt.multimodal.processors.base_processor import ( ) from sglang.srt.utils.common import sample_video_frames -if TYPE_CHECKING: - from decord import VideoReader - DEFAULT_NUM_TILES = 12 NUM_VIDEO_TILES = 1 DESIRED_FPS = 2 # TODO: allow desired fps/num frames to be configurable @@ -99,13 +95,16 @@ class NanoNemotronVLImageProcessor(BaseMultimodalProcessor): return f"Frame {frame_index + 1} sampled at {timestamp:.2f} seconds: {self.PLACEHOLDER}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}" @staticmethod - def parse_video(video: "VideoReader") -> tuple[np.ndarray, list[float]]: + def parse_video(video) -> tuple[np.ndarray, list[float]]: frames = sample_video_frames( video, desired_fps=DESIRED_FPS, max_frames=MAX_FRAMES ) - video_array = video.get_batch(frames).asnumpy() - # doing the `1000 /` and then `/ 1000` is to match vllm's timestamping *exactly*, for reference. - frame_duration_ms = int(1000 / video.get_avg_fps()) + video_array = video.get_frames_at(frames) + avg_fps = video.avg_fps + if avg_fps > 0: + frame_duration_ms = int(1000 / avg_fps) + else: + frame_duration_ms = 0 timestamps = [i * frame_duration_ms / 1000.0 for i in frames] return video_array, timestamps diff --git a/python/sglang/srt/multimodal/processors/qwen_vl.py b/python/sglang/srt/multimodal/processors/qwen_vl.py index 4395654e4..ec5195745 100644 --- a/python/sglang/srt/multimodal/processors/qwen_vl.py +++ b/python/sglang/srt/multimodal/processors/qwen_vl.py @@ -7,7 +7,6 @@ from typing import List, Union import numpy as np import torch import torchvision -from decord import VideoReader from PIL import Image from torchvision.transforms import InterpolationMode @@ -29,6 +28,7 @@ from sglang.srt.multimodal.processors.base_processor import ( from sglang.srt.multimodal.processors.base_processor import ( MultimodalSpecialTokens, ) +from sglang.srt.utils.video_decoder import VideoDecoderWrapper from sglang.utils import logger IMAGE_FACTOR = 28 @@ -156,19 +156,22 @@ async def preprocess_video( video_config: dict = {}, ) -> torch.Tensor: # preprocessed video - if not isinstance(vr, VideoReader): + is_video_obj = isinstance(vr, VideoDecoderWrapper) + if not is_video_obj: return vr entry_time = time.perf_counter() - total_frames, video_fps = len(vr), vr.get_avg_fps() + total_frames, video_fps = len(vr), vr.avg_fps + nframes = smart_nframes( video_config, total_frames=total_frames, video_fps=video_fps ) idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64) idx = np.unique(idx) - video_np = vr.get_batch(idx).asnumpy() - video = torch.from_numpy(video_np).pin_memory() - video = video.permute(0, 3, 1, 2) # Convert to TCHW format + + video = vr.get_frames_as_tensor(idx.tolist()) + + video = video.permute(0, 3, 1, 2) # NHWC -> TCHW nframes, _, height, width = video.shape min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index fbbea936e..6bba0ae3a 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -94,11 +94,9 @@ from typing_extensions import Literal from sglang.srt.environ import envs from sglang.srt.observability.func_timer import enable_func_timer +from sglang.srt.utils.video_decoder import VideoDecoderWrapper if TYPE_CHECKING: - # Apparently importing this here is necessary to avoid a segfault, see comment in load_video below - from decord import VideoReader - from sglang.srt.server_args import ServerArgs logger = logging.getLogger(__name__) @@ -956,75 +954,55 @@ def get_image_bytes(image_file: Union[str, bytes]): raise NotImplementedError(f"Invalid image: {image_file}") -def load_video(video_file: Union[str, bytes], use_gpu: bool = True): - # We import decord here to avoid a strange Segmentation fault (core dumped) issue. - from decord import VideoReader, cpu, gpu +def _normalize_video_input( + video_file: Union[str, bytes], +) -> Union[str, bytes, None]: + """Normalize video input (URL, base64, file://, etc.) to a file path or bytes. - try: - from decord.bridge import decord_bridge - - ctx = gpu(0) - _ = decord_bridge.get_ctx_device(ctx) - except Exception: - ctx = cpu(0) - - tmp_file = None - vr = None - try: - if isinstance(video_file, bytes): - tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") - tmp_file.write(video_file) - tmp_file.close() - vr = VideoReader(tmp_file.name, ctx=ctx) - elif isinstance(video_file, str): - if video_file.startswith(("http://", "https://")): - timeout = int(os.getenv("REQUEST_TIMEOUT", "10")) - response = requests.get(video_file, stream=True, timeout=timeout) - response.raise_for_status() - tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") - for chunk in response.iter_content(chunk_size=8192): - tmp_file.write(chunk) - tmp_file.close() - vr = VideoReader(tmp_file.name, ctx=ctx) - elif video_file.startswith("data:"): - _, encoded = video_file.split(",", 1) - video_bytes = pybase64.b64decode(encoded, validate=True) - tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") - tmp_file.write(video_bytes) - tmp_file.close() - vr = VideoReader(tmp_file.name, ctx=ctx) - elif video_file.startswith("file://"): - video_file = unquote(urlparse(video_file).path) - vr = VideoReader(video_file, ctx=ctx) - # `urlparse` supports file:// paths, and so does VideoReader - elif os.path.isfile(unquote(urlparse(video_file).path)): - vr = VideoReader(video_file, ctx=ctx) - else: - video_bytes = pybase64.b64decode(video_file, validate=True) - tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") - tmp_file.write(video_bytes) - tmp_file.close() - vr = VideoReader(tmp_file.name, ctx=ctx) - elif isinstance(video_file, (list, tuple, torch.Tensor, np.ndarray)): - vr = video_file + Returns a file path or bytes suitable for a decoder, or None on failure. + URLs and base64 are returned as bytes (no temp files needed since both + torchcodec and VideoDecoderWrapper accept bytes natively). + """ + if isinstance(video_file, bytes): + return video_file + elif isinstance(video_file, str): + if video_file.startswith(("http://", "https://")): + timeout = int(os.getenv("REQUEST_TIMEOUT", "10")) + response = requests.get(video_file, stream=True, timeout=timeout) + response.raise_for_status() + return response.content + elif video_file.startswith("data:"): + _, encoded = video_file.split(",", 1) + return pybase64.b64decode(encoded, validate=True) + elif video_file.startswith("file://"): + return unquote(urlparse(video_file).path) + elif os.path.isfile(unquote(urlparse(video_file).path)): + return video_file else: - raise ValueError(f"Unsupported video input type: {type(video_file)}") - - return vr - - finally: - if tmp_file and os.path.exists(tmp_file.name): - os.unlink(tmp_file.name) + return pybase64.b64decode(video_file, validate=True) + else: + return None -def sample_video_frames( - video: "VideoReader", *, desired_fps: int, max_frames: int -) -> list[int]: +def load_video(video_file: Union[str, bytes], use_gpu: bool = True): + if isinstance(video_file, (list, tuple, torch.Tensor, np.ndarray)): + return video_file + + source = _normalize_video_input(video_file) + if source is None: + raise ValueError(f"Unsupported video input type: {type(video_file)}") + + device = "cuda" if use_gpu else "cpu" + return VideoDecoderWrapper(source, device=device) + + +def sample_video_frames(video, *, desired_fps: int, max_frames: int) -> list[int]: total_frames = len(video) assert total_frames > 0, "Video must have at least one frame" - duration = total_frames / video.get_avg_fps() - fps = min(desired_fps, video.get_avg_fps()) + avg_fps = video.avg_fps + duration = total_frames / avg_fps if avg_fps > 0 else 0 + fps = min(desired_fps, avg_fps) num_frames = math.floor(duration * fps) num_frames = min(max_frames, num_frames, total_frames) @@ -1036,9 +1014,6 @@ def sample_video_frames( def encode_video(video_path, frame_count_limit=None): - # Lazy import because decord is not available on some arm platforms. - from decord import VideoReader, cpu - if not os.path.exists(video_path): logger.error(f"Video {video_path} does not exist") return [] @@ -1051,14 +1026,23 @@ def encode_video(video_path, frame_count_limit=None): idxs = [int(i * gap + gap / 2) for i in range(n)] return [l[i] for i in idxs] - vr = VideoReader(video_path, ctx=cpu(0)) - sample_fps = round(vr.get_avg_fps() / 1) # FPS - frame_indices = [i for i in range(0, len(vr), sample_fps)] + decoder = VideoDecoderWrapper(video_path) + avg_fps = decoder.avg_fps + total_frames = len(decoder) + + sample_fps = round(avg_fps / 1) + if sample_fps == 0: + sample_fps = 1 + frame_indices = [i for i in range(0, total_frames, sample_fps)] if frame_count_limit is not None and len(frame_indices) > frame_count_limit: frame_indices = uniform_sample(frame_indices, frame_count_limit) - frames = vr.get_batch(frame_indices).asnumpy() - frames = [Image.fromarray(v.astype("uint8")) for v in frames] + if not frame_indices: + return [] + + frames_data = decoder.get_frames_at(frame_indices) + frames = [Image.fromarray(v.astype("uint8")) for v in frames_data] + return frames diff --git a/python/sglang/srt/utils/video_decoder.py b/python/sglang/srt/utils/video_decoder.py new file mode 100644 index 000000000..1153e0382 --- /dev/null +++ b/python/sglang/srt/utils/video_decoder.py @@ -0,0 +1,129 @@ +"""Unified video decoder: torchcodec preferred, decord as fallback.""" + +import logging + +import numpy as np + +logger = logging.getLogger(__name__) + +try: + from torchcodec.decoders import VideoDecoder + + _BACKEND = "torchcodec" +except (ImportError, RuntimeError): + _BACKEND = "decord" + + +_cuda_backend_enabled: bool | None = None + + +def _try_cuda_backend() -> bool: + """Try to enable torchcodec CUDA backend. Caches result after first call.""" + global _cuda_backend_enabled + if _cuda_backend_enabled is not None: + return _cuda_backend_enabled + try: + from torchcodec.decoders import set_cuda_backend + + set_cuda_backend("beta") + _cuda_backend_enabled = True + except Exception: + _cuda_backend_enabled = False + return _cuda_backend_enabled + + +class VideoDecoderWrapper: + """Unified video decoder that uses torchcodec when available, decord as fallback. + + All frames are returned in NHWC uint8 numpy format for consistency. + """ + + def __init__(self, source, device: str = "cpu"): + """source: file path (str) or video bytes. + device: "cpu" or "cuda". GPU decoding only supported with torchcodec. + """ + self._tmp_path = None + if _BACKEND == "torchcodec": + kwargs = {"dimension_order": "NHWC"} + if device == "cuda" and _try_cuda_backend(): + kwargs["device"] = "cuda" + try: + self._decoder = VideoDecoder(source, **kwargs) + except RuntimeError: + if "device" in kwargs: + logger.warning("CUDA video decoding failed, falling back to CPU.") + kwargs.pop("device") + self._decoder = VideoDecoder(source, **kwargs) + else: + raise + else: + from decord import VideoReader, cpu + + if isinstance(source, bytes): + import os + import tempfile + + fd, tmp_path = tempfile.mkstemp(suffix=".mp4") + try: + os.write(fd, source) + finally: + os.close(fd) + self._tmp_path = tmp_path + self._decoder = VideoReader(tmp_path, ctx=cpu(0)) + else: + self._decoder = VideoReader(source, ctx=cpu(0)) + + def __len__(self): + return len(self._decoder) + + def __getitem__(self, idx): + """Return single frame as numpy NHWC uint8.""" + if _BACKEND == "torchcodec": + return self._decoder[idx].numpy() + else: + frame = self._decoder[idx] + return frame.asnumpy() if hasattr(frame, "asnumpy") else np.array(frame) + + @property + def avg_fps(self) -> float: + if _BACKEND == "torchcodec": + return self._decoder.metadata.average_fps + else: + return self._decoder.get_avg_fps() + + def get_frames_at(self, indices: list) -> np.ndarray: + """Return frames at given indices as numpy array with shape (N, H, W, C).""" + if _BACKEND == "torchcodec": + batch = self._decoder.get_frames_at(indices) + return batch.data.numpy() + else: + return self._decoder.get_batch(indices).asnumpy() + + def get_frames_as_tensor(self, indices: list): + """Return frames at given indices as a torch tensor (NHWC, uint8, pinned memory).""" + import torch + + if _BACKEND == "torchcodec": + batch = self._decoder.get_frames_at(indices) + return batch.data.pin_memory() + else: + arr = self._decoder.get_batch(indices).asnumpy() + return torch.from_numpy(arr).pin_memory() + + def close(self): + """Explicitly clean up temporary files.""" + if self._tmp_path is not None: + import os + + if os.path.exists(self._tmp_path): + os.unlink(self._tmp_path) + self._tmp_path = None + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() diff --git a/python/sglang/test/vlm_utils.py b/python/sglang/test/vlm_utils.py index 818c375d3..24ced63eb 100644 --- a/python/sglang/test/vlm_utils.py +++ b/python/sglang/test/vlm_utils.py @@ -378,20 +378,16 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase): # the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa # the size of the video embeds differs from the `modality` argument when preprocessed - # We import decord here to avoid a strange Segmentation fault (core dumped) issue. - # The following import order will cause Segmentation fault. - # import decord - # from transformers import AutoTokenizer - from decord import VideoReader, cpu + from sglang.srt.utils.video_decoder import VideoDecoderWrapper max_frames_num = 10 - vr = VideoReader(video_path, ctx=cpu(0)) - total_frame_num = len(vr) + decoder = VideoDecoderWrapper(video_path) + total_frame_num = len(decoder) uniform_sampled_frames = np.linspace( 0, total_frame_num - 1, max_frames_num, dtype=int ) frame_idx = uniform_sampled_frames.tolist() - frames = vr.get_batch(frame_idx).asnumpy() + frames = decoder.get_frames_at(frame_idx) base64_frames = [] for frame in frames: diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 5e6dc3a3a..527472fcd 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -28,10 +28,10 @@ echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" # The NVIDIA driver packages may have broken dependencies that are unrelated to these packages # Run apt-get update first to refresh package index (stale index causes 404 on security.ubuntu.com) apt-get update || true -apt-get install -y --no-install-recommends python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils || { +apt-get install -y --no-install-recommends python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils ffmpeg libavcodec-dev libavformat-dev libavutil-dev libswscale-dev || { echo "Warning: apt-get install failed, checking if required packages are available..." # Verify the packages we need are actually installed - for pkg in python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils; do + for pkg in python3 python3-pip python3-venv python3-dev git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils ffmpeg; do if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then echo "ERROR: Required package $pkg is not installed and apt-get failed" exit 1 diff --git a/test/registered/vlm/test_video_utils.py b/test/registered/vlm/test_video_utils.py index a124988e4..7ba166609 100644 --- a/test/registered/vlm/test_video_utils.py +++ b/test/registered/vlm/test_video_utils.py @@ -16,7 +16,8 @@ class DummyVideo: def __len__(self): return self._frames - def get_avg_fps(self): + @property + def avg_fps(self): return self._fps diff --git a/test/registered/vlm/test_vision_chunked_prefill.py b/test/registered/vlm/test_vision_chunked_prefill.py index 104f979ca..27cfab3a3 100644 --- a/test/registered/vlm/test_vision_chunked_prefill.py +++ b/test/registered/vlm/test_vision_chunked_prefill.py @@ -40,19 +40,15 @@ logger = logging.getLogger(__name__) class TestVisionChunkedPrefill(CustomTestCase): def prepare_video_messages(self, video_path, max_frames_num=8): - # We import decord here to avoid a strange Segmentation fault (core dumped) issue. - # The following import order will cause Segmentation fault. - # import decord - # from transformers import AutoTokenizer - from decord import VideoReader, cpu + from sglang.srt.utils.video_decoder import VideoDecoderWrapper - vr = VideoReader(video_path, ctx=cpu(0)) - total_frame_num = len(vr) + decoder = VideoDecoderWrapper(video_path) + total_frame_num = len(decoder) uniform_sampled_frames = np.linspace( 0, total_frame_num - 1, max_frames_num, dtype=int ) frame_idx = uniform_sampled_frames.tolist() - frames = vr.get_batch(frame_idx).asnumpy() + frames = decoder.get_frames_at(frame_idx) base64_frames = [] for frame in frames: