chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[VLM] Replace decord with torchcodec for video decoding (#20055)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: BakerBunker <17872844+BakerBunker@users.noreply.github.com>
This commit is contained in:
Xinyuan Tong
2026-03-09 11:23:49 +00:00
committed by GitHub
parent 2d6eb7dff0
commit 4a757990a1
17 changed files with 234 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/pyproject.toml
View File

@@ -23,7 +23,7 @@ dependencies = [
"build",
"compressed-tensors",
"cuda-python==12.9",
"decord2",
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
"datasets",
"einops",
"fastapi",
@@ -68,7 +68,8 @@ dependencies = [
"torch==2.9.1",
"torchao==0.9.0",
"torchaudio==2.9.1",
"torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
"torchcodec==0.9.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec 0.9.1 for torch 2.9.x. Not available on Linux ARM.
"av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
"torchvision",
"tqdm",
"transformers==4.57.1",

1
python/pyproject_cpu.toml
View File

@@ -23,7 +23,6 @@ dependencies = [
"build",
"compressed-tensors",
"datasets",
"decord; platform_machine == 'x86_64'",
"einops",
"fastapi",
"gguf",

1
python/pyproject_npu.toml
View File

@@ -22,7 +22,6 @@ dependencies = [
"av",
"build",
"compressed-tensors",
"decord2",
"datasets",
"einops",
"fastapi",

1
python/pyproject_other.toml
View File

@@ -24,7 +24,6 @@ runtime_common = [
"av",
"build",
"compressed-tensors",
"decord2",
"datasets",
"einops",
"fastapi",

3
python/pyproject_xpu.toml
View File

@@ -16,7 +16,7 @@ classifiers = [
dependencies = [
"torch==2.9.0",
"torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default.
"torchcodec==0.9.1 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. torch==2.9.0 on XPU uses 0.9.1
"av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
"torchaudio==2.9.0",
"torchvision",
@@ -28,7 +28,6 @@ dependencies = [
"build",
"compressed-tensors",
"datasets",
"decord",
"einops",
"fastapi",
"gguf",

2
python/sglang/check_env.py
View File

@@ -50,7 +50,7 @@ PACKAGE_LIST = [
"tiktoken",
"anthropic",
"litellm",
"decord2",
"torchcodec",
]

8
python/sglang/srt/multimodal/processors/base_processor.py
View File

@@ -385,8 +385,7 @@ class BaseMultimodalProcessor(ABC):
"""
estimate the total frame count from all visual input
"""
# Lazy import because decord is not available on some arm platforms.
from decord import VideoReader, cpu
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
# Before processing inputs
if not image_data or len(image_data) == 0:
@@ -395,9 +394,8 @@ class BaseMultimodalProcessor(ABC):
for image in image_data:
if isinstance(image, str) and image.startswith("video:"):
path = image[len("video:") :]
# Estimate frames for the video
vr = VideoReader(path, ctx=cpu(0))
num_frames = len(vr)
decoder = VideoDecoderWrapper(path)
num_frames = len(decoder)
else:
# For images, each contributes one frame
num_frames = 1

26
python/sglang/srt/multimodal/processors/internvl.py
View File

@@ -6,7 +6,6 @@ from typing import List
import numpy as np
import torch
from decord import VideoReader, cpu, gpu
from PIL import Image
from sglang.srt.managers.schedule_batch import (
@@ -20,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import (
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
logger = logging.getLogger(__name__)
@@ -205,14 +205,8 @@ class InternVLProcessor(BaseMultimodalProcessor):
return torch.stack(tiles).to(torch.bfloat16)
@staticmethod
def _open_video_reader(path: str) -> VideoReader:
try:
return VideoReader(path, ctx=gpu(0), num_threads=1)
except (RuntimeError, OSError) as e:
logger.warning(
"[internvl] VideoReader gpu decode failed (%s), fallback CPU", e
)
return VideoReader(path, ctx=cpu(0), num_threads=1)
def _open_video_reader(path: str):
return VideoDecoderWrapper(path)
def _ensure_placeholders_before_assistant(
self, prompt: str, placeholder: str, want: int
@@ -488,11 +482,8 @@ class InternVLProcessor(BaseMultimodalProcessor):
if base_output.videos and num_frames > 0 and self.video_token_id is not None:
for video in base_output.videos:
vr = (
video
if isinstance(video, VideoReader)
else self._open_video_reader(str(video))
)
is_video_obj = isinstance(video, VideoDecoderWrapper)
vr = video if is_video_obj else self._open_video_reader(str(video))
max_frame = len(vr) - 1
frame_indices = (
[0]
@@ -503,12 +494,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
per_video_tiles = []
per_video_patch_cnt = []
for fi in frame_indices:
frame = vr[int(fi)]
img_np = (
frame.asnumpy()
if hasattr(frame, "asnumpy")
else np.array(frame)
)
img_np = vr[int(fi)]
frame_t = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
)

15
python/sglang/srt/multimodal/processors/nano_nemotron_vl.py
View File

@@ -12,7 +12,6 @@
# limitations under the License.
from math import sqrt
from typing import TYPE_CHECKING
import numpy as np
import torch
@@ -28,9 +27,6 @@ from sglang.srt.multimodal.processors.base_processor import (
)
from sglang.srt.utils.common import sample_video_frames
if TYPE_CHECKING:
from decord import VideoReader
DEFAULT_NUM_TILES = 12
NUM_VIDEO_TILES = 1
DESIRED_FPS = 2 # TODO: allow desired fps/num frames to be configurable
@@ -99,13 +95,16 @@ class NanoNemotronVLImageProcessor(BaseMultimodalProcessor):
return f"Frame {frame_index + 1} sampled at {timestamp:.2f} seconds: {self.PLACEHOLDER}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}"
@staticmethod
def parse_video(video: "VideoReader") -> tuple[np.ndarray, list[float]]:
def parse_video(video) -> tuple[np.ndarray, list[float]]:
frames = sample_video_frames(
video, desired_fps=DESIRED_FPS, max_frames=MAX_FRAMES
)
video_array = video.get_batch(frames).asnumpy()
# doing the `1000 /` and then `/ 1000` is to match vllm's timestamping *exactly*, for reference.
frame_duration_ms = int(1000 / video.get_avg_fps())
video_array = video.get_frames_at(frames)
avg_fps = video.avg_fps
if avg_fps > 0:
frame_duration_ms = int(1000 / avg_fps)
else:
frame_duration_ms = 0
timestamps = [i * frame_duration_ms / 1000.0 for i in frames]
return video_array, timestamps

15
python/sglang/srt/multimodal/processors/qwen_vl.py
View File

@@ -7,7 +7,6 @@ from typing import List, Union
import numpy as np
import torch
import torchvision
from decord import VideoReader
from PIL import Image
from torchvision.transforms import InterpolationMode
@@ -29,6 +28,7 @@ from sglang.srt.multimodal.processors.base_processor import (
from sglang.srt.multimodal.processors.base_processor import (
MultimodalSpecialTokens,
)
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
from sglang.utils import logger
IMAGE_FACTOR = 28
@@ -156,19 +156,22 @@ async def preprocess_video(
video_config: dict = {},
) -> torch.Tensor:
# preprocessed video
if not isinstance(vr, VideoReader):
is_video_obj = isinstance(vr, VideoDecoderWrapper)
if not is_video_obj:
return vr
entry_time = time.perf_counter()
total_frames, video_fps = len(vr), vr.get_avg_fps()
total_frames, video_fps = len(vr), vr.avg_fps
nframes = smart_nframes(
video_config, total_frames=total_frames, video_fps=video_fps
)
idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
idx = np.unique(idx)
video_np = vr.get_batch(idx).asnumpy()
video = torch.from_numpy(video_np).pin_memory()
video = video.permute(0, 3, 1, 2) # Convert to TCHW format
video = vr.get_frames_as_tensor(idx.tolist())
video = video.permute(0, 3, 1, 2) # NHWC -> TCHW
nframes, _, height, width = video.shape
min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS)

130
python/sglang/srt/utils/common.py
View File

@@ -94,11 +94,9 @@ from typing_extensions import Literal
from sglang.srt.environ import envs
from sglang.srt.observability.func_timer import enable_func_timer
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
if TYPE_CHECKING:
# Apparently importing this here is necessary to avoid a segfault, see comment in load_video below
from decord import VideoReader
from sglang.srt.server_args import ServerArgs
logger = logging.getLogger(__name__)
@@ -956,75 +954,55 @@ def get_image_bytes(image_file: Union[str, bytes]):
raise NotImplementedError(f"Invalid image: {image_file}")
def load_video(video_file: Union[str, bytes], use_gpu: bool = True):
# We import decord here to avoid a strange Segmentation fault (core dumped) issue.
from decord import VideoReader, cpu, gpu
def _normalize_video_input(
video_file: Union[str, bytes],
) -> Union[str, bytes, None]:
"""Normalize video input (URL, base64, file://, etc.) to a file path or bytes.
try:
from decord.bridge import decord_bridge
ctx = gpu(0)
_ = decord_bridge.get_ctx_device(ctx)
except Exception:
ctx = cpu(0)
tmp_file = None
vr = None
try:
if isinstance(video_file, bytes):
tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
tmp_file.write(video_file)
tmp_file.close()
vr = VideoReader(tmp_file.name, ctx=ctx)
elif isinstance(video_file, str):
if video_file.startswith(("http://", "https://")):
timeout = int(os.getenv("REQUEST_TIMEOUT", "10"))
response = requests.get(video_file, stream=True, timeout=timeout)
response.raise_for_status()
tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
for chunk in response.iter_content(chunk_size=8192):
tmp_file.write(chunk)
tmp_file.close()
vr = VideoReader(tmp_file.name, ctx=ctx)
elif video_file.startswith("data:"):
_, encoded = video_file.split(",", 1)
video_bytes = pybase64.b64decode(encoded, validate=True)
tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
tmp_file.write(video_bytes)
tmp_file.close()
vr = VideoReader(tmp_file.name, ctx=ctx)
elif video_file.startswith("file://"):
video_file = unquote(urlparse(video_file).path)
vr = VideoReader(video_file, ctx=ctx)
# `urlparse` supports file:// paths, and so does VideoReader
elif os.path.isfile(unquote(urlparse(video_file).path)):
vr = VideoReader(video_file, ctx=ctx)
else:
video_bytes = pybase64.b64decode(video_file, validate=True)
tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
tmp_file.write(video_bytes)
tmp_file.close()
vr = VideoReader(tmp_file.name, ctx=ctx)
elif isinstance(video_file, (list, tuple, torch.Tensor, np.ndarray)):
vr = video_file
Returns a file path or bytes suitable for a decoder, or None on failure.
URLs and base64 are returned as bytes (no temp files needed since both
torchcodec and VideoDecoderWrapper accept bytes natively).
"""
if isinstance(video_file, bytes):
return video_file
elif isinstance(video_file, str):
if video_file.startswith(("http://", "https://")):
timeout = int(os.getenv("REQUEST_TIMEOUT", "10"))
response = requests.get(video_file, stream=True, timeout=timeout)
response.raise_for_status()
return response.content
elif video_file.startswith("data:"):
_, encoded = video_file.split(",", 1)
return pybase64.b64decode(encoded, validate=True)
elif video_file.startswith("file://"):
return unquote(urlparse(video_file).path)
elif os.path.isfile(unquote(urlparse(video_file).path)):
return video_file
else:
raise ValueError(f"Unsupported video input type: {type(video_file)}")
return vr
finally:
if tmp_file and os.path.exists(tmp_file.name):
os.unlink(tmp_file.name)
return pybase64.b64decode(video_file, validate=True)
else:
return None
def sample_video_frames(
video: "VideoReader", *, desired_fps: int, max_frames: int
) -> list[int]:
def load_video(video_file: Union[str, bytes], use_gpu: bool = True):
if isinstance(video_file, (list, tuple, torch.Tensor, np.ndarray)):
return video_file
source = _normalize_video_input(video_file)
if source is None:
raise ValueError(f"Unsupported video input type: {type(video_file)}")
device = "cuda" if use_gpu else "cpu"
return VideoDecoderWrapper(source, device=device)
def sample_video_frames(video, *, desired_fps: int, max_frames: int) -> list[int]:
total_frames = len(video)
assert total_frames > 0, "Video must have at least one frame"
duration = total_frames / video.get_avg_fps()
fps = min(desired_fps, video.get_avg_fps())
avg_fps = video.avg_fps
duration = total_frames / avg_fps if avg_fps > 0 else 0
fps = min(desired_fps, avg_fps)
num_frames = math.floor(duration * fps)
num_frames = min(max_frames, num_frames, total_frames)
@@ -1036,9 +1014,6 @@ def sample_video_frames(
def encode_video(video_path, frame_count_limit=None):
# Lazy import because decord is not available on some arm platforms.
from decord import VideoReader, cpu
if not os.path.exists(video_path):
logger.error(f"Video {video_path} does not exist")
return []
@@ -1051,14 +1026,23 @@ def encode_video(video_path, frame_count_limit=None):
idxs = [int(i * gap + gap / 2) for i in range(n)]
return [l[i] for i in idxs]
vr = VideoReader(video_path, ctx=cpu(0))
sample_fps = round(vr.get_avg_fps() / 1) # FPS
frame_indices = [i for i in range(0, len(vr), sample_fps)]
decoder = VideoDecoderWrapper(video_path)
avg_fps = decoder.avg_fps
total_frames = len(decoder)
sample_fps = round(avg_fps / 1)
if sample_fps == 0:
sample_fps = 1
frame_indices = [i for i in range(0, total_frames, sample_fps)]
if frame_count_limit is not None and len(frame_indices) > frame_count_limit:
frame_indices = uniform_sample(frame_indices, frame_count_limit)
frames = vr.get_batch(frame_indices).asnumpy()
frames = [Image.fromarray(v.astype("uint8")) for v in frames]
if not frame_indices:
return []
frames_data = decoder.get_frames_at(frame_indices)
frames = [Image.fromarray(v.astype("uint8")) for v in frames_data]
return frames

129
python/sglang/srt/utils/video_decoder.py Normal file
View File

@@ -0,0 +1,129 @@
"""Unified video decoder: torchcodec preferred, decord as fallback."""
import logging
import numpy as np
logger = logging.getLogger(__name__)
try:
from torchcodec.decoders import VideoDecoder
_BACKEND = "torchcodec"
except (ImportError, RuntimeError):
_BACKEND = "decord"
_cuda_backend_enabled: bool | None = None
def _try_cuda_backend() -> bool:
"""Try to enable torchcodec CUDA backend. Caches result after first call."""
global _cuda_backend_enabled
if _cuda_backend_enabled is not None:
return _cuda_backend_enabled
try:
from torchcodec.decoders import set_cuda_backend
set_cuda_backend("beta")
_cuda_backend_enabled = True
except Exception:
_cuda_backend_enabled = False
return _cuda_backend_enabled
class VideoDecoderWrapper:
"""Unified video decoder that uses torchcodec when available, decord as fallback.
All frames are returned in NHWC uint8 numpy format for consistency.
"""
def __init__(self, source, device: str = "cpu"):
"""source: file path (str) or video bytes.
device: "cpu" or "cuda". GPU decoding only supported with torchcodec.
"""
self._tmp_path = None
if _BACKEND == "torchcodec":
kwargs = {"dimension_order": "NHWC"}
if device == "cuda" and _try_cuda_backend():
kwargs["device"] = "cuda"
try:
self._decoder = VideoDecoder(source, **kwargs)
except RuntimeError:
if "device" in kwargs:
logger.warning("CUDA video decoding failed, falling back to CPU.")
kwargs.pop("device")
self._decoder = VideoDecoder(source, **kwargs)
else:
raise
else:
from decord import VideoReader, cpu
if isinstance(source, bytes):
import os
import tempfile
fd, tmp_path = tempfile.mkstemp(suffix=".mp4")
try:
os.write(fd, source)
finally:
os.close(fd)
self._tmp_path = tmp_path
self._decoder = VideoReader(tmp_path, ctx=cpu(0))
else:
self._decoder = VideoReader(source, ctx=cpu(0))
def __len__(self):
return len(self._decoder)
def __getitem__(self, idx):
"""Return single frame as numpy NHWC uint8."""
if _BACKEND == "torchcodec":
return self._decoder[idx].numpy()
else:
frame = self._decoder[idx]
return frame.asnumpy() if hasattr(frame, "asnumpy") else np.array(frame)
@property
def avg_fps(self) -> float:
if _BACKEND == "torchcodec":
return self._decoder.metadata.average_fps
else:
return self._decoder.get_avg_fps()
def get_frames_at(self, indices: list) -> np.ndarray:
"""Return frames at given indices as numpy array with shape (N, H, W, C)."""
if _BACKEND == "torchcodec":
batch = self._decoder.get_frames_at(indices)
return batch.data.numpy()
else:
return self._decoder.get_batch(indices).asnumpy()
def get_frames_as_tensor(self, indices: list):
"""Return frames at given indices as a torch tensor (NHWC, uint8, pinned memory)."""
import torch
if _BACKEND == "torchcodec":
batch = self._decoder.get_frames_at(indices)
return batch.data.pin_memory()
else:
arr = self._decoder.get_batch(indices).asnumpy()
return torch.from_numpy(arr).pin_memory()
def close(self):
"""Explicitly clean up temporary files."""
if self._tmp_path is not None:
import os
if os.path.exists(self._tmp_path):
os.unlink(self._tmp_path)
self._tmp_path = None
def __del__(self):
self.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()

12
python/sglang/test/vlm_utils.py
View File

@@ -378,20 +378,16 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase):
# the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa
# the size of the video embeds differs from the `modality` argument when preprocessed
# We import decord here to avoid a strange Segmentation fault (core dumped) issue.
# The following import order will cause Segmentation fault.
# import decord
# from transformers import AutoTokenizer
from decord import VideoReader, cpu
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
max_frames_num = 10
vr = VideoReader(video_path, ctx=cpu(0))
total_frame_num = len(vr)
decoder = VideoDecoderWrapper(video_path)
total_frame_num = len(decoder)
uniform_sampled_frames = np.linspace(
0, total_frame_num - 1, max_frames_num, dtype=int
)
frame_idx = uniform_sampled_frames.tolist()
frames = vr.get_batch(frame_idx).asnumpy()
frames = decoder.get_frames_at(frame_idx)
base64_frames = []
for frame in frames: