chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[VLM] Replace decord with torchcodec for video decoding (#20055)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: BakerBunker <17872844+BakerBunker@users.noreply.github.com>
This commit is contained in:
Xinyuan Tong
2026-03-09 11:23:49 +00:00
committed by GitHub
parent 2d6eb7dff0
commit 4a757990a1
17 changed files with 234 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/multimodal/processors/base_processor.py
View File

@@ -385,8 +385,7 @@ class BaseMultimodalProcessor(ABC):
"""
estimate the total frame count from all visual input
"""
# Lazy import because decord is not available on some arm platforms.
from decord import VideoReader, cpu
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
# Before processing inputs
if not image_data or len(image_data) == 0:
@@ -395,9 +394,8 @@ class BaseMultimodalProcessor(ABC):
for image in image_data:
if isinstance(image, str) and image.startswith("video:"):
path = image[len("video:") :]
# Estimate frames for the video
vr = VideoReader(path, ctx=cpu(0))
num_frames = len(vr)
decoder = VideoDecoderWrapper(path)
num_frames = len(decoder)
else:
# For images, each contributes one frame
num_frames = 1

26
python/sglang/srt/multimodal/processors/internvl.py
View File

@@ -6,7 +6,6 @@ from typing import List
import numpy as np
import torch
from decord import VideoReader, cpu, gpu
from PIL import Image
from sglang.srt.managers.schedule_batch import (
@@ -20,6 +19,7 @@ from sglang.srt.multimodal.processors.base_processor import (
BaseMultiModalProcessorOutput,
MultimodalSpecialTokens,
)
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
logger = logging.getLogger(__name__)
@@ -205,14 +205,8 @@ class InternVLProcessor(BaseMultimodalProcessor):
return torch.stack(tiles).to(torch.bfloat16)
@staticmethod
def _open_video_reader(path: str) -> VideoReader:
try:
return VideoReader(path, ctx=gpu(0), num_threads=1)
except (RuntimeError, OSError) as e:
logger.warning(
"[internvl] VideoReader gpu decode failed (%s), fallback CPU", e
)
return VideoReader(path, ctx=cpu(0), num_threads=1)
def _open_video_reader(path: str):
return VideoDecoderWrapper(path)
def _ensure_placeholders_before_assistant(
self, prompt: str, placeholder: str, want: int
@@ -488,11 +482,8 @@ class InternVLProcessor(BaseMultimodalProcessor):
if base_output.videos and num_frames > 0 and self.video_token_id is not None:
for video in base_output.videos:
vr = (
video
if isinstance(video, VideoReader)
else self._open_video_reader(str(video))
)
is_video_obj = isinstance(video, VideoDecoderWrapper)
vr = video if is_video_obj else self._open_video_reader(str(video))
max_frame = len(vr) - 1
frame_indices = (
[0]
@@ -503,12 +494,7 @@ class InternVLProcessor(BaseMultimodalProcessor):
per_video_tiles = []
per_video_patch_cnt = []
for fi in frame_indices:
frame = vr[int(fi)]
img_np = (
frame.asnumpy()
if hasattr(frame, "asnumpy")
else np.array(frame)
)
img_np = vr[int(fi)]
frame_t = (
torch.from_numpy(img_np).permute(2, 0, 1).cuda().float() / 255.0
)

15
python/sglang/srt/multimodal/processors/nano_nemotron_vl.py
View File

@@ -12,7 +12,6 @@
# limitations under the License.
from math import sqrt
from typing import TYPE_CHECKING
import numpy as np
import torch
@@ -28,9 +27,6 @@ from sglang.srt.multimodal.processors.base_processor import (
)
from sglang.srt.utils.common import sample_video_frames
if TYPE_CHECKING:
from decord import VideoReader
DEFAULT_NUM_TILES = 12
NUM_VIDEO_TILES = 1
DESIRED_FPS = 2 # TODO: allow desired fps/num frames to be configurable
@@ -99,13 +95,16 @@ class NanoNemotronVLImageProcessor(BaseMultimodalProcessor):
return f"Frame {frame_index + 1} sampled at {timestamp:.2f} seconds: {self.PLACEHOLDER}{self.IMG_CONTEXT_TOKEN * num_tokens}{self.IMG_END_TOKEN}"
@staticmethod
def parse_video(video: "VideoReader") -> tuple[np.ndarray, list[float]]:
def parse_video(video) -> tuple[np.ndarray, list[float]]:
frames = sample_video_frames(
video, desired_fps=DESIRED_FPS, max_frames=MAX_FRAMES
)
video_array = video.get_batch(frames).asnumpy()
# doing the `1000 /` and then `/ 1000` is to match vllm's timestamping *exactly*, for reference.
frame_duration_ms = int(1000 / video.get_avg_fps())
video_array = video.get_frames_at(frames)
avg_fps = video.avg_fps
if avg_fps > 0:
frame_duration_ms = int(1000 / avg_fps)
else:
frame_duration_ms = 0
timestamps = [i * frame_duration_ms / 1000.0 for i in frames]
return video_array, timestamps

15
python/sglang/srt/multimodal/processors/qwen_vl.py
View File

@@ -7,7 +7,6 @@ from typing import List, Union
import numpy as np
import torch
import torchvision
from decord import VideoReader
from PIL import Image
from torchvision.transforms import InterpolationMode
@@ -29,6 +28,7 @@ from sglang.srt.multimodal.processors.base_processor import (
from sglang.srt.multimodal.processors.base_processor import (
MultimodalSpecialTokens,
)
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
from sglang.utils import logger
IMAGE_FACTOR = 28
@@ -156,19 +156,22 @@ async def preprocess_video(
video_config: dict = {},
) -> torch.Tensor:
# preprocessed video
if not isinstance(vr, VideoReader):
is_video_obj = isinstance(vr, VideoDecoderWrapper)
if not is_video_obj:
return vr
entry_time = time.perf_counter()
total_frames, video_fps = len(vr), vr.get_avg_fps()
total_frames, video_fps = len(vr), vr.avg_fps
nframes = smart_nframes(
video_config, total_frames=total_frames, video_fps=video_fps
)
idx = np.linspace(0, total_frames - 1, num=nframes, dtype=np.int64)
idx = np.unique(idx)
video_np = vr.get_batch(idx).asnumpy()
video = torch.from_numpy(video_np).pin_memory()
video = video.permute(0, 3, 1, 2) # Convert to TCHW format
video = vr.get_frames_as_tensor(idx.tolist())
video = video.permute(0, 3, 1, 2) # NHWC -> TCHW
nframes, _, height, width = video.shape
min_pixels = video_config.get("min_pixels", VIDEO_MIN_PIXELS)