[VLM] feat: true on policy for vlm + fsdp (#14636)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Nan Jiang
2026-01-01 19:54:39 -05:00
committed by GitHub
parent 00562ee14a
commit 7254986342
12 changed files with 203 additions and 57 deletions

View File

@@ -17,6 +17,7 @@ from sglang.srt.managers.schedule_batch import (
MultimodalDataItem,
MultimodalInputFormat,
)
from sglang.srt.server_args import get_global_server_args
from sglang.srt.utils import envs, is_npu, load_audio, load_image, load_video, logger
from sglang.srt.utils.cuda_ipc_transport_utils import (
MM_FEATURE_CACHE_SIZE,
@@ -316,7 +317,9 @@ class BaseMultimodalProcessor(ABC):
and isinstance(processor.image_processor, BaseImageProcessorFast)
and not self.server_args.disable_fast_image_processor
):
if not _is_npu:
if get_global_server_args().rl_on_policy_target is not None:
kwargs["device"] = "cpu"
elif not _is_npu:
kwargs["device"] = "cuda"
elif processor.__class__.__name__ not in {
"Qwen2_5_VLProcessor",