model: support DeepSeek-OCR-2 (#17897)

python/sglang/srt/configs/deepseek_ocr.py

@@ -196,6 +196,7 @@ class DeepseekOCRProcessor(ProcessorMixin):
         sft_format: str = "deepseek",
         mask_prompt: bool = True,
         ignore_id: int = -100,
+        ocr2_mode: bool = False,
         **kwargs,
     ):
 
@@ -243,6 +244,7 @@ class DeepseekOCRProcessor(ProcessorMixin):
         self.sft_format = sft_format
         self.mask_prompt = mask_prompt
         self.ignore_id = ignore_id
+        self.ocr2_mode = ocr2_mode
 
         super().__init__(
             tokenizer,
@@ -359,6 +361,13 @@ class DeepseekOCRProcessor(ProcessorMixin):
 
         target_ids = torch.LongTensor(masked_tokenized_str)
 
+        has_images = len(images_list) > 0
+        has_local_crops = False
+        if len(images_spatial_crop) > 0:
+            has_local_crops = any(
+                crop[0] > 1 or crop[1] > 1 for crop in images_spatial_crop
+            )
+
         if len(images_list) == 0:
             images = torch.zeros((1, 3, self.image_size, self.image_size))
         else:
@@ -376,6 +385,8 @@ class DeepseekOCRProcessor(ProcessorMixin):
             images_seq_mask=images_seq_mask,
             images_spatial_crop=images_spatial_crop,
         )
+        prepare.has_images = has_images
+        prepare.has_local_crops = has_local_crops
 
         return prepare
 
@@ -481,15 +492,27 @@ class DeepseekOCRProcessor(ProcessorMixin):
                 (self.base_size // self.patch_size) / self.downsample_ratio
             )
 
-            tokenized_image = (
-                [self.image_token_id] * num_queries_base + [self.image_token_id]
-            ) * num_queries_base
-            tokenized_image += [self.image_token_id]
-            if num_width_tiles > 1 or num_height_tiles > 1:
-                tokenized_image += (
-                    [self.image_token_id] * (num_queries * num_width_tiles)
-                    + [self.image_token_id]
-                ) * (num_queries * num_height_tiles)
+            if self.ocr2_mode:
+                tokenized_image = []
+                if num_width_tiles > 1 or num_height_tiles > 1:
+                    tokenized_image += [self.image_token_id] * (
+                        num_queries * num_width_tiles * num_queries * num_height_tiles
+                    )
+                tokenized_image += [self.image_token_id] * (
+                    num_queries_base * num_queries_base
+                )
+                # One extra token for the view separator.
+                tokenized_image += [self.image_token_id]
+            else:
+                tokenized_image = (
+                    [self.image_token_id] * num_queries_base + [self.image_token_id]
+                ) * num_queries_base
+                tokenized_image += [self.image_token_id]
+                if num_width_tiles > 1 or num_height_tiles > 1:
+                    tokenized_image += (
+                        [self.image_token_id] * (num_queries * num_width_tiles)
+                        + [self.image_token_id]
+                    ) * (num_queries * num_height_tiles)
             tokenized_str += tokenized_image
 
             images_seq_mask += [True] * len(tokenized_image)

python/sglang/srt/configs/model_config.py

@@ -1050,7 +1050,12 @@ def _get_and_verify_dtype(
 ) -> torch.dtype:
     # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
     # because config.torch_dtype can be None.
-    config_dtype = getattr(config, "dtype", None)
+    if isinstance(config, dict):
+        config_dtype = config.get("dtype", None) or config.get("torch_dtype", None)
+        model_type = config.get("model_type", "")
+    else:
+        config_dtype = getattr(config, "dtype", None)
+        model_type = getattr(config, "model_type", "")
     if isinstance(config_dtype, str):
         config_dtype = _STR_DTYPE_TO_TORCH_DTYPE.get(config_dtype, None)
     if config_dtype is None:
@@ -1060,11 +1065,11 @@ def _get_and_verify_dtype(
         dtype = dtype.lower()
         if dtype == "auto":
             if config_dtype == torch.float32:
-                if config.model_type.startswith("gemma"):
-                    if config.model_type == "gemma":
+                if model_type.startswith("gemma"):
+                    if model_type == "gemma":
                         gemma_version = ""
                     else:
-                        gemma_version = config.model_type[5]
+                        gemma_version = model_type[5]
                     logger.info(
                         f"For Gemma {gemma_version}, we downcast float32 to bfloat16 instead "
                         "of float16 by default. Please specify `dtype` if you "