[BUGFIX]: using language-only should not reserve space for the vision encoder (#18011)

Co-authored-by: root <root@ubuntu-nvidia.localdomain>

python/sglang/srt/server_args.py

@@ -1057,7 +1057,7 @@ class ServerArgs:
             # Multimodal models need more memory for the image processing,
             # so we adjust the mem_fraction_static accordingly.
             model_config = self.get_model_config()
-            if model_config.is_multimodal:
+            if model_config.is_multimodal and not self.language_only:
                 self.adjust_mem_fraction_for_vlm(model_config)
 
             # If symm mem is enabled and prealloc size is not set, set it to 4GB