diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py
index b1fd7edf7..dedb90c6c 100644
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -1057,7 +1057,7 @@ class ServerArgs:
             # Multimodal models need more memory for the image processing,
             # so we adjust the mem_fraction_static accordingly.
             model_config = self.get_model_config()
-            if model_config.is_multimodal:
+            if model_config.is_multimodal and not self.language_only:
                 self.adjust_mem_fraction_for_vlm(model_config)
 
             # If symm mem is enabled and prealloc size is not set, set it to 4GB