[diffusion] fix: make lora compatible with layerwise-offload (#16298)

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

python/sglang/multimodal_gen/runtime/loader/component_loader.py

@@ -43,7 +43,6 @@ from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
     get_diffusers_component_config,
     get_hf_config,
 )
-from sglang.multimodal_gen.runtime.utils.layerwise_offload import OffloadableDiTMixin
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 from sglang.multimodal_gen.utils import PRECISION_TO_TYPE
 
@@ -738,15 +737,6 @@ class TransformerLoader(ComponentLoader):
 
         model = model.eval()
 
-        if server_args.dit_layerwise_offload:
-            # enable layerwise offload if possible
-            if isinstance(model, OffloadableDiTMixin):
-                model.configure_layerwise_offload(server_args)
-            else:
-                logger.info(
-                    "Disabling layerwise offload since current model does not support this feature"
-                )
-
         return model