[diffusion] fix: fix serving with dit-layerwise-offload enabled (#16066)

Co-authored-by: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com>

python/sglang/multimodal_gen/runtime/utils/layerwise_offload.py

@@ -98,8 +98,11 @@ class LayerwiseOffloadManager:
                 continue
             self._offload_tensor(name, buf, layer_idx)
 
-        self.prefetch_layer(0, non_blocking=False)
-        if self.copy_stream is not None:
+        self.prepare_for_next_denoise(non_blocking=False)
+
+    def prepare_for_next_denoise(self, non_blocking=True):
+        self.prefetch_layer(0, non_blocking=non_blocking)
+        if not non_blocking and self.copy_stream is not None:
             torch.cuda.current_stream().wait_stream(self.copy_stream)
 
     @torch.compiler.disable