Reduce MoE memory usage (#6147)

2025-05-16 00:38:28 +08:00
parent cfc9f9ab8d
commit f194e14fb7
4 changed files with 75 additions and 40 deletions
--- a/python/sglang/srt/utils.py
+++ b/python/sglang/srt/utils.py
@@ -2100,3 +2100,7 @@ def log_info_on_rank0(logger, msg):

    if get_tensor_model_parallel_rank() == 0:
        logger.info(msg)
+
+
+def dispose_tensor(x: torch.Tensor):
+    x.set_(torch.empty((0,), device=x.device, dtype=x.dtype))