[Bug fix] [PP] fix wrong dtype for quantified model (#12247)

Signed-off-by: Xuchun Shang <xuchun.shang@gmail.com>
2025-10-28 16:27:24 +08:00
parent ea96106000
commit a1f2dc90e4
1 changed files with 2 additions and 2 deletions
--- a/python/sglang/srt/model_executor/cuda_graph_runner.py
+++ b/python/sglang/srt/model_executor/cuda_graph_runner.py
@@ -323,11 +323,11 @@ class CudaGraphRunner:
                self.pp_proxy_tensors = {
                    "hidden_states": torch.zeros(
                        (self.max_bs, self.model_runner.model_config.hidden_size),
-                        dtype=torch.bfloat16,
+                        dtype=self.model_runner.model_config.dtype,
                    ),
                    "residual": torch.zeros(
                        (self.max_bs, self.model_runner.model_config.hidden_size),
-                        dtype=torch.bfloat16,
+                        dtype=self.model_runner.model_config.dtype,
                    ),
                }