diff --git a/python/sglang/srt/models/glm4v.py b/python/sglang/srt/models/glm4v.py index ae6edb631..05bb14af4 100644 --- a/python/sglang/srt/models/glm4v.py +++ b/python/sglang/srt/models/glm4v.py @@ -51,7 +51,7 @@ from sglang.srt.managers.mm_utils import ( general_mm_embed_routine, ) from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs -from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.glm4 import Glm4Model from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model @@ -659,6 +659,7 @@ class Glm4vForConditionalGeneration(nn.Module): positions: torch.Tensor, forward_batch: ForwardBatch, get_embedding: bool = False, + pp_proxy_tensors: Optional[PPProxyTensors] = None, ): """Run forward pass for GLM-4.1V. @@ -691,6 +692,7 @@ class Glm4vForConditionalGeneration(nn.Module): language_model=self.model, multimodal_model=self, positions=positions, + pp_proxy_tensors=pp_proxy_tensors, ) aux_hidden_states = None @@ -750,6 +752,17 @@ class Glm4vForConditionalGeneration(nn.Module): (".gate_up_proj", ".gate_proj", 0), ] params_dict = dict(self.named_parameters(remove_duplicate=False)) + + # For the PP case, we add special handling for lm_head.weight, + # - On non–last ranks: we continue, because this stage is supposed to + # be just an empty PPMissingLayer shell. + # - On the last rank: params_dict is expected to contain lm_head.weight, + # so it will never hit the branch "if name not in params_dict". + # + # For all other parameters, such like + # "model.visual.blocks.20.mlp.gate_proj.weight", the unified rule is: + # If this name does not exist in the current rank’s params_dict, + # it does not belong to this pipeline stage, thus we simply continue. for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: continue @@ -757,6 +770,8 @@ class Glm4vForConditionalGeneration(nn.Module): name = name.replace(r"model.language_model.", r"model.") if "model.visual." in name: name = name.replace("model.visual.", "visual.") + if name.startswith("lm_head.") and not self.pp_group.is_last_rank: + continue for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: @@ -766,6 +781,10 @@ class Glm4vForConditionalGeneration(nn.Module): # Skip loading extra bias for GPTQ models. if name.endswith(".bias") and name not in params_dict: continue + + if name not in params_dict: + continue + param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) @@ -779,6 +798,10 @@ class Glm4vForConditionalGeneration(nn.Module): # Skip loading extra bias for GPTQ models. if name.endswith(".bias") and name not in params_dict: continue + + if name not in params_dict: + continue + param = params_dict[name] except KeyError: print(params_dict.keys()) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 98b2e8a4d..2c3e821be 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -57,6 +57,9 @@ DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instru DEFAULT_MODEL_NAME_FOR_TEST_MLA = "lmsys/sglang-ci-dsv3-test" DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = "lmsys/sglang-ci-dsv3-test-NextN" +# VL test models +DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP = "zai-org/GLM-4.1V-9B-Thinking" + # NVFP4 models DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST = "nvidia/DeepSeek-V3-0324-FP4" DEFAULT_MODEL_NAME_FOR_TEST_MOE_NVFP4 = "nvidia/Qwen3-30B-A3B-FP4" diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index ae97062c5..7812054e5 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -150,7 +150,7 @@ suites = { TestFile("test_gpt_oss_4gpu.py", 300), TestFile("test_local_attn.py", 411), TestFile("test_multi_instance_release_memory_occupation.py", 64), - TestFile("test_pp_single_node.py", 481), + TestFile("test_pp_single_node.py", 800), TestFile("test_piecewise_cuda_graph.py", 1200), ], "per-commit-8-gpu-h200": [ diff --git a/test/srt/test_pp_single_node.py b/test/srt/test_pp_single_node.py index e333c2d4c..037cdb2e0 100644 --- a/test/srt/test_pp_single_node.py +++ b/test/srt/test_pp_single_node.py @@ -19,6 +19,7 @@ from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -318,5 +319,42 @@ class TestFixedBugs(unittest.TestCase): ) +class TestGLM41VPPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + 1, + "--pp-size", + 2, + "--chunked-prefill-size", + 8192, + "--enable-multimodal", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmmu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmmu", + num_examples=None, + num_threads=32, + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.55) + + if __name__ == "__main__": unittest.main()