diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 859f1c8b7..e2ef7d482 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -775,14 +775,36 @@ class ModelConfig: quant_algo = json_quant_configs.get("quant_algo", None) if quant_algo == "MIXED_PRECISION": - return {"quant_method": "w4afp8"} + return {"quant_method": "w4afp8", "quant_algo": quant_algo} elif quant_algo and ("FP4" in quant_algo or "NVFP4" in quant_algo): - return {"quant_method": "modelopt_fp4"} + return {"quant_method": "modelopt_fp4", "quant_algo": quant_algo} elif quant_algo and "FP8" in quant_algo: - return {"quant_method": "modelopt_fp8"} + return {"quant_method": "modelopt_fp8", "quant_algo": quant_algo} else: return None + def get_quantization_config_log_str(self) -> Optional[str]: + """ + Get a concise string representation of the quantization config for logging. + Returns something like "quant=fp8, fmt=e4m3" or "quant=gptq, bits=4". + """ + try: + quant_cfg = self._parse_quant_hf_config() + if not quant_cfg: + return None + + quant_method = quant_cfg.get("quant_method", "quantized") + log_str = f"quant={quant_method}" + + # Append interesting fields if they exist + for field in ["bits", "quant_algo", "fmt"]: + if field in quant_cfg: + log_str += f", {field}={quant_cfg[field]}" + + return log_str + except Exception: + return None + def _is_already_quantized(self) -> bool: """Check if the model is already quantized based on config files.""" # Check for quantization in hf_config (config.json) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 585ab89f4..28e004f35 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1041,11 +1041,15 @@ class ModelRunner(ModelRunnerKVCacheMixin): after_avail_memory = get_available_gpu_memory(self.device, self.gpu_id) self.weight_load_mem_usage = before_avail_memory - after_avail_memory + # Get quantization config from ModelConfig + # This handles both config.json (standard) and hf_quant_config.json (ModelOpt) + quant_str = self.model_config.get_quantization_config_log_str() + logger.info( f"Load weight end. " f"elapsed={time.perf_counter() - tic_total:.2f} s, " f"type={type(self.model).__name__}, " - f"dtype={self.dtype}, " + f"{quant_str + ', ' if quant_str else ''}" f"avail mem={after_avail_memory:.2f} GB, " f"mem usage={self.weight_load_mem_usage:.2f} GB." ) diff --git a/test/srt/test_quant_config_parsing.py b/test/srt/test_quant_config_parsing.py new file mode 100644 index 000000000..40f130b46 --- /dev/null +++ b/test/srt/test_quant_config_parsing.py @@ -0,0 +1,79 @@ +import unittest +from unittest.mock import MagicMock + +from sglang.srt.configs.model_config import ModelConfig + + +class MockHfConfig: + def __init__(self, quant_config=None): + self.quantization_config = quant_config + self.architectures = ["LlamaForCausalLM"] + self.model_type = "llama" + + +class TestQuantLogString(unittest.TestCase): + def test_qwen_fp8_config(self): + # Example from Qwen/Qwen3-4B-Thinking-2507-FP8 + quant_config = { + "activation_scheme": "dynamic", + "modules_to_not_convert": ["lm_head"], + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [128, 128], + } + + # Create a raw instance + model_config = ModelConfig.__new__(ModelConfig) + model_config._parse_quant_hf_config = MagicMock(return_value=quant_config) + + expected = "quant=fp8, fmt=e4m3" + result = model_config.get_quantization_config_log_str() + print(f"\n[Test Qwen FP8] Result: {result}") + self.assertEqual(result, expected) + + def test_llama_gptq_int4_config(self): + # Example from hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4 + quant_config = {"bits": 4, "quant_method": "gptq", "group_size": 128} + model_config = ModelConfig.__new__(ModelConfig) + model_config._parse_quant_hf_config = MagicMock(return_value=quant_config) + + expected = "quant=gptq, bits=4" + result = model_config.get_quantization_config_log_str() + print(f"\n[Test Llama GPTQ] Result: {result}") + self.assertEqual(result, expected) + + def test_awq_config(self): + quant_config = { + "quant_method": "awq", + "bits": 4, + "group_size": 128, + } + model_config = ModelConfig.__new__(ModelConfig) + model_config._parse_quant_hf_config = MagicMock(return_value=quant_config) + + expected = "quant=awq, bits=4" + result = model_config.get_quantization_config_log_str() + print(f"\n[Test AWQ] Result: {result}") + self.assertEqual(result, expected) + + def test_modelopt_nvfp4(self): + quant_config = {"quant_method": "modelopt_fp4", "quant_algo": "NVFP4"} + model_config = ModelConfig.__new__(ModelConfig) + model_config._parse_quant_hf_config = MagicMock(return_value=quant_config) + + expected = "quant=modelopt_fp4, quant_algo=NVFP4" + result = model_config.get_quantization_config_log_str() + print(f"\n[Test ModelOpt] Result: {result}") + self.assertEqual(result, expected) + + def test_no_quant_config(self): + model_config = ModelConfig.__new__(ModelConfig) + model_config._parse_quant_hf_config = MagicMock(return_value=None) + + result = model_config.get_quantization_config_log_str() + print(f"\n[Test No Quant] Result: {result}") + self.assertIsNone(result) + + +if __name__ == "__main__": + unittest.main()