fix: small bug for llama-405b fp16 (#733)

2024-07-25 21:14:54 -07:00
parent 7f6f2f0f09
commit 252e0f7bbd
2 changed files with 2 additions and 1 deletions
--- a/python/sglang/srt/utils.py
+++ b/python/sglang/srt/utils.py
@@ -626,6 +626,7 @@ def is_llama3_405b_fp8(model_config):
        and model_config.hf_config.intermediate_size == 53248
        and model_config.hf_config.num_hidden_layers == 126
        and model_config.hf_config.num_key_value_heads == 16
+        and hasattr(model_config.hf_config, "quantization_config")
        and model_config.hf_config.quantization_config["quant_method"] == "fbgemm_fp8"
    ):
        return True