chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Quantization] Supported w8a8 int8 quantized Gemma3 and Qwen-VL models (#8619)

Browse Source 
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
ichernob
2025-08-12 23:31:18 +03:00
committed by GitHub
parent 48afa8f14f
commit 83123f481e
3 changed files with 131 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
python/sglang/srt/model_loader/loader.py
View File

@@ -162,12 +162,24 @@ def _initialize_model(
model_class, _ = get_model_architecture(model_config)
packed_modules_mapping = getattr(model_class, "packed_modules_mapping", {})
if _is_npu:
packed_modules_mapping["fused_qkv_a_proj_with_mqa"] = [
"q_a_proj",
"kv_a_proj_with_mqa",
]
packed_modules_mapping["qkv_proj"] = ["q_proj", "k_proj", "v_proj"]
packed_modules_mapping["gate_up_proj"] = ["gate_proj", "up_proj"]
packed_modules_mapping.update(
{
"visual": {"qkv_proj": ["qkv"]},
"vision_model": {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"proj": ["out_proj"],
},
"model": {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"fused_qkv_a_proj_with_mqa": [
"q_a_proj",
"kv_a_proj_with_mqa",
],
},
}
)
quant_config = _get_quantization_config(
model_config, load_config, packed_modules_mapping
)