chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Quantization] Supported w8a8 int8 quantized Gemma3 and Qwen-VL models (#8619)

Browse Source 
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
ichernob
2025-08-12 23:31:18 +03:00
committed by GitHub
parent 48afa8f14f
commit 83123f481e
3 changed files with 131 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
python/sglang/srt/layers/quantization/w8a8_int8.py
View File

@@ -255,17 +255,23 @@ class W8A8Int8Config(QuantizationConfig):
if _is_npu:
if isinstance(layer, LinearBase):
key = "model"
if "vision_model" in prefix:
key = "vision_model"
elif "visual" in prefix:
key = "visual"
packed_modules_mapping_subset = self.packed_modules_mapping.get(key, {})
prefix_in_quant_config = prefix
proj_name = prefix.split(".")[-1]
if proj_name in self.packed_modules_mapping:
if proj_name in packed_modules_mapping_subset:
prefix_in_quant_config = prefix.replace(
proj_name, self.packed_modules_mapping[proj_name][0]
proj_name, packed_modules_mapping_subset[proj_name][0]
)
self.is_dynamic = (
self.quant_description[prefix_in_quant_config + ".weight"]
== "W8A8_DYNAMIC"
)
if self.is_layer_skipped(prefix, self.packed_modules_mapping):
if self.is_layer_skipped(prefix, packed_modules_mapping_subset):
return UnquantizedLinearMethod()
return (
NPU_W8A8DynamicLinearMethod(self)