chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[MUSA][10/N] Add GGUF support (#18357)

Browse Source 
Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>
This commit is contained in:
R0CKSTAR
2026-03-07 02:50:35 +08:00
committed by GitHub
parent e8f2b80340
commit de1a0afcbc
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
python/sglang/srt/layers/quantization/gguf.py
View File

@@ -20,7 +20,7 @@ from sglang.srt.layers.quantization.base_config import (
QuantizeMethodBase,
)
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.utils import is_cuda, is_hip, is_xpu, set_weight_attrs
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_xpu, set_weight_attrs
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher import (
@@ -31,8 +31,9 @@ if TYPE_CHECKING:
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_xpu = is_xpu()
_is_musa = is_musa()
if _is_cuda:
if _is_cuda or _is_musa:
from sgl_kernel import gelu_and_mul, moe_align_block_size, moe_sum, silu_and_mul
from sgl_kernel.quantization import (
ggml_dequantize,
@@ -44,7 +45,7 @@ if _is_cuda:
)
else:
if not _is_hip:
warnings.warn(f"Only CUDA support GGUF quantization currently.")
warnings.warn(f"Only CUDA and MUSA support GGUF quantization currently.")
logger = logging.getLogger(__name__)
@@ -55,7 +56,7 @@ class GGUFConfig(QuantizationConfig):
def __init__(self, modules_to_not_convert: list[str] | None = None) -> None:
super().__init__()
if _is_hip:
warnings.warn(f"Only CUDA support GGUF quantization currently.")
warnings.warn(f"Only CUDA and MUSA support GGUF quantization currently.")
self.modules_to_not_convert = modules_to_not_convert or []
def __repr__(self) -> str:
@@ -72,7 +73,7 @@ class GGUFConfig(QuantizationConfig):
@classmethod
def get_min_capability(cls) -> int:
return 60
return 60 if not _is_musa else 21
@classmethod
def get_config_filenames(cls) -> list[str]: