From feda2b11c463961a1510f8b4655220c146c740ed Mon Sep 17 00:00:00 2001 From: Bruce Changlong Xu Date: Wed, 4 Mar 2026 19:50:55 -0800 Subject: [PATCH] [AMD] Add AWQ AMD CI coverage and quantization platform compatibility docs (#19550) --- docs/advanced_features/quantization.md | 28 +++++++++++++++++++++ docs/platforms/amd_gpu.md | 35 ++++++++++++++++++++++++++ test/registered/quant/test_awq.py | 6 ++++- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/docs/advanced_features/quantization.md b/docs/advanced_features/quantization.md index 7aa21c747..ce14bbaf2 100644 --- a/docs/advanced_features/quantization.md +++ b/docs/advanced_features/quantization.md @@ -17,6 +17,34 @@ or [NeuralMagic](https://huggingface.co/collections/neuralmagic) collections on popular quality validated quantized models. Quantized models must be validated via benchmarks post-quantization to guard against abnormal quantization loss regressions. +## Platform Compatibility + +The following table summarizes quantization method support across NVIDIA and AMD GPUs. + +| Method | NVIDIA GPUs | AMD GPUs (MI300X/MI325X/MI350X) | Notes | +|--------|:-----------:|:-------------------------------:|-------| +| `fp8` | Yes | Yes | Aiter or Triton backend on AMD | +| `mxfp4` | Yes | Yes | Requires CDNA3/CDNA4 with MXFP support; uses Aiter | +| `blockwise_int8` | Yes | Yes | Triton-based, works on both platforms | +| `w8a8_int8` | Yes | Yes | | +| `w8a8_fp8` | Yes | Yes | Aiter or Triton FP8 on AMD | +| `awq` | Yes | Yes | Uses Triton dequantize on AMD (vs. optimized CUDA kernels on NVIDIA) | +| `gptq` | Yes | Yes | Uses Triton or vLLM kernels on AMD | +| `compressed-tensors` | Yes | Yes | Aiter paths for FP8/MoE on AMD | +| `quark` | Yes | Yes | AMD Quark quantization; Aiter GEMM paths on AMD | +| `auto-round` | Yes | Yes | Platform-agnostic (Intel auto-round) | +| `quark_int4fp8_moe` | No | Yes | AMD-only; online INT4-to-FP8 MoE quantization (CDNA3/CDNA4) | +| `awq_marlin` | Yes | No | Marlin kernels are CUDA-only | +| `gptq_marlin` | Yes | No | Marlin kernels are CUDA-only | +| `gguf` | Yes | No | CUDA-only kernels in sgl-kernel | +| `modelopt` / `modelopt_fp8` | Yes | No | NVIDIA ModelOpt, requires NVIDIA hardware | +| `modelopt_fp4` | Yes (Blackwell) | No | NVIDIA Blackwell only | +| `petit_nvfp4` | Yes (Blackwell) | No | NVIDIA NvFP4, Blackwell only | +| `bitsandbytes` | Yes | Experimental | Depends on bitsandbytes ROCm support | +| `torchao` (`int4wo`, etc.) | Yes | Partial | `int4wo` not supported on AMD; other methods may work | + +On AMD, several of these methods use [Aiter](https://github.com/ROCm/aiter) for acceleration -- set `SGLANG_USE_AITER=1` where noted. See [AMD GPU setup](../platforms/amd_gpu.md) for installation and configuration details. + ## Offline Quantization To load already quantized models, simply load the model weights and config. **Again, if the model has been quantized offline, diff --git a/docs/platforms/amd_gpu.md b/docs/platforms/amd_gpu.md index 613bd39ba..e3eae156b 100644 --- a/docs/platforms/amd_gpu.md +++ b/docs/platforms/amd_gpu.md @@ -114,6 +114,41 @@ The steps below show how to build and use an image. With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang’s machine learning capabilities. +## Quantization on AMD GPUs + +The [Quantization documentation](../advanced_features/quantization.md#platform-compatibility) has a full compatibility matrix. The short version: FP8, AWQ, MXFP4, W8A8, GPTQ, compressed-tensors, and Quark all work on AMD. Methods that depend on Marlin or NVIDIA-specific kernels (`awq_marlin`, `gptq_marlin`, `gguf`, `modelopt_fp8`, `modelopt_fp4`, `petit_nvfp4`) do not. + +A few things to keep in mind: + +- FP8 works via Aiter or Triton. Pre-quantized FP8 models like DeepSeek-V3/R1 work out of the box. +- AWQ uses Triton dequantization kernels on AMD. The faster Marlin path is not available. +- MXFP4 requires CDNA3/CDNA4 and `SGLANG_USE_AITER=1`. +- `quark_int4fp8_moe` is an AMD-only online quantization method for MoE models on CDNA3/CDNA4. + +Several of these backends are accelerated by [Aiter](https://github.com/ROCm/aiter). Enable it with: + +```bash +export SGLANG_USE_AITER=1 +``` + +Example -- serving an AWQ model: + +```bash +python3 -m sglang.launch_server \ + --model-path hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4 \ + --trust-remote-code \ + --port 30000 --host 0.0.0.0 +``` + +Example -- FP8 online quantization: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --quantization fp8 \ + --port 30000 --host 0.0.0.0 +``` + ## Examples ### Running DeepSeek-V3 diff --git a/test/registered/quant/test_awq.py b/test/registered/quant/test_awq.py index 42d2e7f52..cc640b851 100644 --- a/test/registered/quant/test_awq.py +++ b/test/registered/quant/test_awq.py @@ -2,17 +2,19 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + is_in_amd_ci, popen_launch_server, ) register_cuda_ci(est_time=163, suite="stage-b-test-large-1-gpu") +register_amd_ci(est_time=200, suite="stage-b-test-large-1-gpu-amd") class TestAWQ(CustomTestCase): @@ -44,6 +46,7 @@ class TestAWQ(CustomTestCase): self.assertGreater(metrics["score"], 0.64) +@unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs") class TestAWQMarlinBfloat16(CustomTestCase): """ Verify that the model can be loaded with bfloat16 dtype and awq_marlin quantization @@ -77,6 +80,7 @@ class TestAWQMarlinBfloat16(CustomTestCase): self.assertGreater(metrics["score"], 0.83) +@unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs") class TestAWQMarlinFloat16(CustomTestCase): """ Verify that the model can be loaded with float16 dtype and awq_marlin quantization