chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 14 Packages Projects Releases Wiki Activity

[AMD] rocm 7.2 image release, PR test, Nightly Test (#17799)

Browse Source 
Co-authored-by: Alan Kao <akao@amd.com>
Co-authored-by: bingxche <Bingxu.Chen@amd.com>
Co-authored-by: Michael <13900043+michaelzhang-ai@users.noreply.github.com>
This commit is contained in:
YC Tseng
2026-02-11 21:29:25 -08:00
committed by GitHub
co-authored by Alan Kao bingxche Michael
parent 93ede0db19
commit 20554a0a4f
25 changed files with 2716 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/amd/accuracy/mi30x/test_gpt_oss_eval_amd.py
+2 -2
View File
@@ -68,7 +68,7 @@ GPT_OSS_MODELS = [
"triton",
"--trust-remote-code",
],
env_vars={"SGLANG_USE_AITER": "0"},
env_vars={"SGLANG_USE_AITER": "1"},
),
ModelConfig(
model_path="lmsys/gpt-oss-120b-bf16",
@@ -86,7 +86,7 @@ GPT_OSS_MODELS = [
"triton",
"--trust-remote-code",
],
env_vars={"SGLANG_USE_AITER": "0"},
env_vars={"SGLANG_USE_AITER": "1"},
),
]
test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py
+4 -4
View File
@@ -41,7 +41,7 @@ MODEL_SCORE_THRESHOLDS = {
# Llama 3.2 series (smaller models)
"meta-llama/Llama-3.2-3B-Instruct": 0.55,
# Mistral series
"mistralai/Mistral-7B-Instruct-v0.3": 0.58,
"mistralai/Mistral-7B-Instruct-v0.3": 0.55,
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.61,
# DeepSeek series
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
@@ -108,10 +108,10 @@ DISABLE_HF_XET_MODELS = {
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8",
}
TRITON_MOE_MODELS = {
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8",
# "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8",
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8",
"mistralai/Mixtral-8x7B-Instruct-v0.1",
"mistralai/Mistral-7B-Instruct-v0.3",
# "mistralai/Mixtral-8x7B-Instruct-v0.1",
# "mistralai/Mistral-7B-Instruct-v0.3",
}
# AMD-specific models that need special launch config (matching in-house CI sanity_check.py)
# AMD_SPECIAL_CONFIG_MODELS = {
test/registered/amd/accuracy/mi30x/test_vlms_mmmu_eval_amd.py
+3 -3
View File
@@ -120,9 +120,9 @@ AMD_VLM_MODELS = [
# Models that need special handling on AMD (MoE models)
TRITON_ATTENTION_MODELS = {
"deepseek-ai/deepseek-vl2-small",
"Qwen/Qwen3-VL-30B-A3B-Instruct",
"moonshotai/Kimi-VL-A3B-Instruct",
# "deepseek-ai/deepseek-vl2-small",
# "Qwen/Qwen3-VL-30B-A3B-Instruct",
# "moonshotai/Kimi-VL-A3B-Instruct",
}
# Models known to fail on AMD - exclude from testing
test/registered/amd/accuracy/mi35x/test_deepseek_v32_eval_mi35x.py
+3 -3
View File
@@ -32,9 +32,9 @@ from sglang.test.test_utils import (
)
from sglang.utils import download_and_cache_file, read_jsonl
# Register for AMD CI - MI35x DeepSeek-V3.2 accuracy test (~60 min for basic only)
# Register for AMD CI - MI35x DeepSeek-V3.2 accuracy test (~90 min for basic only)
register_amd_ci(
est_time=3600,
est_time=5400,
suite="nightly-amd-8-gpu-mi35x-deepseek-v32",
nightly=True,
)
@@ -74,7 +74,7 @@ MI35X_DEEPSEEK_V32_MODELS = [
model_path="deepseek-ai/DeepSeek-V3.2",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
timeout=5400,
variant="basic",
other_args=[
"--trust-remote-code",
test/registered/amd/accuracy/mi35x/test_deepseek_v32_mtp_eval_mi35x.py
+12 -10
View File
@@ -22,7 +22,6 @@ from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
@@ -32,7 +31,7 @@ from sglang.test.test_utils import (
# Register for AMD CI - MI35x DeepSeek-V3.2 TP+MTP accuracy test
register_amd_ci(
est_time=3600,
est_time=5400,
suite="nightly-amd-accuracy-8-gpu-mi35x-deepseek-v32-mtp",
nightly=True,
)
@@ -55,10 +54,15 @@ class TestDeepseekV32TPMTP(CustomTestCase):
def setUpClass(cls):
cls.model = DEEPSEEK_V32_MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
# Use same args as perf test (which passes successfully)
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
@@ -67,19 +71,17 @@ class TestDeepseekV32TPMTP(CustomTestCase):
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-frac",
"--mem-fraction-static",
"0.7",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
"--watchdog-timeout",
"1200",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
timeout=5400,
other_args=other_args,
)
@@ -97,8 +99,8 @@ class TestDeepseekV32TPMTP(CustomTestCase):
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
num_questions=200,
parallel=64,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
test/registered/amd/accuracy/mi35x/test_gpt_oss_eval_mi35x.py
+2 -6
View File
@@ -75,9 +75,7 @@ MI35X_GPT_OSS_MODELS = [
"triton",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_AITER": "0"
}, # Disabled due to SWA eviction bug with aiter (#17220)
env_vars={"SGLANG_USE_AITER": "1"},
),
ModelConfig(
model_path="openai/gpt-oss-120b",
@@ -95,9 +93,7 @@ MI35X_GPT_OSS_MODELS = [
"triton",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_AITER": "0"
}, # Disabled due to SWA eviction bug with aiter (#17220)
env_vars={"SGLANG_USE_AITER": "1"},
),
]
test/registered/amd/perf/mi35x/test_deepseek_v32_basic_perf_mi35x.py
+1
View File
@@ -115,6 +115,7 @@ class TestNightlyDeepseekV32BasicPerformance(unittest.TestCase):
variant=self.variant_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False, # Disable profiling for AMD tests
timeout=5400, # Extended timeout for large model loading
)
results = result_tuple[0]
success = result_tuple[1]
test/registered/layers/mamba/test_mamba_ssm_ssd.py
+5
View File
@@ -5,6 +5,7 @@ register_amd_ci(est_time=30, suite="stage-b-test-small-1-gpu-amd")
# Adapted from https://github.com/vllm-project/vllm/blob/633f943e30a4444d890d26b81850f7217736f840/tests/kernels/mamba/test_mamba_ssm_ssd.py
import os
import pytest
import torch
@@ -13,8 +14,12 @@ from einops import rearrange, repeat
from sglang.srt.layers.attention.mamba.mamba2_metadata import Mamba2Metadata
from sglang.srt.layers.attention.mamba.ops import mamba_chunk_scan_combined
from sglang.srt.utils.common import is_hip
from sglang.utils import is_in_ci
if is_hip():
os.environ["AMDGCN_USE_BUFFER_OPS"] = "0"
# Added by the IBM Team, 2024
# Adapted from https://github.com/state-spaces/mamba/blob/v2.2.4/mamba_ssm/modules/ssd_minimal.py
test/registered/rl/test_update_weights_from_distributed.py
+82
View File
@@ -37,6 +37,7 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
)
@@ -64,6 +65,60 @@ def verify_params_not_close(params1, params2, error_msg):
assert not np.allclose(np.array(params1), np.array(params2)), error_msg
def _warmup_broadcast(
hf_base_model,
state_dict_key_to_shape,
tie_word_embeddings,
load_format,
group,
):
"""Run one broadcast round to warm up RCCL before timing."""
broadcast_parameters = list(state_dict_key_to_shape.keys())
if tie_word_embeddings:
broadcast_parameters.remove("lm_head.weight")
if load_format == "flattened_bucket":
named_tensors = [
(name, hf_base_model.get_parameter(name)) for name in broadcast_parameters
]
bucket = FlattenedTensorBucket(named_tensors=named_tensors)
flattened_tensor = bucket.get_flattened_tensor()
torch.distributed.broadcast(flattened_tensor, src=0, group=group)
else:
for name in broadcast_parameters:
torch.distributed.broadcast(
hf_base_model.get_parameter(name),
src=0,
group=group,
)
def _warmup_update(
backend, engine, url, names, dtypes, shapes, load_format, pause_generation_mode
):
"""Run one update round to warm up RCCL before timing."""
if backend == "Engine":
engine.update_weights_from_distributed(
names,
dtypes=dtypes,
shapes=shapes,
group_name="test_parameter_update_group",
load_format=load_format,
)
else:
requests.post(
f"{url}/update_weights_from_distributed",
json={
"names": names,
"dtypes": dtypes,
"shapes": shapes,
"group_name": "test_parameter_update_group",
"load_format": load_format,
"flush_cache": not (pause_generation_mode == "in_place"),
},
)
def init_process(
rank,
world_size,
@@ -180,6 +235,18 @@ def init_process_hf(
)
torch.cuda.synchronize()
barrier.wait()
# Warmup: trigger RCCL initialization so it's excluded from timing
if is_in_amd_ci():
_warmup_broadcast(
hf_base_model,
state_dict_key_to_shape,
tie_word_embeddings,
load_format,
group,
)
torch.cuda.synchronize()
time_begin_broadcast = time.perf_counter()
# The last parameter is lm_head.weight, which is tied
@@ -354,6 +421,21 @@ def init_process_sgl(
)
torch.cuda.synchronize()
barrier.wait()
# Warmup: trigger RCCL initialization so it's excluded from timing
if is_in_amd_ci():
_warmup_update(
backend,
engine if backend == "Engine" else None,
url if backend != "Engine" else None,
names,
dtypes,
shapes,
load_format,
pause_generation_mode,
)
torch.cuda.synchronize()
time_begin_update = time.perf_counter()
if backend == "Engine":
engine.update_weights_from_distributed(
test/run_suite.py
+4 -2
View File
@@ -120,8 +120,10 @@ def auto_partition(files: List[CIRegistry], rank, size):
if not files or size <= 0:
return []
# Sort files by estimated_time in descending order (LPT heuristic)
sorted_files = sorted(files, key=lambda f: f.est_time, reverse=True)
# Sort files by estimated_time in descending order (LPT heuristic).
# Use filename as tie-breaker to ensure deterministic partitioning
# regardless of glob ordering.
sorted_files = sorted(files, key=lambda f: (-f.est_time, f.filename))
partitions = [[] for _ in range(size)]
partition_sums = [0.0] * size