chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] Add MI35x nightly CI tests (#16588)

Browse Source 
Co-authored-by: michaelzhang-ai <michaelzhang-ai@users.noreply.github.com>
Co-authored-by: Bingxu Chen <bingxche@amd.com>
Co-authored-by: HAI <hixiao@gmail.com>
This commit is contained in:
Michael
2026-01-08 19:29:15 -08:00
committed by GitHub
co-authored by michaelzhang-ai Bingxu Chen HAI
parent 75da784d48
commit fcec35dc4a
8 changed files with 1273 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/amd/nightly/test_gsm8k_completion_eval_amd.py
+937
View File
@@ -0,0 +1,937 @@
"""
AMD GSM8K Completion Evaluation Test (Migrated from test/srt/nightly/)
This test uses the completion-based gsm8k benchmark (few-shot prompting)
which works with base models that don't have chat templates.
This complements test_gsm8k_eval_amd.py which uses mgsm_en (chat completions)
for instruction-tuned models.
Base models tested here:
- GPT-OSS series (lmsys/gpt-oss-20b-bf16, lmsys/gpt-oss-120b-bf16)
- GROK series (lmzheng/grok-1, amd/grok-1-W4A8KV8, xai-org/grok-2)
- DeepSeek series (deepseek-ai/DeepSeek-V3-0324, deepseek-ai/DeepSeek-R1-0528)
Model groups are selected via AMD_TEST_MODEL_GROUP environment variable:
- "gpt-oss" (default): GPT-OSS models only (nightly-amd-8-gpu-gpt-oss)
- "grok": All GROK models (nightly-amd-8-gpu-grok)
- "deepseek-v3-dp": DeepSeek-V3 with DP attention (nightly-amd-8-gpu-deepseek-v3-dp)
- "deepseek-v3-tc": DeepSeek-V3 with torch compile (nightly-amd-8-gpu-deepseek-v3-tc)
- "deepseek-v3-mtp": DeepSeek-V3 with MTP/EAGLE (nightly-amd-8-gpu-deepseek-v3-mtp)
- "deepseek-r1": DeepSeek-R1 reasoning model (nightly-amd-8-gpu-deepseek-r1)
- "all": All models
Registry: nightly-amd-8-gpu suite (8-GPU tests)
"""
import ast
import os
import re
import subprocess
import time
import unittest
from dataclasses import dataclass
from typing import List, Optional, Tuple
import numpy as np
# HuggingFace Hub for model cache checking and download progress
try:
from huggingface_hub import HfFileSystem, snapshot_download
from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
HF_HUB_AVAILABLE = True
except ImportError:
HF_HUB_AVAILABLE = False
print("[WARNING] huggingface_hub not available - model cache checking disabled")
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
from sglang.utils import download_and_cache_file, read_jsonl
# Register for AMD CI - GSM8K completion tests (~120 min)
register_amd_ci(est_time=7200, suite="nightly-amd-8-gpu", nightly=True)
INVALID = -9999999
@dataclass
class BaseModelConfig:
"""Configuration for a base model to test."""
model_path: str
tp_size: int = 8
accuracy_threshold: float = 0.50
other_args: Optional[List[str]] = None
env_vars: Optional[dict] = None
tokenizer_path: Optional[str] = None
timeout: Optional[int] = None # Custom timeout for server launch (seconds)
variant: Optional[str] = (
None # Test variant name (e.g., "basic", "MTP", "DP", "TC")
)
def __post_init__(self):
if self.other_args is None:
self.other_args = []
if self.env_vars is None:
self.env_vars = {}
def get_display_name(self) -> str:
"""Return display name for logs/summary (model + variant if set)."""
if self.variant:
return f"{self.model_path} ({self.variant})"
return self.model_path
# =============================================================================
# MODEL GROUPS - Each group runs on a separate 8-GPU runner
# =============================================================================
# Group 1: GPT-OSS models (cached on upstream CI)
# Runner: nightly-amd-8-gpu
AMD_GPT_OSS_MODELS = [
# GPT-OSS-20B - smaller model, run first for faster feedback
BaseModelConfig(
model_path="lmsys/gpt-oss-20b-bf16",
tp_size=8,
accuracy_threshold=0.47,
other_args=[
"--chunked-prefill-size",
"130172",
"--max-running-requests",
"128",
"--mem-fraction-static",
"0.85",
"--attention-backend",
"triton",
"--trust-remote-code",
],
env_vars={"SGLANG_USE_AITER": "0"},
),
# GPT-OSS-120B - large model, needs longer timeout
BaseModelConfig(
model_path="lmsys/gpt-oss-120b-bf16",
tp_size=8,
accuracy_threshold=0.79,
timeout=900, # 15 minutes for 120B model
other_args=[
"--chunked-prefill-size",
"130172",
"--max-running-requests",
"128",
"--mem-fraction-static",
"0.85",
"--attention-backend",
"triton",
"--trust-remote-code",
],
env_vars={"SGLANG_USE_AITER": "0"},
),
]
# Group 2: All GROK models
# Runner: nightly-amd-8-gpu-grok
# Order: GROK1-FP8 -> GROK1-IN4 -> GROK2.5
AMD_GROK_MODELS = [
# GROK1-FP8 - verified accuracy: 0.860, runtime: ~12.5min
BaseModelConfig(
model_path="lmzheng/grok-1",
tp_size=8,
accuracy_threshold=0.80,
timeout=3600, # 1 hour for kernel compilation
tokenizer_path="Xenova/grok-1-tokenizer",
other_args=[
"--quantization",
"fp8",
"--attention-backend",
"aiter",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"RCCL_MSCCL_ENABLE": "0",
"SGLANG_USE_AITER": "1",
"SGLANG_INT4_WEIGHT": "0",
},
),
# GROK1-IN4 - verified accuracy: 0.820, runtime: ~12.5min
BaseModelConfig(
model_path="amd/grok-1-W4A8KV8",
tp_size=8,
accuracy_threshold=0.80,
timeout=3600, # 1 hour for kernel compilation
tokenizer_path="Xenova/grok-1-tokenizer",
other_args=[
"--quantization",
"fp8",
"--attention-backend",
"aiter",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"RCCL_MSCCL_ENABLE": "0",
"SGLANG_USE_AITER": "1",
"SGLANG_INT4_WEIGHT": "1",
},
),
# GROK2.5 - verified accuracy: 0.945, runtime: ~14.5min
BaseModelConfig(
model_path="xai-org/grok-2",
tp_size=8,
accuracy_threshold=0.915,
timeout=3600, # 1 hour for download + kernel compilation
tokenizer_path="alvarobartt/grok-2-tokenizer",
other_args=[
"--quantization",
"fp8",
"--attention-backend",
"aiter",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"RCCL_MSCCL_ENABLE": "0",
"SGLANG_USE_AITER": "1",
"SGLANG_INT4_WEIGHT": "0",
},
),
]
# Note: DeepSeek-V3 accuracy tests removed - V3 only used for perf tests
# See test_deepseek_v3_perf.py and test_deepseek_v31_perf.py for V3 perf tests
# Group 3: DeepSeek-R1 (reasoning model) - Basic + MTP combined
# Runner: nightly-amd-8-gpu-deepseek-r1
AMD_DEEPSEEK_R1_MODELS = [
# DeepSeek-R1-0528 basic - reasoning model, ~80GB per GPU
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600, # 1 hour for large model
variant="basic",
other_args=[
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--disable-radix-cache",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_AITER": "1",
},
),
# DeepSeek-R1-0528 with MTP (EAGLE speculative decoding)
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
variant="MTP",
other_args=[
"--chunked-prefill-size",
"131072",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.7",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_AITER": "1",
},
),
]
# Group 5: DeepSeek-R1 with DP + TC combined
# Runner: nightly-amd-8-gpu-deepseek-r1-dp-tc
# Combines DP attention and Torch Compile tests for DeepSeek-R1
AMD_DEEPSEEK_R1_DP_TC_MODELS = [
# DeepSeek-R1-0528 with DP attention
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
variant="DP",
other_args=[
"--chunked-prefill-size",
"131072",
"--dp-size",
"8",
"--enable-dp-attention",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_ROCM700A": "1",
"SGLANG_USE_AITER": "1",
},
),
# DeepSeek-R1-0528 with torch compile
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=7200, # 2 hours for compilation
variant="TC",
other_args=[
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"8",
"--enable-torch-compile",
"--disable-cuda-graph",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_ROCM700A": "1",
"SGLANG_USE_AITER": "1",
},
),
]
def get_model_group() -> str:
"""Get the model group to test from environment variable."""
return os.environ.get("AMD_TEST_MODEL_GROUP", "gpt-oss")
def get_models_for_group(group: str) -> List[BaseModelConfig]:
"""Get the list of models for a given group.
Note: DeepSeek-V3 is only used for perf tests, not accuracy tests.
See test_deepseek_v3_perf.py and test_deepseek_v31_perf.py.
"""
if group == "gpt-oss":
return AMD_GPT_OSS_MODELS
elif group == "grok":
return AMD_GROK_MODELS
elif group == "deepseek-r1":
return AMD_DEEPSEEK_R1_MODELS
elif group == "deepseek-r1-dp-tc":
return AMD_DEEPSEEK_R1_DP_TC_MODELS
elif group == "deepseek-r1-all":
# All DeepSeek-R1 variants: basic, MTP, DP, TC
return AMD_DEEPSEEK_R1_MODELS + AMD_DEEPSEEK_R1_DP_TC_MODELS
elif group == "all":
return (
AMD_GPT_OSS_MODELS
+ AMD_GROK_MODELS
+ AMD_DEEPSEEK_R1_MODELS
+ AMD_DEEPSEEK_R1_DP_TC_MODELS
)
else:
print(f"[WARNING] Unknown model group '{group}', using 'gpt-oss'")
return AMD_GPT_OSS_MODELS
# =============================================================================
# MODEL CACHE AND DOWNLOAD UTILITIES
# =============================================================================
def check_local_cache(model_path: str) -> Tuple[bool, str]:
"""
Check if model is cached locally.
Returns:
Tuple of (is_cached, cache_path_or_message)
"""
# Check common HF cache locations
cache_dirs = [
os.path.expanduser("~/.cache/huggingface/hub"),
"/sgl-data/hf-cache/hub",
"/home/runner/sgl-data/hf-cache",
]
# Convert model_path to cache directory format (org--model)
cache_name = f"models--{model_path.replace('/', '--')}"
for cache_dir in cache_dirs:
cache_path = os.path.join(cache_dir, cache_name)
if os.path.exists(cache_path):
# Check if there are snapshots
snapshots_dir = os.path.join(cache_path, "snapshots")
if os.path.exists(snapshots_dir) and os.listdir(snapshots_dir):
return True, cache_path
return False, f"Not found in: {', '.join(cache_dirs)}"
def check_hf_repo_access(model_path: str) -> Tuple[bool, str]:
"""
Check if HuggingFace repository is accessible.
Returns:
Tuple of (is_accessible, message)
"""
if not HF_HUB_AVAILABLE:
return True, "huggingface_hub not available, skipping access check"
try:
fs = HfFileSystem()
# Try to list files in the repo
files = fs.ls(model_path, detail=False)
if files:
return True, f"Repository accessible ({len(files)} files)"
else:
return False, "Repository exists but is empty"
except GatedRepoError:
return False, "GATED REPO - requires authentication/approval"
except RepositoryNotFoundError:
return False, "REPO NOT FOUND on HuggingFace"
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "unauthorized" in error_msg.lower():
return False, f"AUTH ERROR - may need HF_TOKEN: {error_msg[:100]}"
elif "404" in error_msg:
return False, f"NOT FOUND: {error_msg[:100]}"
elif "timeout" in error_msg.lower() or "connection" in error_msg.lower():
return False, f"NETWORK ERROR: {error_msg[:100]}"
else:
return False, f"ERROR: {error_msg[:100]}"
def log_model_status(config: BaseModelConfig) -> Tuple[bool, str]:
"""
Log detailed model availability status.
Returns:
Tuple of (is_available, status_message)
"""
model_path = config.model_path
print(f"\n📦 Checking model: {model_path}")
print("-" * 50)
# Check local cache first
is_cached, cache_msg = check_local_cache(model_path)
if is_cached:
print(f" ✅ LOCAL CACHE: Found at {cache_msg}")
return True, f"Cached locally at {cache_msg}"
else:
print(f" ⚠️ LOCAL CACHE: {cache_msg}")
# Check HF repo access
is_accessible, access_msg = check_hf_repo_access(model_path)
if is_accessible:
print(f" ✅ HF ACCESS: {access_msg}")
print(f" 📥 Model will be downloaded from HuggingFace (this may take a while)")
return True, f"Will download from HF: {access_msg}"
else:
print(f" ❌ HF ACCESS: {access_msg}")
return False, access_msg
# Also check tokenizer if specified
if config.tokenizer_path:
tok_cached, tok_msg = check_local_cache(config.tokenizer_path)
if tok_cached:
print(f" ✅ TOKENIZER CACHE: Found at {tok_msg}")
else:
tok_accessible, tok_access_msg = check_hf_repo_access(config.tokenizer_path)
if tok_accessible:
print(f" ✅ TOKENIZER HF: {tok_access_msg}")
else:
print(f" ⚠️ TOKENIZER: {tok_access_msg}")
return is_accessible, access_msg
def download_model_with_progress(
model_path: str, timeout: int = 3600
) -> Tuple[bool, str]:
"""
Download model with progress logging.
Returns:
Tuple of (success, message)
"""
if not HF_HUB_AVAILABLE:
return True, "huggingface_hub not available, skipping pre-download"
print(f"\n📥 Pre-downloading model: {model_path}")
print(f" Timeout: {timeout}s ({timeout/60:.0f} minutes)")
print("-" * 50)
start_time = time.time()
try:
# Use snapshot_download which shows progress
local_dir = snapshot_download(
repo_id=model_path,
local_files_only=False,
resume_download=True,
)
elapsed = time.time() - start_time
print(f" ✅ Download complete in {elapsed:.1f}s")
print(f" 📁 Location: {local_dir}")
return True, f"Downloaded to {local_dir}"
except GatedRepoError:
return False, "GATED REPO - requires authentication/approval"
except RepositoryNotFoundError:
return False, "REPO NOT FOUND on HuggingFace"
except Exception as e:
error_msg = str(e)
elapsed = time.time() - start_time
if elapsed >= timeout:
return False, f"TIMEOUT after {elapsed:.0f}s: {error_msg[:100]}"
elif "timeout" in error_msg.lower() or "connection" in error_msg.lower():
return False, f"NETWORK ERROR after {elapsed:.0f}s: {error_msg[:100]}"
else:
return False, f"ERROR after {elapsed:.0f}s: {error_msg[:100]}"
# =============================================================================
# BENCHMARK UTILITIES
# =============================================================================
def get_one_example(lines, i, include_answer):
"""Format a single GSM8K example."""
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
"""Get k few-shot examples for prompting."""
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
"""Extract numerical answer from response."""
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def run_gsm8k_benchmark(
base_url: str,
num_questions: int = 200,
num_shots: int = 5,
parallel: int = 64,
) -> Tuple[float, float, float]:
"""
Run GSM8K few-shot completion benchmark.
Returns:
Tuple of (accuracy, invalid_rate, latency)
"""
import sglang as sgl
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
# Download and load data
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
data_path = download_and_cache_file(url)
lines = list(read_jsonl(data_path))
# Construct prompts
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
# Define sglang function
@sgl.function
def few_shot_gsm8k(s, question):
s += few_shot_examples + question
s += sgl.gen(
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
)
# Set backend
backend = RuntimeEndpoint(base_url)
sgl.set_default_backend(backend)
# Run benchmark
tic = time.perf_counter()
states = few_shot_gsm8k.run_batch(
arguments,
temperature=0,
num_threads=parallel,
progress_bar=True,
)
latency = time.perf_counter() - tic
# Extract predictions
preds = []
for i in range(len(states)):
preds.append(get_answer_value(states[i]["answer"]))
# Compute metrics
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
return float(acc), float(invalid), float(latency)
def popen_launch_server_for_base_model(
base_url: str,
config: BaseModelConfig,
) -> "subprocess.Popen":
"""Launch server for a base model with appropriate configuration."""
# Build environment - start with current env and add config-specific vars
env = os.environ.copy()
for key, value in config.env_vars.items():
env[key] = value
print(f"Setting env: {key}={value}")
# Build other_args
other_args = list(config.other_args)
other_args.extend(["--tp", str(config.tp_size)])
other_args.extend(["--log-level-http", "warning"])
if config.tokenizer_path:
other_args.extend(["--tokenizer-path", config.tokenizer_path])
# Use custom timeout if provided, otherwise use default
timeout = config.timeout if config.timeout else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
process = popen_launch_server(
model=config.model_path,
base_url=base_url,
timeout=timeout,
other_args=other_args,
env=env, # Pass environment explicitly
)
return process
class TestNightlyGsm8kCompletionEvalAMD(unittest.TestCase):
"""
AMD GSM8K Completion Evaluation Test
Tests base models using few-shot completion benchmark.
This is different from mgsm_en which uses chat completions.
Model group is selected via AMD_TEST_MODEL_GROUP env var:
- "gpt-oss": GPT-OSS models only (default, nightly-amd-8-gpu)
- "grok": All GROK models (nightly-amd-8-gpu-grok)
- "all": All models
"""
@classmethod
def setUpClass(cls):
# Get model group from environment
cls.model_group = get_model_group()
cls.models = get_models_for_group(cls.model_group)
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
print(f"\n{'='*60}")
print(f"AMD GSM8K Completion Evaluation Test")
print(f"{'='*60}")
print(f"Model group: {cls.model_group}")
print(f"Models to test: {len(cls.models)}")
for m in cls.models:
print(f" - {m.model_path}")
print(f"Questions per model: {cls.num_questions}")
print(f"{'='*60}\n")
def test_gsm8k_completion_all_models(self):
"""Test all configured base models with GSM8K completion benchmark."""
all_results = []
total_test_start = time.time()
# Summary table with runtime columns
summary = f"### Model Group: {self.model_group}\n\n"
summary += (
"| Model | TP | Accuracy | Threshold | Startup | Bench | Total | Status |\n"
)
summary += (
"| ----- | -- | -------- | --------- | ------- | ----- | ----- | ------ |\n"
)
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"Testing: {display_name} (TP={config.tp_size})")
print(f"{'='*60}")
error_message = None
acc, invalid, latency = None, None, None
startup_time, bench_time, total_time = None, None, None
skipped = False
model_start = time.time()
# Check model availability with detailed logging
is_available, status_msg = log_model_status(config)
if not is_available:
print(f"\n❌ MODEL NOT AVAILABLE: {status_msg}")
print(f"⏭️ SKIPPING: {display_name}")
status = f"⏭️ SKIP"
skipped = True
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": None,
"threshold": config.accuracy_threshold,
"invalid": None,
"latency": None,
"startup_time": None,
"bench_time": None,
"total_time": None,
"passed": True, # Don't count as failure
"skipped": True,
"error": status_msg,
}
)
else:
try:
# Launch server with timing
print(f"\n🚀 Launching server for {display_name}...")
server_start = time.time()
process = popen_launch_server_for_base_model(
self.base_url, config
)
startup_time = time.time() - server_start
print(f"⏱️ Server startup: {startup_time:.1f}s")
try:
# Run benchmark with timing and retries
print(
f"📊 Running GSM8K benchmark ({self.num_questions} questions)..."
)
bench_start = time.time()
acc, invalid, latency = None, None, None
for attempt in range(3):
try:
acc, invalid, latency = run_gsm8k_benchmark(
self.base_url,
num_questions=self.num_questions,
num_shots=5,
parallel=64,
)
print(
f" Attempt {attempt + 1}: accuracy={acc:.3f}"
)
if acc >= config.accuracy_threshold:
break
except Exception as e:
print(
f" Attempt {attempt + 1} failed with error: {e}"
)
if attempt == 2:
raise
bench_time = time.time() - bench_start
total_time = time.time() - model_start
print(f"\n📈 Results for {display_name}:")
print(
f" Accuracy: {acc:.3f} (threshold: {config.accuracy_threshold})"
)
print(f" Invalid: {invalid:.3f}")
print(f" Benchmark latency: {latency:.1f}s")
print(f"\n⏱️ Runtime breakdown:")
print(f" Server startup: {startup_time:.1f}s")
print(f" Benchmark: {bench_time:.1f}s")
print(f" Total: {total_time:.1f}s")
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
if passed:
print(f"\n Status: ✅ PASSED")
else:
print(f"\n Status: ❌ FAILED (below threshold)")
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": acc,
"threshold": config.accuracy_threshold,
"invalid": invalid,
"latency": latency,
"startup_time": startup_time,
"bench_time": bench_time,
"total_time": total_time,
"passed": passed,
"skipped": False,
"error": None,
}
)
except Exception as e:
error_message = str(e)
total_time = time.time() - model_start
print(f"\n❌ Error during benchmark: {error_message}")
status = "❌ ERROR"
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": None,
"threshold": config.accuracy_threshold,
"invalid": None,
"latency": None,
"startup_time": startup_time,
"bench_time": None,
"total_time": total_time,
"passed": False,
"skipped": False,
"error": error_message,
}
)
finally:
print(f"\n🛑 Stopping server for {display_name}...")
kill_process_tree(process.pid)
except Exception as e:
error_message = str(e)
total_time = time.time() - model_start
print(f"\n❌ Error launching server: {error_message}")
status = "❌ ERROR"
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": None,
"threshold": config.accuracy_threshold,
"invalid": None,
"latency": None,
"startup_time": None,
"bench_time": None,
"total_time": total_time,
"passed": False,
"skipped": False,
"error": error_message,
}
)
# Add to summary with runtime (use display name to show variant)
acc_str = f"{acc:.3f}" if acc is not None else "N/A"
startup_str = (
f"{startup_time:.0f}s" if startup_time is not None else "N/A"
)
bench_str = f"{bench_time:.0f}s" if bench_time is not None else "N/A"
total_str = f"{total_time:.0f}s" if total_time is not None else "N/A"
summary += f"| {display_name} | {config.tp_size} | {acc_str} | {config.accuracy_threshold} | {startup_str} | {bench_str} | {total_str} | {status} |\n"
# Calculate total test runtime
total_test_time = time.time() - total_test_start
# Print summary
print(f"\n{'='*60}")
print(f"SUMMARY - Model Group: {self.model_group}")
print(f"{'='*60}")
print(summary)
print(
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
)
# Check for failures (exclude skipped models)
failed_models = [
r for r in all_results if not r["passed"] and not r.get("skipped", False)
]
skipped_models = [r for r in all_results if r.get("skipped", False)]
passed_models = [
r for r in all_results if r["passed"] and not r.get("skipped", False)
]
# Build GitHub summary with results and failure details
# Note: summary already includes the "### Model Group:" header
github_summary = f"{summary}\n"
github_summary += f"\n**Statistics:** ✅ Passed: {len(passed_models)} | ❌ Failed: {len(failed_models)} | ⏭️ Skipped: {len(skipped_models)}\n"
github_summary += f"\n**Total Runtime:** {total_test_time:.1f}s ({total_test_time/60:.1f} min)\n"
if failed_models:
github_summary += "\n#### ❌ Failed Models\n"
for r in failed_models:
acc_str = f"{r['accuracy']:.3f}" if r["accuracy"] is not None else "N/A"
github_summary += f"- **{r['model']}**: accuracy={acc_str}, threshold={r['threshold']}"
if r.get("error"):
# Truncate long errors for display
error_short = (
r["error"][:200] + "..."
if len(r["error"]) > 200
else r["error"]
)
github_summary += f"\n - Error: `{error_short}`"
github_summary += "\n"
if skipped_models:
github_summary += "\n#### ⏭️ Skipped Models\n"
for r in skipped_models:
github_summary += (
f"- **{r['model']}**: {r.get('error', 'Not available')}\n"
)
# Write GitHub step summary
if is_in_ci():
write_github_step_summary(github_summary)
print(f"\n📊 Final Statistics:")
print(f" Passed: {len(passed_models)}")
print(f" Failed: {len(failed_models)}")
print(f" Skipped: {len(skipped_models)}")
if skipped_models:
print(f"\n⏭️ Skipped models (not available):")
for r in skipped_models:
print(f" - {r['model']}: {r['error']}")
if failed_models:
print(f"\n❌ Failed models:")
for r in failed_models:
acc_str = f"{r['accuracy']:.3f}" if r["accuracy"] is not None else "N/A"
print(
f" - {r['model']}: accuracy={acc_str}, threshold={r['threshold']}"
)
if r.get("error"):
print(f" Error: {r['error'][:200]}")
failure_msg = "\n".join(
[
f"- {r['model']}: accuracy={r['accuracy']}, threshold={r['threshold']}, error={r['error']}"
for r in failed_models
]
)
raise AssertionError(f"The following models failed:\n{failure_msg}")
if __name__ == "__main__":
unittest.main()
test/registered/amd/nightly/test_gsm8k_completion_eval_mi35x.py
+726
View File
@@ -0,0 +1,726 @@
"""
MI35x GSM8K Completion Evaluation Test (8-GPU)
This test uses the completion-based gsm8k benchmark (few-shot prompting)
for MI35x-specific models that differ from MI300X configurations.
MI35x-specific models:
- GPT-OSS series: Uses openai/gpt-oss-* (not lmsys/gpt-oss-*-bf16)
- DeepSeek-R1-0528: Same model as MI300X (MXFP4 only used for perf tests)
Model groups are selected via AMD_TEST_MODEL_GROUP environment variable:
- "gpt-oss" (default): GPT-OSS models with MI35x paths
- "deepseek-r1": DeepSeek-R1-0528 basic + MTP (same as MI300X)
- "deepseek-r1-dp-tc": DeepSeek-R1-0528 DP + TC (same as MI300X)
- "deepseek-r1-all": All DeepSeek-R1-0528 variants (basic, MTP, DP, TC)
Registry: nightly-amd-8-gpu-mi35x suite (8-GPU tests on MI35x)
"""
import ast
import os
# Set HF cache to /data2/models/ for MI35x so HF models download there
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
import re
import subprocess
import time
import unittest
from dataclasses import dataclass
from typing import List, Optional, Tuple
import numpy as np
# HuggingFace Hub for model cache checking and download progress
try:
from huggingface_hub import HfFileSystem
from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
HF_HUB_AVAILABLE = True
except ImportError:
HF_HUB_AVAILABLE = False
print("[WARNING] huggingface_hub not available - model cache checking disabled")
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
from sglang.utils import download_and_cache_file, read_jsonl
# Register for AMD CI - MI35x 8-GPU GSM8K completion tests (~120 min)
register_amd_ci(est_time=7200, suite="nightly-amd-8-gpu-mi35x", nightly=True)
INVALID = -9999999
@dataclass
class BaseModelConfig:
"""Configuration for a base model to test."""
model_path: str # HuggingFace model ID (e.g., "amd/DeepSeek-R1-MXFP4-Preview")
tp_size: int = 8
accuracy_threshold: float = 0.50
other_args: Optional[List[str]] = None
env_vars: Optional[dict] = None
tokenizer_path: Optional[str] = None
timeout: Optional[int] = None
local_path: Optional[str] = None # Preferred local path (checked first before HF)
variant: Optional[str] = (
None # Test variant name (e.g., "basic", "MTP", "DP", "TC")
)
def __post_init__(self):
if self.other_args is None:
self.other_args = []
if self.env_vars is None:
self.env_vars = {}
def get_effective_model_path(self) -> str:
"""Return local_path if it exists, otherwise model_path (HF ID)."""
if self.local_path and os.path.exists(self.local_path):
return self.local_path
return self.model_path
def get_display_name(self) -> str:
"""Return display name for logs/summary (model + variant if set)."""
if self.variant:
return f"{self.model_path} ({self.variant})"
return self.model_path
# =============================================================================
# MI35x MODEL GROUPS - Different from MI300X configurations
# =============================================================================
# Group 1: GPT-OSS models (MI35x uses openai/* paths, not lmsys/*)
MI35X_GPT_OSS_MODELS = [
# GPT-OSS-20B - MI35x specific path
BaseModelConfig(
model_path="openai/gpt-oss-20b",
tp_size=8,
accuracy_threshold=0.47,
other_args=[
"--chunked-prefill-size",
"130172",
"--max-running-requests",
"128",
"--mem-fraction-static",
"0.85",
"--attention-backend",
"triton",
"--trust-remote-code",
],
env_vars={"SGLANG_USE_AITER": "1"},
),
# GPT-OSS-120B - MI35x specific path
BaseModelConfig(
model_path="openai/gpt-oss-120b",
tp_size=8,
accuracy_threshold=0.79,
timeout=900, # 15 minutes for 120B model
other_args=[
"--chunked-prefill-size",
"130172",
"--max-running-requests",
"128",
"--mem-fraction-static",
"0.85",
"--attention-backend",
"triton",
"--trust-remote-code",
],
env_vars={"SGLANG_USE_AITER": "1"},
),
]
# Group 2: DeepSeek-R1-0528 basic + MTP (same model as MI300X for consistency)
# Runner: nightly-test-8-gpu-mi35x-deepseek-r1
# Note: MXFP4 variant only used for perf tests (test_deepseek_r1_mxfp4_perf.py)
MI35X_DEEPSEEK_R1_MODELS = [
# DeepSeek-R1-0528 basic - reasoning model, ~80GB per GPU
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600, # 1 hour for large model
variant="basic",
other_args=[
"--attention-backend",
"aiter",
"--chunked-prefill-size",
"131072",
"--disable-radix-cache",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_AITER": "1",
},
),
# DeepSeek-R1-0528 with MTP (EAGLE speculative decoding)
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
variant="MTP",
other_args=[
"--chunked-prefill-size",
"131072",
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--mem-fraction-static",
"0.7",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_AITER": "1",
},
),
]
# Group 3: DeepSeek-R1-0528 with DP + TC (requires ROCm 7.0+)
# Runner: nightly-test-8-gpu-mi35x-deepseek-r1-dp-tc
MI35X_DEEPSEEK_R1_DP_TC_MODELS = [
# DeepSeek-R1-0528 with DP attention
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
variant="DP",
other_args=[
"--chunked-prefill-size",
"131072",
"--dp-size",
"8",
"--enable-dp-attention",
"--mem-fraction-static",
"0.85",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_ROCM700A": "1",
"SGLANG_USE_AITER": "1",
},
),
# DeepSeek-R1-0528 with torch compile
BaseModelConfig(
model_path="deepseek-ai/DeepSeek-R1-0528",
tp_size=8,
accuracy_threshold=0.93,
timeout=7200, # 2 hours for compilation
variant="TC",
other_args=[
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.70",
"--cuda-graph-max-bs",
"8",
"--enable-torch-compile",
"--disable-cuda-graph",
"--trust-remote-code",
],
env_vars={
"SGLANG_USE_ROCM700A": "1",
"SGLANG_USE_AITER": "1",
},
),
]
def get_model_group() -> str:
"""Get the model group to test from environment variable."""
return os.environ.get("AMD_TEST_MODEL_GROUP", "gpt-oss")
def get_models_for_group(group: str) -> List[BaseModelConfig]:
"""Get the list of models for a given group.
Note: DeepSeek-R1-MXFP4 is only used for perf tests, not accuracy tests.
See test_deepseek_r1_mxfp4_perf.py for MXFP4 perf tests.
"""
if group == "gpt-oss":
return MI35X_GPT_OSS_MODELS
elif group == "deepseek-r1":
return MI35X_DEEPSEEK_R1_MODELS
elif group == "deepseek-r1-dp-tc":
return MI35X_DEEPSEEK_R1_DP_TC_MODELS
elif group == "deepseek-r1-all":
# All DeepSeek-R1-0528 variants: basic, MTP, DP, TC
return MI35X_DEEPSEEK_R1_MODELS + MI35X_DEEPSEEK_R1_DP_TC_MODELS
elif group == "all":
return (
MI35X_GPT_OSS_MODELS
+ MI35X_DEEPSEEK_R1_MODELS
+ MI35X_DEEPSEEK_R1_DP_TC_MODELS
)
else:
print(f"[WARNING] Unknown model group '{group}', using 'gpt-oss'")
return MI35X_GPT_OSS_MODELS
# =============================================================================
# MODEL CACHE AND DOWNLOAD UTILITIES
# =============================================================================
def check_local_cache(model_path: str) -> Tuple[bool, str]:
"""
Check if model is cached locally.
Returns:
Tuple of (is_cached, cache_path_or_message)
"""
# Check common HF cache locations for MI35x
cache_dirs = [
os.path.expanduser("~/.cache/huggingface/hub"),
"/data2/models/huggingface/hub",
os.environ.get("HF_HUB_CACHE", ""),
]
cache_dirs = [d for d in cache_dirs if d] # Remove empty
# Convert model_path to cache directory format (org--model)
cache_name = f"models--{model_path.replace('/', '--')}"
for cache_dir in cache_dirs:
cache_path = os.path.join(cache_dir, cache_name)
if os.path.exists(cache_path):
# Check if there are snapshots
snapshots_dir = os.path.join(cache_path, "snapshots")
if os.path.exists(snapshots_dir) and os.listdir(snapshots_dir):
return True, cache_path
return False, f"Not found in: {', '.join(cache_dirs)}"
def check_hf_repo_access(model_path: str) -> Tuple[bool, str]:
"""
Check if HuggingFace repository is accessible.
Returns:
Tuple of (is_accessible, message)
"""
if not HF_HUB_AVAILABLE:
return True, "huggingface_hub not available, skipping access check"
try:
fs = HfFileSystem()
# Try to list files in the repo
files = fs.ls(model_path, detail=False)
if files:
return True, f"Repository accessible ({len(files)} files)"
else:
return False, "Repository exists but is empty"
except GatedRepoError:
return False, "GATED REPO - requires authentication/approval"
except RepositoryNotFoundError:
return False, "REPO NOT FOUND on HuggingFace"
except Exception as e:
error_msg = str(e)
if "401" in error_msg or "unauthorized" in error_msg.lower():
return False, f"AUTH ERROR - may need HF_TOKEN: {error_msg[:100]}"
elif "404" in error_msg:
return False, f"NOT FOUND: {error_msg[:100]}"
elif "timeout" in error_msg.lower() or "connection" in error_msg.lower():
return False, f"NETWORK ERROR: {error_msg[:100]}"
else:
return False, f"ERROR: {error_msg[:100]}"
def log_model_status(config: "BaseModelConfig") -> Tuple[bool, str]:
"""
Log detailed model availability status.
Checks in order:
1. local_path (if specified) - preferred local path
2. model_path as local path (if starts with /)
3. model_path as HF model ID - check cache then HF access
Returns:
Tuple of (is_available, status_message)
"""
model_path = config.model_path
local_path = config.local_path
print(f"\n📦 Checking model: {model_path}")
if local_path:
print(f" (preferred local: {local_path})")
print("-" * 50)
# Step 1: Check preferred local_path first (if specified)
if local_path:
if os.path.exists(local_path):
print(f" ✅ LOCAL PATH: Found at {local_path}")
return True, f"Local path exists at {local_path}"
else:
print(f" ⚠️ LOCAL PATH: Not found at {local_path}, trying HF fallback...")
# Step 2: For absolute paths (starting with /), check if exists
if model_path.startswith("/"):
if os.path.exists(model_path):
print(f" ✅ LOCAL PATH: Found at {model_path}")
return True, f"Local path exists at {model_path}"
else:
print(f" ❌ LOCAL PATH: Not found at {model_path}")
return False, f"Local path not found at {model_path}"
# Step 3: For HF model IDs, check local cache first
is_cached, cache_msg = check_local_cache(model_path)
if is_cached:
print(f" ✅ LOCAL CACHE: Found at {cache_msg}")
return True, f"Cached locally at {cache_msg}"
else:
print(f" ⚠️ LOCAL CACHE: {cache_msg}")
# Step 4: Check HF repo access (will download if accessible)
is_accessible, access_msg = check_hf_repo_access(model_path)
if is_accessible:
print(f" ✅ HF ACCESS: {access_msg}")
print(
f" 📥 Model will be downloaded from HuggingFace to {os.environ.get('HF_HOME', '~/.cache/huggingface')}"
)
return True, f"Will download from HF: {access_msg}"
else:
print(f" ❌ HF ACCESS: {access_msg}")
return False, access_msg
# =============================================================================
# BENCHMARK UTILITIES
# =============================================================================
def get_one_example(lines, i, include_answer):
"""Format a single GSM8K example."""
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
"""Get k few-shot examples for prompting."""
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
"""Extract numerical answer from response."""
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def run_gsm8k_benchmark(
base_url: str,
num_questions: int = 200,
num_shots: int = 5,
parallel: int = 64,
) -> Tuple[float, float, float]:
"""Run GSM8K few-shot completion benchmark."""
import sglang as sgl
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
data_path = download_and_cache_file(url)
lines = list(read_jsonl(data_path))
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
@sgl.function
def few_shot_gsm8k(s, question):
s += few_shot_examples + question
s += sgl.gen(
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
)
backend = RuntimeEndpoint(base_url)
sgl.set_default_backend(backend)
tic = time.perf_counter()
states = few_shot_gsm8k.run_batch(
arguments,
temperature=0,
num_threads=parallel,
progress_bar=True,
)
latency = time.perf_counter() - tic
preds = []
for i in range(len(states)):
preds.append(get_answer_value(states[i]["answer"]))
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
return float(acc), float(invalid), float(latency)
def popen_launch_server_for_base_model(
base_url: str,
config: BaseModelConfig,
) -> "subprocess.Popen":
"""Launch server for a base model with appropriate configuration."""
env = os.environ.copy()
for key, value in config.env_vars.items():
env[key] = value
print(f"Setting env: {key}={value}")
other_args = list(config.other_args)
other_args.extend(["--tp", str(config.tp_size)])
other_args.extend(["--log-level-http", "warning"])
if config.tokenizer_path:
other_args.extend(["--tokenizer-path", config.tokenizer_path])
timeout = config.timeout if config.timeout else DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
# Use effective model path (local if exists, else HF model ID)
effective_model_path = config.get_effective_model_path()
print(f"Using model path: {effective_model_path}")
process = popen_launch_server(
model=effective_model_path,
base_url=base_url,
timeout=timeout,
other_args=other_args,
env=env,
)
return process
class TestMI35xGsm8kCompletionEval(unittest.TestCase):
"""MI35x GSM8K Completion Evaluation Test (8-GPU)
Tests MI35x-specific base models using few-shot completion benchmark.
"""
@classmethod
def setUpClass(cls):
cls.model_group = get_model_group()
cls.models = get_models_for_group(cls.model_group)
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
print(f"\n{'='*60}")
print(f"MI35x GSM8K Completion Evaluation Test (8-GPU)")
print(f"{'='*60}")
print(f"Model group: {cls.model_group}")
print(f"Models to test: {len(cls.models)}")
for m in cls.models:
print(f" - {m.model_path}")
print(f"Questions per model: {cls.num_questions}")
print(f"{'='*60}\n")
def test_gsm8k_completion_all_models(self):
"""Test all configured MI35x models with GSM8K completion benchmark."""
all_results = []
total_test_start = time.time()
summary = f"### MI35x Model Group: {self.model_group}\n\n"
summary += (
"| Model | TP | Accuracy | Threshold | Startup | Bench | Total | Status |\n"
)
summary += (
"| ----- | -- | -------- | --------- | ------- | ----- | ----- | ------ |\n"
)
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"Testing: {display_name} (TP={config.tp_size})")
print(f"{'='*60}")
error_message = None
acc, invalid, latency = None, None, None
startup_time, bench_time, total_time = None, None, None
model_start = time.time()
# Check model availability with detailed logging
is_available, status_msg = log_model_status(config)
if not is_available:
print(f"\n❌ MODEL NOT AVAILABLE: {status_msg}")
print(f"⏭️ SKIPPING: {display_name}")
status = "⏭️ SKIP"
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": None,
"threshold": config.accuracy_threshold,
"passed": True,
"skipped": True,
"error": status_msg,
}
)
else:
try:
print(f"\n🚀 Launching server for {display_name}...")
server_start = time.time()
process = popen_launch_server_for_base_model(
self.base_url, config
)
startup_time = time.time() - server_start
print(f"⏱️ Server startup: {startup_time:.1f}s")
try:
print(
f"📊 Running GSM8K benchmark ({self.num_questions} questions)..."
)
bench_start = time.time()
for attempt in range(3):
try:
acc, invalid, latency = run_gsm8k_benchmark(
self.base_url,
num_questions=self.num_questions,
num_shots=5,
parallel=64,
)
print(
f" Attempt {attempt + 1}: accuracy={acc:.3f}"
)
if acc >= config.accuracy_threshold:
break
except Exception as e:
print(f" Attempt {attempt + 1} failed: {e}")
if attempt == 2:
raise
bench_time = time.time() - bench_start
total_time = time.time() - model_start
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f"\n📈 Results: accuracy={acc:.3f} (threshold: {config.accuracy_threshold})"
)
print(f"⏱️ Total: {total_time:.1f}s")
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": acc,
"threshold": config.accuracy_threshold,
"startup_time": startup_time,
"bench_time": bench_time,
"total_time": total_time,
"passed": passed,
"skipped": False,
"error": None,
}
)
except Exception as e:
error_message = str(e)
total_time = time.time() - model_start
print(f"\n❌ Error: {error_message}")
status = "❌ ERROR"
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": None,
"threshold": config.accuracy_threshold,
"passed": False,
"skipped": False,
"error": error_message,
}
)
finally:
print(f"\n🛑 Stopping server...")
kill_process_tree(process.pid)
except Exception as e:
error_message = str(e)
total_time = time.time() - model_start
print(f"\n❌ Error launching server: {error_message}")
status = "❌ ERROR"
all_results.append(
{
"model": display_name,
"tp_size": config.tp_size,
"accuracy": None,
"threshold": config.accuracy_threshold,
"passed": False,
"skipped": False,
"error": error_message,
}
)
# Add to summary (use display name to show variant)
acc_str = f"{acc:.3f}" if acc is not None else "N/A"
startup_str = (
f"{startup_time:.0f}s" if startup_time is not None else "N/A"
)
bench_str = f"{bench_time:.0f}s" if bench_time is not None else "N/A"
total_str = f"{total_time:.0f}s" if total_time is not None else "N/A"
summary += f"| {display_name} | {config.tp_size} | {acc_str} | {config.accuracy_threshold} | {startup_str} | {bench_str} | {total_str} | {status} |\n"
# Final summary
total_test_time = time.time() - total_test_start
failed_models = [
r for r in all_results if not r["passed"] and not r.get("skipped", False)
]
skipped_models = [r for r in all_results if r.get("skipped", False)]
passed_models = [
r for r in all_results if r["passed"] and not r.get("skipped", False)
]
print(f"\n{'='*60}")
print(f"SUMMARY - MI35x Model Group: {self.model_group}")
print(f"{'='*60}")
print(summary)
print(
f"\n📊 Passed: {len(passed_models)} | Failed: {len(failed_models)} | Skipped: {len(skipped_models)}"
)
print(f"⏱️ Total: {total_test_time:.1f}s ({total_test_time/60:.1f} min)")
if is_in_ci():
write_github_step_summary(summary)
if failed_models:
failure_msg = "\n".join(
[
f"- {r['model']}: {r.get('error', 'below threshold')}"
for r in failed_models
]
)
raise AssertionError(f"The following models failed:\n{failure_msg}")
if __name__ == "__main__":
unittest.main()
test/registered/amd/nightly/test_gsm8k_eval_amd.py
+338
View File
@@ -0,0 +1,338 @@
"""
AMD GSM8K Evaluation Test (Migrated from test/srt/nightly/)
This test evaluates instruction-tuned models on the mgsm_en benchmark using chat completions.
Models are tested with various TP configurations on AMD GPUs.
Registry: nightly-amd suite (2-GPU tests)
"""
import json
import os
import time
import unittest
import warnings
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
parse_models,
popen_launch_server,
write_github_step_summary,
write_results_to_json,
)
# Register for AMD CI - GSM8K evaluation tests (~60 min)
register_amd_ci(est_time=3600, suite="nightly-amd", nightly=True)
MODEL_SCORE_THRESHOLDS = {
# Llama 3.1 series
"meta-llama/Llama-3.1-8B-Instruct": 0.82,
"meta-llama/Llama-3.1-70B-Instruct": 0.95,
# Llama 3.2 series (smaller models)
"meta-llama/Llama-3.2-3B-Instruct": 0.55,
# Mistral series
"mistralai/Mistral-7B-Instruct-v0.3": 0.58,
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.61,
# DeepSeek series
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
# Qwen2 series
"Qwen/Qwen2-57B-A14B-Instruct": 0.86,
"Qwen/Qwen2.5-7B-Instruct": 0.85,
# Qwen3 series
"Qwen/Qwen3-30B-A3B-Thinking-2507": 0.84, # MoE model verified on MI300X
"Qwen/Qwen3-8B": 0.77,
# Google Gemma
"google/gemma-2-27b-it": 0.91,
"google/gemma-2-9b-it": 0.72,
# "neuralmagic/gemma-2-2b-it-FP8": 0.4, # Small 2B model - OOM on single GPU
# FP8 quantized models
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.8,
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.86,
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.62,
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84,
}
failing_models = {
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8", # RuntimeError: This GEMM is not supported!
"zai-org/GLM-4.5-Air-FP8", # TypeError: cannot unpack non-iterable ForwardMetadata object
"google/gemma-2-9b-it", # OOM on single GPU (exit code -9)
"neuralmagic/gemma-2-2b-it-FP8", # OOM on single GPU (exit code -9)
}
def remove_failing_models(model_str):
models = model_str.split(",")
filtered = [m for m in models if m not in failing_models]
return ",".join(filtered)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = remove_failing_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = remove_failing_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2
)
# AMD-specific models verified on MI300X
# TP1 models - smaller models that fit on single GPU
AMD_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models(
"meta-llama/Llama-3.2-3B-Instruct,Qwen/Qwen2.5-7B-Instruct,Qwen/Qwen3-8B,google/gemma-2-9b-it"
)
# TP2 models - larger models requiring 2 GPUs
AMD_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models(
"Qwen/Qwen3-30B-A3B-Thinking-2507"
)
NO_MOE_PADDING_MODELS = {"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"}
DISABLE_HF_XET_MODELS = {
"Qwen/Qwen2-57B-A14B-Instruct",
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8",
}
TRITON_MOE_MODELS = {
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8",
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8",
"mistralai/Mixtral-8x7B-Instruct-v0.1",
"mistralai/Mistral-7B-Instruct-v0.3",
}
# AMD-specific models that need special launch config (matching in-house CI sanity_check.py)
# AMD_SPECIAL_CONFIG_MODELS = {
# "Qwen/Qwen3-30B-A3B-Thinking-2507", # default config works
# }
def popen_launch_server_wrapper(base_url, model, is_tp2):
other_args = ["--log-level-http", "warning", "--trust-remote-code"]
if is_tp2:
other_args.extend(["--tp", "2"])
# Use same config as sanity_check.py for AMD-specific models (scaled for tp=2)
# Original tp=8: chunked-prefill-size=130172, max-running-requests=128
# Scaled tp=2: chunked-prefill-size=32543, max-running-requests=32
# if model in AMD_SPECIAL_CONFIG_MODELS:
# other_args.extend([
# "--chunked-prefill-size", "32543",
# "--max-running-requests", "32",
# "--mem-fraction-static", "0.85",
# "--attention-backend", "aiter",
# ])
process = popen_launch_server(
model,
base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
return process
def check_model_scores(results):
"""Check model scores and generate summary table with pass/fail status."""
failed_models = []
passed_count = 0
failed_count = 0
summary = "| Model | TP | Score | Threshold | Startup | Eval | Total | Status |\n"
summary += "| ----- | -- | ----- | --------- | ------- | ---- | ----- | ------ |\n"
for result in results:
model = result["model"]
score = result["score"]
tp_size = result.get("tp_size", 2)
startup_time = result.get("startup_time")
eval_time = result.get("eval_time")
total_time = result.get("total_time")
threshold = MODEL_SCORE_THRESHOLDS.get(model)
if threshold is None:
print(f"Warning: No threshold defined for model {model}")
status = "⚠️ NO THRESHOLD"
elif score >= threshold:
status = "✅ PASS"
passed_count += 1
else:
status = "❌ FAIL"
failed_count += 1
failed_models.append(
f"- {model}: score={score:.4f}, threshold={threshold:.4f}"
)
# Format times
startup_str = f"{startup_time:.0f}s" if startup_time is not None else "N/A"
eval_str = f"{eval_time:.0f}s" if eval_time is not None else "N/A"
total_str = f"{total_time:.0f}s" if total_time is not None else "N/A"
threshold_str = f"{threshold:.2f}" if threshold is not None else "N/A"
line = f"| {model} | {tp_size} | {score:.3f} | {threshold_str} | {startup_str} | {eval_str} | {total_str} | {status} |\n"
summary += line
print(f"\n{'='*60}")
print("SUMMARY - TP=2 Instruction Models (mgsm_en)")
print(f"{'='*60}")
print(summary)
print(f"\n📊 Final Statistics:")
print(f" Passed: {passed_count}")
print(f" Failed: {failed_count}")
if is_in_ci():
write_github_step_summary(f"### TestNightlyGsm8KEval (TP=2)\n{summary}")
if failed_models:
failure_msg = "\n".join(failed_models)
raise AssertionError(f"The following models failed:\n{failure_msg}")
# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry
class TestNightlyGsm8KEval(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model_groups = [
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
# AMD-specific models verified on MI300X
(parse_models(AMD_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
(parse_models(AMD_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
]
cls.base_url = DEFAULT_URL_FOR_TEST
def test_mgsm_en_all_models(self):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
is_first = True
all_results = []
total_test_start = time.time()
print(f"\n{'='*60}")
print("AMD GSM8K Evaluation Test (TP=2 Instruction Models)")
print(f"{'='*60}")
print(f"Benchmark: mgsm_en (chat completions)")
print(f"{'='*60}\n")
for model_group, is_fp8, is_tp2 in self.model_groups:
for model in model_group:
with self.subTest(model=model):
tp_size = 2 if is_tp2 else 1
print(f"\n{'='*60}")
print(f"Testing: {model} (TP={tp_size}, FP8={is_fp8})")
print(f"{'='*60}")
model_start = time.time()
startup_time = None
eval_time = None
os.environ["SGLANG_MOE_PADDING"] = (
"0" if model in NO_MOE_PADDING_MODELS else "1"
)
os.environ["HF_HUB_DISABLE_XET"] = (
"1" if model in DISABLE_HF_XET_MODELS else "0"
)
os.environ["SGLANG_USE_AITER"] = (
"0" if model in TRITON_MOE_MODELS else "1"
)
# Launch server with timing
print(f"🚀 Launching server...")
server_start = time.time()
process = popen_launch_server_wrapper(self.base_url, model, is_tp2)
startup_time = time.time() - server_start
print(f"⏱️ Server startup: {startup_time:.1f}s")
args = SimpleNamespace(
base_url=self.base_url,
model=model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
# Run eval with timing and retries
print(f"📊 Running mgsm_en evaluation...")
eval_start = time.time()
threshold = MODEL_SCORE_THRESHOLDS.get(model)
metrics = None
for attempt in range(3):
try:
metrics = run_eval(args)
score = metrics["score"]
if threshold and score >= threshold:
break
except Exception as e:
print(f" Attempt {attempt + 1} failed with error: {e}")
eval_time = time.time() - eval_start
total_time = time.time() - model_start
# Print results
score = metrics["score"] if metrics else 0.0
threshold_str = f"{threshold:.2f}" if threshold else "N/A"
passed = threshold and score >= threshold
print(f"\n📈 Results for {model}:")
print(f" Score: {score:.3f} (threshold: {threshold_str})")
print(f"\n⏱️ Runtime breakdown:")
print(f" Server startup: {startup_time:.1f}s")
print(f" Evaluation: {eval_time:.1f}s")
print(f" Total: {total_time:.1f}s")
if passed:
print(f"\n Status: ✅ PASSED")
else:
print(f"\n Status: ❌ FAILED")
write_results_to_json(model, metrics, "w" if is_first else "a")
is_first = False
all_results.append(
{
"model": model,
"score": score,
"tp_size": tp_size,
"is_fp8": is_fp8,
"startup_time": startup_time,
"eval_time": eval_time,
"total_time": total_time,
}
)
print(f"\n🛑 Stopping server...")
kill_process_tree(process.pid)
# Calculate total test runtime
total_test_time = time.time() - total_test_start
try:
with open("results.json", "r") as f:
print("\nFinal Results from results.json:")
print(json.dumps(json.load(f), indent=2))
except Exception as e:
print(f"Error reading results.json: {e}")
# Check all scores after collecting all results
check_model_scores(all_results)
print(
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
)
if __name__ == "__main__":
unittest.main()
test/registered/amd/nightly/test_vlms_mmmu_eval_amd.py
+323
View File
@@ -0,0 +1,323 @@
"""
AMD VLM MMMU Evaluation Test (Migrated from test/srt/nightly/)
This test evaluates Vision-Language Models (VLMs) on the MMMU benchmark on AMD GPUs.
Models are selected based on compatibility with AMD/ROCm platform.
VLMs tested here:
- Qwen2-VL series (Qwen2-VL-7B, Qwen2.5-VL-7B)
- InternVL2 series
- MiniCPM-v series
- deepseek-vl2-small
Note: Some VLMs from the Nvidia test are excluded due to AMD compatibility issues.
Registry: nightly-amd-vlm suite (2-GPU VLM tests)
"""
import os
import time
import unittest
import warnings
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
write_results_to_json,
)
# Register for AMD CI - VLM MMMU evaluation tests (~120 min)
register_amd_ci(est_time=7200, suite="nightly-amd-vlm", nightly=True)
# AMD-verified VLM models with conservative thresholds on 100 MMMU samples
# Format: (model_path, tp_size, accuracy_threshold, extra_args)
AMD_VLM_MODELS = [
# Qwen2-VL series - well supported on AMD
{
"model_path": "Qwen/Qwen2-VL-7B-Instruct",
"tp_size": 1,
"accuracy_threshold": 0.30,
"extra_args": ["--trust-remote-code"],
},
{
"model_path": "Qwen/Qwen2.5-VL-7B-Instruct",
"tp_size": 1,
"accuracy_threshold": 0.33,
"extra_args": ["--trust-remote-code"],
},
# InternVL2 - smaller model, good for testing
{
"model_path": "OpenGVLab/InternVL2_5-2B",
"tp_size": 1,
"accuracy_threshold": 0.29,
"extra_args": ["--trust-remote-code"],
},
# MiniCPM-v - lightweight VLM
{
"model_path": "openbmb/MiniCPM-v-2_6",
"tp_size": 1,
"accuracy_threshold": 0.25,
"extra_args": ["--trust-remote-code"],
},
# DeepSeek VL2 small - MoE VLM
{
"model_path": "deepseek-ai/deepseek-vl2-small",
"tp_size": 1,
"accuracy_threshold": 0.31,
"extra_args": ["--trust-remote-code"],
},
]
# Models that need special handling on AMD
TRITON_ATTENTION_MODELS = {
"deepseek-ai/deepseek-vl2-small", # MoE model
}
# Models known to fail on AMD - exclude from testing
AMD_FAILING_VLM_MODELS = {
# Add models here as they are discovered to fail
}
def get_active_models():
"""Get list of models to test, excluding known failures."""
return [m for m in AMD_VLM_MODELS if m["model_path"] not in AMD_FAILING_VLM_MODELS]
class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
"""AMD VLM MMMU Evaluation Test.
Tests Vision-Language Models on MMMU benchmark using AMD GPUs.
"""
@classmethod
def setUpClass(cls):
cls.models = get_active_models()
cls.base_url = DEFAULT_URL_FOR_TEST
def test_mmmu_vlm_models(self):
"""Test all configured VLM models on MMMU benchmark."""
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
is_first = True
all_results = []
total_test_start = time.time()
print(f"\n{'='*60}")
print("AMD VLM MMMU Evaluation Test")
print(f"{'='*60}")
print(f"Benchmark: MMMU (100 samples)")
print(f"Models to test: {len(self.models)}")
for m in self.models:
print(f" - {m['model_path']} (TP={m['tp_size']})")
print(f"{'='*60}\n")
for model_config in self.models:
model_path = model_config["model_path"]
tp_size = model_config["tp_size"]
accuracy_threshold = model_config["accuracy_threshold"]
extra_args = model_config.get("extra_args", [])
error_message = None
with self.subTest(model=model_path):
print(f"\n{'='*60}")
print(f"Testing: {model_path} (TP={tp_size})")
print(f"{'='*60}")
model_start = time.time()
startup_time = None
eval_time = None
score = None
# Set AMD-specific environment variables
if model_path in TRITON_ATTENTION_MODELS:
os.environ["SGLANG_USE_AITER"] = "0"
else:
os.environ["SGLANG_USE_AITER"] = "1"
# Build launch args
other_args = list(extra_args)
other_args.extend(["--log-level-http", "warning"])
if tp_size > 1:
other_args.extend(["--tp", str(tp_size)])
# Launch server with timing
print(f"🚀 Launching server...")
server_start = time.time()
process = popen_launch_server(
model=model_path,
base_url=self.base_url,
other_args=other_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
)
startup_time = time.time() - server_start
print(f"⏱️ Server startup: {startup_time:.1f}s")
try:
args = SimpleNamespace(
base_url=self.base_url,
model=model_path,
eval_name="mmmu",
num_examples=100,
num_threads=64,
max_tokens=30,
)
# Run evaluation with timing
print(f"📊 Running MMMU evaluation (100 samples)...")
eval_start = time.time()
# Retry up to 3 times
metrics = None
for attempt in range(3):
try:
metrics = run_eval(args)
score = metrics["score"]
if score >= accuracy_threshold:
break
except Exception as e:
print(f" Attempt {attempt + 1} failed with error: {e}")
if attempt == 2:
raise
eval_time = time.time() - eval_start
total_time = time.time() - model_start
# Print results
print(f"\n📈 Results for {model_path}:")
print(
f" Score: {score:.3f} (threshold: {accuracy_threshold:.2f})"
)
print(f"\n⏱️ Runtime breakdown:")
print(f" Server startup: {startup_time:.1f}s")
print(f" Evaluation: {eval_time:.1f}s")
print(f" Total: {total_time:.1f}s")
passed = score >= accuracy_threshold
if passed:
print(f"\n Status: ✅ PASSED")
else:
print(f"\n Status: ❌ FAILED")
write_results_to_json(model_path, metrics, "w" if is_first else "a")
is_first = False
all_results.append(
{
"model": model_path,
"tp_size": tp_size,
"score": score,
"threshold": accuracy_threshold,
"startup_time": startup_time,
"eval_time": eval_time,
"total_time": total_time,
"passed": passed,
"error": None,
}
)
except Exception as e:
error_message = str(e)
total_time = time.time() - model_start
print(f"\n❌ Error evaluating {model_path}: {error_message}")
all_results.append(
{
"model": model_path,
"tp_size": tp_size,
"score": None,
"threshold": accuracy_threshold,
"startup_time": startup_time,
"eval_time": None,
"total_time": total_time,
"passed": False,
"error": error_message,
}
)
finally:
print(f"\n🛑 Stopping server...")
kill_process_tree(process.pid)
# Calculate total test runtime
total_test_time = time.time() - total_test_start
# Generate summary
self._check_results(all_results, total_test_time)
def _check_results(self, results, total_test_time):
"""Check results and generate summary."""
failed_models = []
passed_count = 0
failed_count = 0
summary = (
"| Model | TP | Score | Threshold | Startup | Eval | Total | Status |\n"
)
summary += (
"| ----- | -- | ----- | --------- | ------- | ---- | ----- | ------ |\n"
)
for result in results:
model = result["model"]
score = result["score"]
tp_size = result["tp_size"]
threshold = result["threshold"]
startup_time = result.get("startup_time")
eval_time = result.get("eval_time")
total_time = result.get("total_time")
error = result.get("error")
if error:
status = "❌ ERROR"
failed_count += 1
failed_models.append(f"- {model}: ERROR - {error[:100]}")
elif result["passed"]:
status = "✅ PASS"
passed_count += 1
else:
status = "❌ FAIL"
failed_count += 1
failed_models.append(
f"- {model}: score={score:.4f}, threshold={threshold:.4f}"
)
# Format values
score_str = f"{score:.3f}" if score is not None else "N/A"
startup_str = f"{startup_time:.0f}s" if startup_time is not None else "N/A"
eval_str = f"{eval_time:.0f}s" if eval_time is not None else "N/A"
total_str = f"{total_time:.0f}s" if total_time is not None else "N/A"
summary += f"| {model} | {tp_size} | {score_str} | {threshold:.2f} | {startup_str} | {eval_str} | {total_str} | {status} |\n"
print(f"\n{'='*60}")
print("SUMMARY - AMD VLM MMMU Evaluation")
print(f"{'='*60}")
print(summary)
print(f"\n📊 Final Statistics:")
print(f" Passed: {passed_count}")
print(f" Failed: {failed_count}")
print(
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
)
if is_in_ci():
write_github_step_summary(
f"### TestNightlyVLMMmmuEvalAMD\n{summary}\n\n"
f"**Total Runtime:** {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
)
if failed_models:
failure_msg = "\n".join(failed_models)
raise AssertionError(f"The following models failed:\n{failure_msg}")
if __name__ == "__main__":
unittest.main()
test/registered/amd/test_deepseek_r1_mxfp4_perf.py
+166
View File
@@ -0,0 +1,166 @@
"""Nightly performance benchmark for DeepSeek-R1-MXFP4 model (MI35x).
This test benchmarks the DeepSeek-R1-MXFP4 quantized model on MI35x with 8 GPUs.
The model path can be configured via DEEPSEEK_R1_MXFP4_MODEL_PATH environment variable.
Example usage:
DEEPSEEK_R1_MXFP4_MODEL_PATH=/data2/models/amd-DeepSeek-R1-MXFP4-Preview python -m pytest test_deepseek_r1_mxfp4_perf.py -v
"""
import os
# Set HF cache to /data2/models/ for MI35x so HF models download there
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
import unittest
from typing import List
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
# Register for AMD CI - DeepSeek-R1-MXFP4 benchmark (~300 min)
register_amd_ci(
est_time=18000, suite="nightly-perf-8-gpu-deepseek-r1-mxfp4", nightly=True
)
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
"""Generate a simplified markdown report without traces and cost columns."""
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
gpu_config = os.getenv("GPU_CONFIG", "")
if gpu_config:
model_header += f" [{gpu_config}]"
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
for result in results:
itl = 1 / (result.output_throughput / result.batch_size) * 1000
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
return summary
# Model path configuration for MI35x DeepSeek-R1-MXFP4
# Priority: 1) env var, 2) local path, 3) HuggingFace model ID
DEEPSEEK_R1_MXFP4_LOCAL_PATH = "/data2/models/amd-DeepSeek-R1-MXFP4-Preview"
DEEPSEEK_R1_MXFP4_HF_MODEL_ID = "amd/DeepSeek-R1-MXFP4-Preview"
PROFILE_DIR = "performance_profiles_deepseek_r1_mxfp4"
def get_model_path() -> str:
"""Get effective model path: env var > local path > HF model ID."""
# Check env var first
env_path = os.environ.get("DEEPSEEK_R1_MXFP4_MODEL_PATH")
if env_path:
return env_path
# Check local path
if os.path.exists(DEEPSEEK_R1_MXFP4_LOCAL_PATH):
return DEEPSEEK_R1_MXFP4_LOCAL_PATH
# Fall back to HF model ID
return DEEPSEEK_R1_MXFP4_HF_MODEL_ID
class TestNightlyDeepseekR1MXFP4Performance(unittest.TestCase):
"""Nightly performance benchmark for DeepSeek-R1-MXFP4 model (MI35x).
Tests the DeepSeek-R1-MXFP4 quantized model on TP=8 with DP=8.
Uses local path if available, otherwise downloads from HuggingFace.
"""
@classmethod
def setUpClass(cls):
cls.model = get_model_path()
print(f"Using model path: {cls.model}")
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
# Define variant configurations for DeepSeek-R1-MXFP4 on MI35x
# Only run basic variant for perf (DP/TC/MTP covered in accuracy tests)
cls.variants = [
{
"name": "basic",
"other_args": [
"--trust-remote-code",
"--tp",
"8",
"--chunked-prefill-size",
"131072",
"--disable-radix-cache",
"--mem-fraction-static",
"0.85",
],
},
]
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
# Override full_report to remove traces help text
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_one_batch(self):
"""Run benchmark across all configured variants."""
failed_variants = []
# For local paths, check if exists. HF model IDs will download automatically.
is_local_path = self.model.startswith("/")
if is_local_path and not os.path.exists(self.model):
print(f"\n⏭️ SKIPPING: Local model not found at {self.model}")
self.runner.full_report += (
f"\n⏭️ Test skipped: Local model not found at {self.model}\n"
)
self.runner.write_final_report()
return
# Log model source
if is_local_path:
print(f"📁 Using local model: {self.model}")
else:
print(
f"📥 Using HuggingFace model: {self.model} (will download if not cached)"
)
try:
for variant_config in self.variants:
with self.subTest(variant=variant_config["name"]):
result_tuple = self.runner.run_benchmark_for_model(
model_path=self.model,
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=variant_config["other_args"],
variant=variant_config["name"],
extra_bench_args=["--trust-remote-code"],
)
results = result_tuple[0]
success = result_tuple[1]
if not success:
failed_variants.append(variant_config["name"])
# Use simplified report format without traces
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
)
finally:
self.runner.write_final_report()
if failed_variants:
raise AssertionError(
f"Benchmark failed for {self.model} with the following variants: "
f"{', '.join(failed_variants)}"
)
if __name__ == "__main__":
unittest.main()