chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

A few updates to the night tests (#17694)

Browse Source
This commit is contained in:
Kangyan-Zhou
2026-01-25 11:20:17 -08:00
committed by GitHub
parent a883906a24
commit 9123491430
11 changed files with 52 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
python/sglang/test/accuracy_test_runner.py
View File

@@ -40,6 +40,7 @@ class AccuracyTestResult:
baseline_accuracy: float
error: Optional[str]
latency: Optional[float] = None
variant: Optional[str] = None
def write_accuracy_github_summary(
@@ -54,16 +55,18 @@ def write_accuracy_github_summary(
dataset: Dataset name used for evaluation
results: List of AccuracyTestResult objects
"""
summary = f"## {test_name} - Accuracy ({dataset})\n"
summary += "| model | status | score | baseline | error |\n"
summary += "| ----- | ------ | ----- | -------- | ----- |\n"
summary = f"#### {test_name} - Accuracy ({dataset})\n"
summary += "| config | status | score | baseline | error |\n"
summary += "| ------ | ------ | ----- | -------- | ----- |\n"
for result in results:
status_emoji = "✅" if result.passed else "❌"
score_str = f"{result.score:.4f}" if result.score is not None else "N/A"
baseline_str = f"{result.baseline_accuracy:.4f}"
error_str = result.error if result.error else "-"
summary += f"| {result.model} | {status_emoji} | {score_str} | {baseline_str} | {error_str} |\n"
# Use variant name if available, otherwise use model path
config_name = result.variant if result.variant else result.model
summary += f"| {config_name} | {status_emoji} | {score_str} | {baseline_str} | {error_str} |\n"
write_github_step_summary(summary)
@@ -239,6 +242,7 @@ def run_accuracy_test(
score=None,
baseline_accuracy=params.baseline_accuracy,
error=error,
variant=model.variant,
)
# Validate against baseline
@@ -265,4 +269,5 @@ def run_accuracy_test(
baseline_accuracy=params.baseline_accuracy,
error=error if not passed else None,
latency=latency,
variant=model.variant,
)

2
test/registered/8-gpu-models/test_deepseek_v31.py
View File

@@ -60,7 +60,7 @@ class TestDeepseekV31(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.1 Unified",
test_name="DeepSeek-V3.1",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=0.935
),

2
test/registered/8-gpu-models/test_deepseek_v32.py
View File

@@ -82,7 +82,7 @@ class TestDeepseekV32(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="DeepSeek-V3.2 Unified",
test_name="DeepSeek-V3.2",
accuracy_params=AccuracyTestParams(
dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
),

2
test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py
View File

@@ -69,7 +69,7 @@ class TestDeepseekV32CPSingleNode(unittest.TestCase):
ModelLaunchSettings(
DEEPSEEK_V32_EXP_MODEL_PATH,
tp_size=8,
extra_args=BASE_ARGS + ["--tp=8"] + MTP_ARGS + CP_ROUND_ROBIN_ARGS,
extra_args=BASE_ARGS + MTP_ARGS + CP_ROUND_ROBIN_ARGS,
variant="CP-round-robin-split",
),
]

2
test/registered/8-gpu-models/test_glm_46.py
View File

@@ -40,7 +40,7 @@ class TestGLM46(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="GLM-4.6 Unified",
test_name="GLM-4.6",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_glm_4_6",

2
test/registered/8-gpu-models/test_glm_46_fp8.py
View File

@@ -51,7 +51,7 @@ class TestGLM46FP8(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="GLM-4.6-FP8 Unified",
test_name="GLM-4.6-FP8",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_glm_4_6_fp8",

2
test/registered/8-gpu-models/test_kimi_k2.py
View File

@@ -41,7 +41,7 @@ class TestKimiK2(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="Kimi-K2-Thinking Unified",
test_name="Kimi-K2-Thinking",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.94),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_kimi_k2_thinking",

2
test/registered/8-gpu-models/test_llama4.py
View File

@@ -44,7 +44,7 @@ class TestLlama4(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="Llama-4-Scout Unified",
test_name="Llama-4-Scout",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.9),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_llama4",

2
test/registered/8-gpu-models/test_minimax_m2.py
View File

@@ -43,7 +43,7 @@ class TestMiniMaxM2(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="MiniMax-M2 Unified",
test_name="MiniMax-M2",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_minimax_m2",

2
test/registered/8-gpu-models/test_mistral_large3.py
View File

@@ -76,7 +76,7 @@ class TestMistralLarge3(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="Mistral-Large-3 Unified",
test_name="Mistral-Large-3",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.90),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_mistral_large3",

47
test/registered/8-gpu-models/test_qwen3_235b.py
View File

@@ -4,46 +4,67 @@ from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.performance_test_runner import PerformanceTestParams
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
from sglang.test.test_utils import ModelLaunchSettings
# Runs on both H200 and B200 via nightly-8-gpu-common suite
register_cuda_ci(est_time=1800, suite="nightly-8-gpu-common", nightly=True)
QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
QWEN3_235B_FP8_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8"
QWEN3_235B_EAGLE3_MODEL_PATH = (
"lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan"
)
@unittest.skipIf(not is_blackwell_system(), "Requires B200")
class TestQwen3235B(unittest.TestCase):
"""Unified test class for Qwen3-235B performance and accuracy.
class TestQwen3235BFP8(unittest.TestCase):
"""Test class for Qwen3-235B-FP8 performance and accuracy.
Single variant with simple TP=8 configuration.
Runs BOTH:
Two variants:
- basic: TP=8
- eagle3: TP=8 + EP=2 + EAGLE3 speculative decoding
Each variant runs BOTH:
- Performance test (using NightlyBenchmarkRunner)
- Accuracy test (using run_eval with mgsm_en)
- Accuracy test (using run_eval with gsm8k)
"""
def test_qwen3_235b(self):
"""Run performance and accuracy for Qwen3-235B."""
def test_qwen3_235b_fp8_all_variants(self):
"""Run performance and accuracy for Qwen3-235B-FP8."""
base_args = [
"--tp=8",
"--trust-remote-code",
]
eagle3_args = [
"--ep=2",
"--speculative-algorithm=EAGLE3",
f"--speculative-draft-model-path={QWEN3_235B_EAGLE3_MODEL_PATH}",
"--speculative-num-steps=3",
"--speculative-eagle-topk=1",
"--speculative-num-draft-tokens=4",
]
variants = [
# Variant: "basic" - TP=8
ModelLaunchSettings(
QWEN3_235B_MODEL_PATH,
QWEN3_235B_FP8_MODEL_PATH,
tp_size=8,
extra_args=base_args,
variant="TP8",
),
# Variant: "eagle3" - TP=8 + EP=2 + EAGLE3 speculative decoding
ModelLaunchSettings(
QWEN3_235B_FP8_MODEL_PATH,
tp_size=8,
extra_args=base_args + eagle3_args,
variant="TP8+EP2+EAGLE3",
),
]
run_combined_tests(
models=variants,
test_name="Qwen3-235B Unified",
test_name="Qwen3-235B-FP8",
accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.88),
performance_params=PerformanceTestParams(
profile_dir="performance_profiles_qwen3_235b",
profile_dir="performance_profiles_qwen3_235b_fp8",
),
)