chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Add nightly accuracy test for DeepSeek V3.2 (#14935)

Browse Source
This commit is contained in:
Baizhou Zhang
2025-12-13 12:11:16 -08:00
committed by GitHub
parent 2285affffa
commit ab3ffd1c8e
8 changed files with 144 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/nightly-test-nvidia.yml vendored
View File

@@ -22,6 +22,7 @@ on:
- 'nightly-test-general-4-gpu-h100'
- 'nightly-test-general-8-gpu-h200'
- 'nightly-test-general-8-gpu-h20'
- 'nightly-test-general-8-gpu-b200'
- 'nightly-test-text-accuracy-2-gpu-runner'
- 'nightly-test-text-perf-2-gpu-runner'
- 'nightly-test-vlm-accuracy-2-gpu-runner'
@@ -215,6 +216,28 @@ jobs:
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error
# General tests - 8 GPU B200
nightly-test-general-8-gpu-b200:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
runs-on: 8-gpu-b200
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
run: |
IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
- name: Run test
timeout-minutes: 120
env:
GPU_CONFIG: "8-gpu-b200"
run: |
cd test
python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
# Text model accuracy tests
nightly-test-text-accuracy-2-gpu-runner:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
@@ -573,6 +596,7 @@ jobs:
- nightly-test-general-4-gpu-h100
- nightly-test-general-8-gpu-h200
- nightly-test-general-8-gpu-h20
- nightly-test-general-8-gpu-b200
- nightly-test-text-accuracy-2-gpu-runner
- nightly-test-text-perf-2-gpu-runner
- nightly-test-vlm-accuracy-2-gpu-runner

11
python/sglang/test/run_eval.py
View File

@@ -125,6 +125,9 @@ def run_eval(args):
if getattr(args, "repeat", 1) == 1:
result, latency, sampler = run_eval_once(args, base_url, eval_obj)
metrics = result.metrics | {"score": result.score}
print(f"Total latency: {latency:.3f} s")
print(f"Score: {metrics['score']:.3f}")
else:
from concurrent.futures import ThreadPoolExecutor
@@ -147,27 +150,23 @@ def run_eval(args):
print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
print(f"Scores: {scores_repeat}")
print("=" * 20)
metrics = result.metrics | {"scores": scores_repeat}
metrics = metrics | {"mean_score": mean_score}
executor.shutdown()
# Dump reports
metrics = result.metrics | {"score": result.score}
file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
report_filename = f"/tmp/{file_stem}.html"
print(f"Writing report to {report_filename}")
with open(report_filename, "w") as fh:
fh.write(make_report(result))
metrics = result.metrics | {"score": result.score}
print(metrics)
result_filename = f"/tmp/{file_stem}.json"
with open(result_filename, "w") as f:
f.write(json.dumps(metrics, indent=2))
print(f"Writing results to {result_filename}")
# Print results
print(f"Total latency: {latency:.3f} s")
print(f"Score: {metrics['score']:.3f}")
if getattr(args, "return_latency", False):
return metrics, latency
return metrics

3
scripts/ci_monitor/ci_analyzer.py
View File

@@ -134,6 +134,7 @@ class SGLangCIAnalyzer:
"nightly-test-general-4-gpu-h100",
"nightly-test-general-8-gpu-h200",
"nightly-test-general-8-gpu-h20",
"nightly-test-general-8-gpu-b200",
"nightly-test-text-accuracy-2-gpu-runner",
"nightly-test-text-perf-2-gpu-runner",
"nightly-test-vlm-accuracy-2-gpu-runner",
@@ -232,6 +233,7 @@ class SGLangCIAnalyzer:
"nightly-test-general-4-gpu-h100",
"nightly-test-general-8-gpu-h200",
"nightly-test-general-8-gpu-h20",
"nightly-test-general-8-gpu-b200",
"nightly-test-text-accuracy-2-gpu-runner",
"nightly-test-text-perf-2-gpu-runner",
"nightly-test-vlm-accuracy-2-gpu-runner",
@@ -728,6 +730,7 @@ class SGLangCIAnalyzer:
"nightly-test-general-4-gpu-h100",
"nightly-test-general-8-gpu-h200",
"nightly-test-general-8-gpu-h20",
"nightly-test-general-8-gpu-b200",
"nightly-test-text-accuracy-2-gpu-runner",
"nightly-test-text-perf-2-gpu-runner",
"nightly-test-vlm-accuracy-2-gpu-runner",

13
test/nightly/test_deepseek_r1_fp8_trtllm_backend.py
View File

@@ -1,4 +1,3 @@
import os
import unittest
from types import SimpleNamespace
@@ -6,7 +5,6 @@ from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
@@ -16,6 +14,7 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True)
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
SERVER_LAUNCH_TIMEOUT = 1000
class TestDeepseekR1Fp8Flashinfer(CustomTestCase):
@@ -52,19 +51,19 @@ class TestDeepseekR1Fp8Flashinfer(CustomTestCase):
"10",
"--attention-backend",
"trtllm_mla",
"--fp8-gemm-backend",
"flashinfer_trtllm",
"--moe-runner-backend",
"flashinfer_trtllm",
"--enable-symm-mem",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
env={
**os.environ,
"SGLANG_ENABLE_FLASHINFER_FP8_GEMM": "1",
},
)
@classmethod

102
test/nightly/test_deepseek_v32_gpqa.py Normal file
View File

@@ -0,0 +1,102 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
try_cached_model,
write_github_step_summary,
)
register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True)
# Use the latest version of DeepSeek-V3.2
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
SERVER_LAUNCH_TIMEOUT = 1200
class TestDeepseekV32Accuracy(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = try_cached_model(DEEPSEEK_V32_MODEL_PATH)
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--tp",
"8",
"--enable-dp-attention",
"--dp",
"8",
"--tool-call-parser",
"deepseekv32",
"--reasoning-parser",
"deepseek-v3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(
self,
):
args = SimpleNamespace(
num_shots=20,
data_path=None,
num_questions=1400,
parallel=1400,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.935)
def test_gpqa(self):
args = SimpleNamespace(
base_url=self.base_url,
model=DEEPSEEK_V32_MODEL_PATH,
eval_name="gpqa",
num_examples=198,
# use enough threads to allow parallelism
num_threads=198,
max_tokens=120000,
thinking_mode="deepseek-v3",
temperature=0.1,
# Repeat 4 times for shorter runtime. Ideally we should repeat at least 8 times.
repeat=4,
)
print(f"Evaluation start for gpqa")
metrics = run_eval(args)
print(f"Evaluation end for gpqa: {metrics=}, expected_score=0.835")
mean_score = metrics["mean_score"]
self.assertGreaterEqual(mean_score, 0.835)
if is_in_ci():
write_github_step_summary(
f"### test_gpqa (deepseek-v32)\n" f"Mean Score: {mean_score:.3f}\n"
)
if __name__ == "__main__":
unittest.main()

3
test/nightly/test_mistral_large3_perf.py
View File

@@ -5,7 +5,6 @@ from types import SimpleNamespace
from nightly_utils import NightlyBenchmarkRunner
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -14,8 +13,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True)
MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle"
PROFILE_DIR = "performance_profiles_mistral_large3"

2
test/run_suite_nightly.py
View File

@@ -30,6 +30,8 @@ suites = {
],
"nightly-8-gpu-b200": [
TestFile("test_deepseek_r1_fp8_trtllm_backend.py", 3600),
TestFile("test_deepseek_v32_gpqa.py", 3600),
TestFile("test_mistral_large3_basic.py", 600),
],
"nightly-4-gpu": [
TestFile("test_encoder_dp.py", 500),

2
test/srt/test_mistral_large3_basic.py
View File

@@ -3,6 +3,7 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
@@ -14,6 +15,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True)
MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"