diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index d86006f1e..072e6cc69 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -22,6 +22,7 @@ on: - 'nightly-test-general-4-gpu-h100' - 'nightly-test-general-8-gpu-h200' - 'nightly-test-general-8-gpu-h20' + - 'nightly-test-general-8-gpu-b200' - 'nightly-test-text-accuracy-2-gpu-runner' - 'nightly-test-text-perf-2-gpu-runner' - 'nightly-test-vlm-accuracy-2-gpu-runner' @@ -215,6 +216,28 @@ jobs: cd test python3 run_suite.py --hw cuda --suite nightly-8-gpu-h20 --nightly --continue-on-error + # General tests - 8 GPU B200 + nightly-test-general-8-gpu-b200: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') + runs-on: 8-gpu-b200 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + run: | + IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 120 + env: + GPU_CONFIG: "8-gpu-b200" + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400 + # Text model accuracy tests nightly-test-text-accuracy-2-gpu-runner: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner') @@ -573,6 +596,7 @@ jobs: - nightly-test-general-4-gpu-h100 - nightly-test-general-8-gpu-h200 - nightly-test-general-8-gpu-h20 + - nightly-test-general-8-gpu-b200 - nightly-test-text-accuracy-2-gpu-runner - nightly-test-text-perf-2-gpu-runner - nightly-test-vlm-accuracy-2-gpu-runner diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index 3819cc87a..1c768ec91 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -125,6 +125,9 @@ def run_eval(args): if getattr(args, "repeat", 1) == 1: result, latency, sampler = run_eval_once(args, base_url, eval_obj) + metrics = result.metrics | {"score": result.score} + print(f"Total latency: {latency:.3f} s") + print(f"Score: {metrics['score']:.3f}") else: from concurrent.futures import ThreadPoolExecutor @@ -147,27 +150,23 @@ def run_eval(args): print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}") print(f"Scores: {scores_repeat}") print("=" * 20) + metrics = result.metrics | {"scores": scores_repeat} + metrics = metrics | {"mean_score": mean_score} executor.shutdown() # Dump reports - metrics = result.metrics | {"score": result.score} file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}" report_filename = f"/tmp/{file_stem}.html" print(f"Writing report to {report_filename}") with open(report_filename, "w") as fh: fh.write(make_report(result)) - metrics = result.metrics | {"score": result.score} print(metrics) result_filename = f"/tmp/{file_stem}.json" with open(result_filename, "w") as f: f.write(json.dumps(metrics, indent=2)) print(f"Writing results to {result_filename}") - # Print results - print(f"Total latency: {latency:.3f} s") - print(f"Score: {metrics['score']:.3f}") - if getattr(args, "return_latency", False): return metrics, latency return metrics diff --git a/scripts/ci_monitor/ci_analyzer.py b/scripts/ci_monitor/ci_analyzer.py index 150e4f4f4..05bbab28c 100755 --- a/scripts/ci_monitor/ci_analyzer.py +++ b/scripts/ci_monitor/ci_analyzer.py @@ -134,6 +134,7 @@ class SGLangCIAnalyzer: "nightly-test-general-4-gpu-h100", "nightly-test-general-8-gpu-h200", "nightly-test-general-8-gpu-h20", + "nightly-test-general-8-gpu-b200", "nightly-test-text-accuracy-2-gpu-runner", "nightly-test-text-perf-2-gpu-runner", "nightly-test-vlm-accuracy-2-gpu-runner", @@ -232,6 +233,7 @@ class SGLangCIAnalyzer: "nightly-test-general-4-gpu-h100", "nightly-test-general-8-gpu-h200", "nightly-test-general-8-gpu-h20", + "nightly-test-general-8-gpu-b200", "nightly-test-text-accuracy-2-gpu-runner", "nightly-test-text-perf-2-gpu-runner", "nightly-test-vlm-accuracy-2-gpu-runner", @@ -728,6 +730,7 @@ class SGLangCIAnalyzer: "nightly-test-general-4-gpu-h100", "nightly-test-general-8-gpu-h200", "nightly-test-general-8-gpu-h20", + "nightly-test-general-8-gpu-b200", "nightly-test-text-accuracy-2-gpu-runner", "nightly-test-text-perf-2-gpu-runner", "nightly-test-vlm-accuracy-2-gpu-runner", diff --git a/test/nightly/test_deepseek_r1_fp8_trtllm_backend.py b/test/nightly/test_deepseek_r1_fp8_trtllm_backend.py index 198e7142a..b822bf3a4 100644 --- a/test/nightly/test_deepseek_r1_fp8_trtllm_backend.py +++ b/test/nightly/test_deepseek_r1_fp8_trtllm_backend.py @@ -1,4 +1,3 @@ -import os import unittest from types import SimpleNamespace @@ -6,7 +5,6 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, @@ -16,6 +14,7 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True) FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" +SERVER_LAUNCH_TIMEOUT = 1000 class TestDeepseekR1Fp8Flashinfer(CustomTestCase): @@ -52,19 +51,19 @@ class TestDeepseekR1Fp8Flashinfer(CustomTestCase): "10", "--attention-backend", "trtllm_mla", + "--fp8-gemm-backend", + "flashinfer_trtllm", "--moe-runner-backend", "flashinfer_trtllm", "--enable-symm-mem", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', ] cls.process = popen_launch_server( cls.model, cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + timeout=SERVER_LAUNCH_TIMEOUT, other_args=other_args, - env={ - **os.environ, - "SGLANG_ENABLE_FLASHINFER_FP8_GEMM": "1", - }, ) @classmethod diff --git a/test/nightly/test_deepseek_v32_gpqa.py b/test/nightly/test_deepseek_v32_gpqa.py new file mode 100644 index 000000000..f44e6efd4 --- /dev/null +++ b/test/nightly/test_deepseek_v32_gpqa.py @@ -0,0 +1,102 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + try_cached_model, + write_github_step_summary, +) + +register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True) + +# Use the latest version of DeepSeek-V3.2 +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" +SERVER_LAUNCH_TIMEOUT = 1200 + + +class TestDeepseekV32Accuracy(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEEPSEEK_V32_MODEL_PATH) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--tool-call-parser", + "deepseekv32", + "--reasoning-parser", + "deepseek-v3", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + def test_gpqa(self): + args = SimpleNamespace( + base_url=self.base_url, + model=DEEPSEEK_V32_MODEL_PATH, + eval_name="gpqa", + num_examples=198, + # use enough threads to allow parallelism + num_threads=198, + max_tokens=120000, + thinking_mode="deepseek-v3", + temperature=0.1, + # Repeat 4 times for shorter runtime. Ideally we should repeat at least 8 times. + repeat=4, + ) + + print(f"Evaluation start for gpqa") + metrics = run_eval(args) + print(f"Evaluation end for gpqa: {metrics=}, expected_score=0.835") + mean_score = metrics["mean_score"] + self.assertGreaterEqual(mean_score, 0.835) + + if is_in_ci(): + write_github_step_summary( + f"### test_gpqa (deepseek-v32)\n" f"Mean Score: {mean_score:.3f}\n" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/nightly/test_mistral_large3_perf.py b/test/nightly/test_mistral_large3_perf.py index 3e6feb52d..56aabbd6d 100644 --- a/test/nightly/test_mistral_large3_perf.py +++ b/test/nightly/test_mistral_large3_perf.py @@ -5,7 +5,6 @@ from types import SimpleNamespace from nightly_utils import NightlyBenchmarkRunner from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -14,8 +13,6 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True) - MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512" MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle" PROFILE_DIR = "performance_profiles_mistral_large3" diff --git a/test/run_suite_nightly.py b/test/run_suite_nightly.py index 14c34b6fe..676d598cd 100644 --- a/test/run_suite_nightly.py +++ b/test/run_suite_nightly.py @@ -30,6 +30,8 @@ suites = { ], "nightly-8-gpu-b200": [ TestFile("test_deepseek_r1_fp8_trtllm_backend.py", 3600), + TestFile("test_deepseek_v32_gpqa.py", 3600), + TestFile("test_mistral_large3_basic.py", 600), ], "nightly-4-gpu": [ TestFile("test_encoder_dp.py", 500), diff --git a/test/srt/test_mistral_large3_basic.py b/test/srt/test_mistral_large3_basic.py index 3b173f1ab..2cd496ac2 100644 --- a/test/srt/test_mistral_large3_basic.py +++ b/test/srt/test_mistral_large3_basic.py @@ -3,6 +3,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( @@ -14,6 +15,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) +register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True) MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"