Split dpsk fp4 4 gpu tests and move the mtp part to real stage b (#15553)
This commit is contained in:
Qiaolin Yu
@@ -685,7 +685,7 @@ jobs:
|
||||
|
||||
|
||||
stage-b-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, unit-test-backend-1-gpu]
|
||||
needs: [check-changes, call-gate, stage-a-test-1, sgl-kernel-build-wheels]
|
||||
if: |
|
||||
always() &&
|
||||
(
|
||||
@@ -721,8 +721,8 @@ jobs:
|
||||
- name: Run test
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/srt
|
||||
IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200-stage-b --timeout-per-file 1800
|
||||
cd test
|
||||
IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200
|
||||
|
||||
|
||||
unit-test-backend-2-gpu:
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=900, suite="stage-b-test-4-gpu-b200")
|
||||
|
||||
|
||||
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
|
||||
SERVER_LAUNCH_TIMEOUT = 1200
|
||||
|
||||
|
||||
class TestDeepseekV3FP4MTP(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
os.environ["SGLANG_ENABLE_SPEC_V2"] = "1"
|
||||
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = [
|
||||
"--tp",
|
||||
"4",
|
||||
"--attention-backend",
|
||||
"trtllm_mla",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--quantization",
|
||||
"modelopt_fp4",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--kv-cache-dtype",
|
||||
"fp8_e4m3",
|
||||
"--model-loader-extra-config",
|
||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||
]
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
if "SGLANG_ENABLE_SPEC_V2" in os.environ:
|
||||
del os.environ["SGLANG_ENABLE_SPEC_V2"]
|
||||
|
||||
def test_a_gsm8k(
|
||||
self,
|
||||
): # Append an "a" to make this test run first (alphabetically) to warm up the server
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=200,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
server_info = requests.get(self.base_url + "/get_server_info")
|
||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
||||
"avg_spec_accept_length"
|
||||
]
|
||||
print(f"{avg_spec_accept_length=}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v3-fp4 mtp)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
f"{avg_spec_accept_length=:.2f}\n"
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["accuracy"], 0.94)
|
||||
self.assertGreater(avg_spec_accept_length, 2.04)
|
||||
|
||||
def test_bs_1_speed(self):
|
||||
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
||||
acc_length, speed = send_one_prompt(args)
|
||||
|
||||
print(f"{acc_length=:.2f} {speed=:.2f}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v3-fp4 mtp)\n"
|
||||
f"{acc_length=:.2f}\n"
|
||||
f"{speed=:.2f} token/s\n"
|
||||
)
|
||||
|
||||
self.assertGreater(acc_length, 2.65)
|
||||
self.assertGreater(speed, 150)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -24,6 +24,7 @@ PER_COMMIT_SUITES = {
|
||||
"stage-b-test-small-1-gpu",
|
||||
"stage-b-test-large-1-gpu",
|
||||
"stage-b-test-large-2-gpu",
|
||||
"stage-b-test-4-gpu-b200",
|
||||
],
|
||||
HWBackend.NPU: [],
|
||||
}
|
||||
|
||||
@@ -163,10 +163,8 @@ suites = {
|
||||
TestFile("test_disaggregation_pp.py", 180),
|
||||
TestFile("test_disaggregation_dp_attention.py", 155),
|
||||
],
|
||||
"per-commit-4-gpu-b200-stage-b": [
|
||||
TestFile("test_deepseek_v3_fp4_4gpu.py", 2000), # Stage B test
|
||||
],
|
||||
"per-commit-4-gpu-b200": [
|
||||
TestFile("test_deepseek_v3_fp4_4gpu.py", 1500),
|
||||
TestFile("test_flash_attention_4.py", 90),
|
||||
TestFile("test_fp8_blockwise_gemm.py", 280),
|
||||
TestFile("test_gpt_oss_4gpu.py", 700),
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.send_one import BenchArgs, send_one_prompt
|
||||
@@ -85,97 +82,6 @@ class TestDeepseekV3FP4(CustomTestCase):
|
||||
self.assertGreater(speed, 75)
|
||||
|
||||
|
||||
class TestDeepseekV3FP4MTP(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
os.environ["SGLANG_ENABLE_SPEC_V2"] = "1"
|
||||
cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = [
|
||||
"--tp",
|
||||
"4",
|
||||
"--attention-backend",
|
||||
"trtllm_mla",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--quantization",
|
||||
"modelopt_fp4",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--kv-cache-dtype",
|
||||
"fp8_e4m3",
|
||||
"--model-loader-extra-config",
|
||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||
]
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
if "SGLANG_ENABLE_SPEC_V2" in os.environ:
|
||||
del os.environ["SGLANG_ENABLE_SPEC_V2"]
|
||||
|
||||
def test_a_gsm8k(
|
||||
self,
|
||||
): # Append an "a" to make this test run first (alphabetically) to warm up the server
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=200,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"{metrics=}")
|
||||
|
||||
server_info = requests.get(self.base_url + "/get_server_info")
|
||||
avg_spec_accept_length = server_info.json()["internal_states"][0][
|
||||
"avg_spec_accept_length"
|
||||
]
|
||||
print(f"{avg_spec_accept_length=}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v3-fp4 mtp)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
f"{avg_spec_accept_length=:.2f}\n"
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["accuracy"], 0.94)
|
||||
self.assertGreater(avg_spec_accept_length, 2.04)
|
||||
|
||||
def test_bs_1_speed(self):
|
||||
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
|
||||
acc_length, speed = send_one_prompt(args)
|
||||
|
||||
print(f"{acc_length=:.2f} {speed=:.2f}")
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v3-fp4 mtp)\n"
|
||||
f"{acc_length=:.2f}\n"
|
||||
f"{speed=:.2f} token/s\n"
|
||||
)
|
||||
|
||||
self.assertGreater(acc_length, 2.65)
|
||||
self.assertGreater(speed, 150)
|
||||
|
||||
|
||||
class TestDeepseekV3FP4PiecewiseCudaGraph(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
|
||||
Reference in New Issue
Block a user