85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
"""
|
|
Performance tests for single GPU that need H200 (80GB) - FP8 and EAGLE tests.
|
|
"""
|
|
|
|
import unittest
|
|
|
|
from sglang.srt.utils import is_hip
|
|
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
|
from sglang.test.test_utils import (
|
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
|
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
|
DEFAULT_TARGET_MODEL_EAGLE,
|
|
CustomTestCase,
|
|
is_in_amd_ci,
|
|
is_in_ci,
|
|
run_bench_serving,
|
|
write_github_step_summary,
|
|
)
|
|
|
|
register_cuda_ci(est_time=300, suite="stage-b-test-large-1-gpu")
|
|
register_amd_ci(est_time=300, suite="stage-b-test-large-1-gpu-amd")
|
|
|
|
|
|
class TestBenchServing1GPULarge(CustomTestCase):
|
|
def test_offline_throughput_default_fp8(self):
|
|
res = run_bench_serving(
|
|
model=DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
|
num_prompts=500,
|
|
request_rate=float("inf"),
|
|
other_server_args=[],
|
|
)
|
|
|
|
if is_in_ci():
|
|
write_github_step_summary(
|
|
f"### test_offline_throughput_default_fp8\n"
|
|
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
|
)
|
|
if is_in_amd_ci():
|
|
self.assertGreater(res["output_throughput"], 3500)
|
|
else:
|
|
self.assertGreater(res["output_throughput"], 4300)
|
|
|
|
@unittest.skipIf(is_hip(), "Skip Eagle test for ROCm")
|
|
def test_online_latency_eagle(self):
|
|
res = run_bench_serving(
|
|
model=DEFAULT_TARGET_MODEL_EAGLE,
|
|
num_prompts=300,
|
|
request_rate=8,
|
|
sharegpt_context_len=3072,
|
|
disable_ignore_eos=True,
|
|
dataset_name="sharegpt",
|
|
other_server_args=[
|
|
"--speculative-algorithm",
|
|
"EAGLE",
|
|
"--speculative-draft-model-path",
|
|
DEFAULT_DRAFT_MODEL_EAGLE,
|
|
"--speculative-num-steps",
|
|
"5",
|
|
"--speculative-eagle-topk",
|
|
"4",
|
|
"--speculative-num-draft-tokens",
|
|
"16",
|
|
"--mem-fraction-static",
|
|
"0.7",
|
|
],
|
|
need_warmup=True,
|
|
seed=42,
|
|
)
|
|
|
|
if is_in_ci():
|
|
write_github_step_summary(
|
|
f"### test_online_latency_eagle\n"
|
|
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
|
f"accept_length: {res['accept_length']:.2f} \n"
|
|
)
|
|
if is_in_amd_ci():
|
|
self.assertLess(res["median_e2e_latency_ms"], 1800)
|
|
else:
|
|
self.assertLess(res["median_e2e_latency_ms"], 900)
|
|
self.assertGreater(res["accept_length"], 3.0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|