Migrate performance, accuracy, and quantization tests to CI registry (#17177)
Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com>
This commit is contained in:
Alison Shao
co-authored by
Kangyan-Zhou
Kangyan-Zhou
parent
a3d9a21882
commit
8916b9d080
@@ -1,111 +0,0 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
class TestAWQ(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--trust-remote-code"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_mmlu(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mmlu",
|
||||
num_examples=64,
|
||||
num_threads=32,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(metrics["score"], 0.64)
|
||||
|
||||
|
||||
class TestAWQMarlinBfloat16(CustomTestCase):
|
||||
"""
|
||||
Verify that the model can be loaded with bfloat16 dtype and awq_marlin quantization
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ"
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--dtype", "bfloat16", "--quantization", "awq_marlin"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_mmlu(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mmlu",
|
||||
num_examples=64,
|
||||
num_threads=32,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(metrics["score"], 0.83)
|
||||
|
||||
|
||||
class TestAWQMarlinFloat16(CustomTestCase):
|
||||
"""
|
||||
Verify that the model can be loaded with float16 dtype and awq_marlin quantization
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ"
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--dtype", "float16", "--quantization", "awq_marlin"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_mmlu(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mmlu",
|
||||
num_examples=64,
|
||||
num_threads=32,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(metrics["score"], 0.85)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,438 +0,0 @@
|
||||
import itertools
|
||||
import unittest
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from sgl_kernel.scalar_type import scalar_types
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import fused_marlin_moe
|
||||
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
|
||||
from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
set_global_server_args_for_scheduler(object.__new__(ServerArgs))
|
||||
|
||||
|
||||
def stack_and_dev(tensors: list[torch.Tensor]):
|
||||
dev = tensors[0].device
|
||||
return torch.stack(tensors, dim=0).to(dev)
|
||||
|
||||
|
||||
def torch_experts(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: Optional[torch.Tensor] = None,
|
||||
quant_dtype: Optional[torch.dtype] = None,
|
||||
apply_router_weights_on_input: bool = False,
|
||||
) -> torch.Tensor:
|
||||
assert (
|
||||
global_num_experts == -1
|
||||
or (global_num_experts == w1.shape[0] and expert_map is None)
|
||||
or (expert_map is not None and global_num_experts == expert_map.shape[0])
|
||||
), "Invalid expert configuration"
|
||||
|
||||
M, K = a.shape
|
||||
topk = topk_ids.shape[1]
|
||||
|
||||
if apply_router_weights_on_input:
|
||||
assert topk == 1, "apply_router_weights_on_input only works with topk=1"
|
||||
a = a * topk_weight.to(a.dtype)
|
||||
|
||||
a = a.view(M, -1, K).repeat(1, topk, 1).reshape(-1, K)
|
||||
|
||||
out = torch.zeros(M * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
|
||||
num_experts = w1.shape[0]
|
||||
|
||||
topk_ids = topk_ids.view(-1)
|
||||
if expert_map is not None:
|
||||
topk_ids = expert_map[topk_ids]
|
||||
|
||||
f32 = torch.float32
|
||||
|
||||
for i in range(num_experts):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
if quant_dtype is None:
|
||||
tmp1 = a[mask] @ w1[i].transpose(0, 1)
|
||||
tmp2 = SiluAndMul()(tmp1)
|
||||
out[mask] = tmp2 @ w2[i].transpose(0, 1)
|
||||
|
||||
if apply_router_weights_on_input:
|
||||
return out
|
||||
else:
|
||||
return (
|
||||
(out.view(M, -1, w2.shape[1]).to(f32) * topk_weight.view(M, -1, 1))
|
||||
.sum(dim=1)
|
||||
.to(out.dtype)
|
||||
)
|
||||
|
||||
|
||||
def torch_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
topk: int,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
return torch_experts(
|
||||
a, w1, w2, topk_weight, topk_ids, global_num_experts, expert_map
|
||||
)
|
||||
|
||||
|
||||
def marlin_moe_generate_valid_test_cases():
|
||||
m_list = [1, 123, 666]
|
||||
n_list = [128, 1024]
|
||||
k_list = [256, 2048]
|
||||
e_list = [4, 12]
|
||||
topk_list = [2, 3]
|
||||
dtype_list = [torch.half, torch.bfloat16]
|
||||
group_size_list = [64, 128]
|
||||
act_order_list = [True, False]
|
||||
quant_type_list = [
|
||||
scalar_types.uint4,
|
||||
scalar_types.uint4b8,
|
||||
]
|
||||
is_k_full_list = [True, False]
|
||||
|
||||
all_combinations = itertools.product(
|
||||
m_list,
|
||||
n_list,
|
||||
k_list,
|
||||
e_list,
|
||||
topk_list,
|
||||
dtype_list,
|
||||
group_size_list,
|
||||
act_order_list,
|
||||
quant_type_list,
|
||||
is_k_full_list,
|
||||
)
|
||||
|
||||
def is_valid(m, n, k, e, topk, dtype, group_size, act_order, quant_type, is_k_full):
|
||||
if group_size > 0 and k % group_size != 0:
|
||||
return False
|
||||
|
||||
if act_order:
|
||||
if group_size in (-1, k, n):
|
||||
return False
|
||||
if quant_type not in [scalar_types.uint4b8]:
|
||||
return False
|
||||
else:
|
||||
if not is_k_full:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
cases = []
|
||||
for case in all_combinations:
|
||||
if is_valid(*case):
|
||||
cases.append(case)
|
||||
return cases
|
||||
|
||||
|
||||
class TestFusedMarlinMoe(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not torch.cuda.is_available():
|
||||
raise unittest.SkipTest("This test requires a CUDA device.")
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
def test_fused_marlin_moe(self):
|
||||
test_cases = marlin_moe_generate_valid_test_cases()
|
||||
|
||||
for (
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
e,
|
||||
topk,
|
||||
dtype,
|
||||
group_size,
|
||||
act_order,
|
||||
quant_type,
|
||||
is_k_full,
|
||||
) in test_cases:
|
||||
with self.subTest(
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
e=e,
|
||||
topk=topk,
|
||||
dtype=dtype,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
quant_type=quant_type,
|
||||
is_k_full=is_k_full,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
|
||||
has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8]
|
||||
|
||||
if act_order:
|
||||
if group_size == -1:
|
||||
continue
|
||||
if group_size in (k, n):
|
||||
continue
|
||||
if has_zp:
|
||||
continue
|
||||
else:
|
||||
if not is_k_full:
|
||||
continue
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 20
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 20
|
||||
|
||||
e_map = None
|
||||
|
||||
w_ref1_l = []
|
||||
qweight1_l = []
|
||||
scales1_l = []
|
||||
zeros1_l = []
|
||||
g_idx1_l = []
|
||||
sort_indices1_l = []
|
||||
|
||||
for i in range(w1.shape[0]):
|
||||
if has_zp:
|
||||
w_ref1, qweight1, scales1, zeros1 = awq_marlin_quantize(
|
||||
w1[i].transpose(1, 0), quant_type, group_size
|
||||
)
|
||||
|
||||
w_ref1_l.append(w_ref1.T)
|
||||
qweight1_l.append(qweight1)
|
||||
scales1_l.append(scales1)
|
||||
zeros1_l.append(zeros1)
|
||||
else:
|
||||
test_perm = torch.randperm(k)
|
||||
w_ref1, qweight1, scales1, g_idx1, sort_indices1, _ = (
|
||||
marlin_quantize(
|
||||
w1[i].transpose(1, 0),
|
||||
quant_type,
|
||||
group_size,
|
||||
act_order,
|
||||
test_perm,
|
||||
)
|
||||
)
|
||||
|
||||
w_ref1_l.append(w_ref1.T)
|
||||
qweight1_l.append(qweight1)
|
||||
scales1_l.append(scales1)
|
||||
g_idx1_l.append(g_idx1)
|
||||
sort_indices1_l.append(sort_indices1)
|
||||
|
||||
w_ref1 = stack_and_dev(w_ref1_l)
|
||||
qweight1 = stack_and_dev(qweight1_l).contiguous()
|
||||
scales1 = stack_and_dev(scales1_l)
|
||||
g_idx1 = stack_and_dev(g_idx1_l) if g_idx1_l else None
|
||||
zeros1 = stack_and_dev(zeros1_l) if zeros1_l else None
|
||||
sort_indices1 = (
|
||||
stack_and_dev(sort_indices1_l) if sort_indices1_l else None
|
||||
)
|
||||
|
||||
w_ref2_l = []
|
||||
qweight2_l = []
|
||||
scales2_l = []
|
||||
zeros2_l = []
|
||||
g_idx2_l = []
|
||||
sort_indices2_l = []
|
||||
|
||||
for i in range(w2.shape[0]):
|
||||
if has_zp:
|
||||
w_ref2, qweight2, scales2, zeros2 = awq_marlin_quantize(
|
||||
w2[i].transpose(1, 0), quant_type, group_size
|
||||
)
|
||||
|
||||
w_ref2_l.append(w_ref2.T)
|
||||
qweight2_l.append(qweight2)
|
||||
scales2_l.append(scales2)
|
||||
zeros2_l.append(zeros2)
|
||||
else:
|
||||
test_perm = torch.randperm(n)
|
||||
w_ref2, qweight2, scales2, g_idx2, sort_indices2, _ = (
|
||||
marlin_quantize(
|
||||
w2[i].transpose(1, 0),
|
||||
quant_type,
|
||||
group_size,
|
||||
act_order,
|
||||
test_perm,
|
||||
)
|
||||
)
|
||||
|
||||
w_ref2_l.append(w_ref2.T)
|
||||
qweight2_l.append(qweight2)
|
||||
scales2_l.append(scales2)
|
||||
g_idx2_l.append(g_idx2)
|
||||
sort_indices2_l.append(sort_indices2)
|
||||
|
||||
w_ref2 = stack_and_dev(w_ref2_l)
|
||||
qweight2 = stack_and_dev(qweight2_l).contiguous()
|
||||
scales2 = stack_and_dev(scales2_l)
|
||||
g_idx2 = stack_and_dev(g_idx2_l) if g_idx2_l else None
|
||||
zeros2 = stack_and_dev(zeros2_l) if zeros2_l else None
|
||||
sort_indices2 = (
|
||||
stack_and_dev(sort_indices2_l) if sort_indices2_l else None
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
from sglang.srt.layers.moe.topk import fused_topk_torch_native
|
||||
|
||||
topk_weights, topk_ids = fused_topk_torch_native(a, score, topk, False)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
w_ref1,
|
||||
w_ref2,
|
||||
score,
|
||||
topk,
|
||||
global_num_experts=e,
|
||||
expert_map=e_map,
|
||||
)
|
||||
|
||||
marlin_output = fused_marlin_moe(
|
||||
a,
|
||||
qweight1,
|
||||
qweight2,
|
||||
scales1,
|
||||
scales2,
|
||||
score,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
global_num_experts=e,
|
||||
expert_map=e_map,
|
||||
g_idx1=g_idx1,
|
||||
g_idx2=g_idx2,
|
||||
sort_indices1=sort_indices1,
|
||||
sort_indices2=sort_indices2,
|
||||
w1_zeros=zeros1,
|
||||
w2_zeros=zeros2,
|
||||
num_bits=4,
|
||||
is_k_full=is_k_full,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
marlin_output, torch_output, atol=5e-2, rtol=0
|
||||
)
|
||||
|
||||
def test_fused_marlin_moe_expert_parallelism(self):
|
||||
m_list = [1, 16, 128]
|
||||
e_list = [8, 16]
|
||||
|
||||
for m in m_list:
|
||||
for e in e_list:
|
||||
with self.subTest(m=m, e=e):
|
||||
torch.manual_seed(100)
|
||||
|
||||
n, k = 256, 256
|
||||
topk = 2
|
||||
ep_size = 2
|
||||
group_size = 128
|
||||
dtype = torch.bfloat16
|
||||
quant_type = scalar_types.uint4b8
|
||||
|
||||
local_e = e // ep_size
|
||||
e_ids = torch.arange(local_e, device="cuda", dtype=torch.int32)
|
||||
e_map = torch.full((e,), -1, device="cuda", dtype=torch.int32)
|
||||
e_map[e_ids] = torch.arange(
|
||||
local_e, device="cuda", dtype=torch.int32
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1_full = (
|
||||
torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 20
|
||||
)
|
||||
w2_full = torch.randn((e, k, n), device="cuda", dtype=dtype) / 20
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
score[:, e_ids] += 10.0
|
||||
|
||||
w1 = w1_full[e_ids]
|
||||
w2 = w2_full[e_ids]
|
||||
|
||||
w_ref1_l, qweight1_l, scales1_l = [], [], []
|
||||
for i in range(local_e):
|
||||
test_perm = torch.randperm(k)
|
||||
w_ref1, qweight1, scales1, _, _, _ = marlin_quantize(
|
||||
w1[i].transpose(1, 0),
|
||||
quant_type,
|
||||
group_size,
|
||||
False,
|
||||
test_perm,
|
||||
)
|
||||
w_ref1_l.append(w_ref1.T)
|
||||
qweight1_l.append(qweight1)
|
||||
scales1_l.append(scales1)
|
||||
|
||||
w_ref2_l, qweight2_l, scales2_l = [], [], []
|
||||
for i in range(local_e):
|
||||
test_perm = torch.randperm(n)
|
||||
w_ref2, qweight2, scales2, _, _, _ = marlin_quantize(
|
||||
w2[i].transpose(1, 0),
|
||||
quant_type,
|
||||
group_size,
|
||||
False,
|
||||
test_perm,
|
||||
)
|
||||
w_ref2_l.append(w_ref2.T)
|
||||
qweight2_l.append(qweight2)
|
||||
scales2_l.append(scales2)
|
||||
|
||||
w_ref1 = stack_and_dev(w_ref1_l)
|
||||
qweight1 = stack_and_dev(qweight1_l).contiguous()
|
||||
scales1 = stack_and_dev(scales1_l)
|
||||
|
||||
w_ref2 = stack_and_dev(w_ref2_l)
|
||||
qweight2 = stack_and_dev(qweight2_l).contiguous()
|
||||
scales2 = stack_and_dev(scales2_l)
|
||||
|
||||
from sglang.srt.layers.moe.topk import fused_topk_torch_native
|
||||
|
||||
topk_weights, topk_ids = fused_topk_torch_native(
|
||||
a, score, topk, False
|
||||
)
|
||||
|
||||
w_ref1_full = w1_full.clone()
|
||||
w_ref2_full = w2_full.clone()
|
||||
w_ref1_full[e_ids] = w_ref1
|
||||
w_ref2_full[e_ids] = w_ref2
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
w_ref1_full,
|
||||
w_ref2_full,
|
||||
score,
|
||||
topk,
|
||||
global_num_experts=e,
|
||||
expert_map=e_map,
|
||||
)
|
||||
|
||||
marlin_output = fused_marlin_moe(
|
||||
a,
|
||||
qweight1,
|
||||
qweight2,
|
||||
scales1,
|
||||
scales2,
|
||||
score,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
global_num_experts=e,
|
||||
expert_map=e_map,
|
||||
num_bits=4,
|
||||
is_k_full=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
marlin_output, torch_output, atol=5e-2, rtol=0
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
+1
-12
@@ -53,21 +53,10 @@ suites = {
|
||||
# "per-commit-8-gpu-h200-deepep": [
|
||||
# TestFile("ep/test_deepep_large.py", 563),
|
||||
# ],
|
||||
"quantization_test": [
|
||||
TestFile("quant/test_awq.py", 163),
|
||||
TestFile("quant/test_marlin_moe.py", 200),
|
||||
TestFile("test_bnb.py", 5),
|
||||
TestFile("test_gptqmodel_dynamic.py", 102),
|
||||
TestFile("test_quantization.py", 185),
|
||||
TestFile("test_gguf.py", 96),
|
||||
],
|
||||
# quantization_test suite migrated to test/registered/quant/
|
||||
"__not_in_ci__": [
|
||||
TestFile("test_release_memory_occupation.py", 200), # Temporarily disabled
|
||||
TestFile("models/test_dummy_grok_models.py"),
|
||||
TestFile("test_bench_one_batch.py"),
|
||||
TestFile("test_bench_serving.py"),
|
||||
TestFile("test_eval_accuracy_large.py"),
|
||||
TestFile("test_moe_eval_accuracy_large.py"),
|
||||
TestFile("test_profile_v2.py"),
|
||||
TestFile("models/test_ministral3_models.py"),
|
||||
TestFile("test_mistral_large3_basic.py"),
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_offline_throughput,
|
||||
run_bench_one_batch,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
# We use `run_bench_offline_throughput`` instead of `run_bench_one_batch` for most cases
|
||||
# because `run_bench_offline_throughput`` has overlap scheduler.
|
||||
|
||||
|
||||
class TestBenchOneBatch(CustomTestCase):
|
||||
|
||||
def test_bs1_small(self):
|
||||
_, output_throughput, _ = run_bench_one_batch(
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"]
|
||||
)
|
||||
self.assertGreater(output_throughput, 50)
|
||||
|
||||
def test_bs1_default(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"]
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs1_default (llama-3.1-8b)\n"
|
||||
f"output_throughput: {output_throughput:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(output_throughput, 135)
|
||||
|
||||
def test_moe_tp2_bs1(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"]
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n"
|
||||
f"output_throughput: {output_throughput:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(output_throughput, 85)
|
||||
else:
|
||||
self.assertGreater(output_throughput, 125)
|
||||
|
||||
def test_torch_compile_tp2_bs1(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n"
|
||||
f"output_throughput: {output_throughput:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(output_throughput, 200)
|
||||
else:
|
||||
self.assertGreater(output_throughput, 220)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,566 +0,0 @@
|
||||
import asyncio
|
||||
import itertools
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TARGET_MODEL_EAGLE,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_serving,
|
||||
run_embeddings_benchmark,
|
||||
run_score_benchmark,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
class TestBenchServing(CustomTestCase):
|
||||
def test_offline_throughput_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_default\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 3050)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 3800)
|
||||
|
||||
def test_offline_throughput_non_stream_small_batch_size(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=200,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--max-running-requests", "10"],
|
||||
dataset_name="sharegpt",
|
||||
random_input_len=None,
|
||||
random_output_len=None,
|
||||
disable_stream=True,
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_non_stream_small_batch_size\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 1000)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 1050)
|
||||
|
||||
def test_offline_throughput_without_radix_cache(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--disable-radix-cache"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_without_radix_cache\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 3050)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 3800)
|
||||
|
||||
def test_offline_throughput_without_chunked_prefill(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--chunked-prefill-size", "-1"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_without_chunked_prefill\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(res["output_throughput"], 2600)
|
||||
|
||||
def test_offline_throughput_with_triton_attention_backend(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--context-length",
|
||||
"8192",
|
||||
],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_with_triton_attention_backend\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 3500)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 3700)
|
||||
|
||||
def test_offline_throughput_default_fp8(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_default_fp8\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 3500)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 4300)
|
||||
|
||||
def test_online_latency_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=100,
|
||||
request_rate=1,
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_online_latency_default\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 11000)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 115)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 86)
|
||||
self.assertLess(res["median_itl_ms"], 10)
|
||||
|
||||
def test_vlm_offline_throughput(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=200,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
],
|
||||
dataset_name="mmmu",
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_vlm_offline_throughput\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 2000)
|
||||
# TODO: not set yet, need AMD machine
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 2500)
|
||||
|
||||
def test_vlm_online_latency(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=250,
|
||||
request_rate=1,
|
||||
other_server_args=[
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
],
|
||||
dataset_name="mmmu",
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_vlm_online_latency\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 16500)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 150)
|
||||
# TODO: not set yet, need AMD machine
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 100)
|
||||
self.assertLess(res["median_itl_ms"], 8)
|
||||
|
||||
def test_lora_online_latency(self):
|
||||
# TODO (lifuhuang): verify LoRA support in AMD.
|
||||
if is_in_amd_ci():
|
||||
pass
|
||||
|
||||
res = self._run_lora_latency_test(enable_background_task=False)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_lora_online_latency\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 2400)
|
||||
self.assertLess(res["median_ttft_ms"], 58)
|
||||
|
||||
def test_lora_online_latency_with_concurrent_adapter_updates(self):
|
||||
# TODO (lifuhuang): verify LoRA support in AMD.
|
||||
if is_in_amd_ci():
|
||||
pass
|
||||
|
||||
res = self._run_lora_latency_test(enable_background_task=True)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_lora_online_latency\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 4000)
|
||||
self.assertLess(res["median_ttft_ms"], 80)
|
||||
|
||||
def _run_lora_latency_test(self, enable_background_task: bool):
|
||||
"""
|
||||
Run a latency test for LoRA with the specified background task setting.
|
||||
"""
|
||||
|
||||
async def lora_loader_unloader_task(
|
||||
base_url: str,
|
||||
start_event: asyncio.Event,
|
||||
stop_event: asyncio.Event,
|
||||
):
|
||||
"""
|
||||
A background task that repeatedly loads and unloads a LoRA adapter.
|
||||
"""
|
||||
await start_event.wait()
|
||||
|
||||
path_cycler = itertools.cycle(
|
||||
[
|
||||
"pbevan11/llama-3.1-8b-ocr-correction",
|
||||
"faridlazuarda/valadapt-llama-3.1-8B-it-chinese",
|
||||
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
|
||||
]
|
||||
)
|
||||
load_url = f"{base_url}/load_lora_adapter"
|
||||
unload_url = f"{base_url}/unload_lora_adapter"
|
||||
num_updates = 0
|
||||
|
||||
while not stop_event.is_set():
|
||||
# 1. Load the LoRA adapter
|
||||
lora_path = next(path_cycler)
|
||||
response = await asyncio.to_thread(
|
||||
requests.post,
|
||||
load_url,
|
||||
json={"lora_name": lora_path, "lora_path": lora_path},
|
||||
)
|
||||
self.assertTrue(
|
||||
response.ok, f"Failed to load LoRA adapter: {response.text}"
|
||||
)
|
||||
num_updates += 1
|
||||
|
||||
if stop_event.is_set():
|
||||
break
|
||||
|
||||
# Yield control to allow other tasks to run.
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# 2. Unload the LoRA adapter
|
||||
response = await asyncio.to_thread(
|
||||
requests.post,
|
||||
unload_url,
|
||||
json={"lora_name": lora_path},
|
||||
)
|
||||
self.assertTrue(
|
||||
response.ok, f"Failed to unload LoRA adapter: {response.text}"
|
||||
)
|
||||
num_updates += 1
|
||||
|
||||
# Yield control to allow other tasks to run.
|
||||
await asyncio.sleep(1)
|
||||
|
||||
background_task = lora_loader_unloader_task if enable_background_task else None
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=400,
|
||||
request_rate=8,
|
||||
other_server_args=[
|
||||
"--enable-lora",
|
||||
"--max-loras-per-batch",
|
||||
"1",
|
||||
"--disable-radix-cache",
|
||||
"--random-seed",
|
||||
"42",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--lora-paths",
|
||||
"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16",
|
||||
"--max-lora-rank",
|
||||
"256",
|
||||
],
|
||||
dataset_name="random",
|
||||
random_input_len=256,
|
||||
random_output_len=256,
|
||||
lora_name=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"],
|
||||
background_task=background_task,
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
def test_online_latency_eagle(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_TARGET_MODEL_EAGLE,
|
||||
num_prompts=300,
|
||||
request_rate=8,
|
||||
sharegpt_context_len=3072,
|
||||
disable_ignore_eos=True,
|
||||
dataset_name="sharegpt",
|
||||
other_server_args=[
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||
"--speculative-num-steps",
|
||||
"5",
|
||||
"--speculative-eagle-topk",
|
||||
"4",
|
||||
"--speculative-num-draft-tokens",
|
||||
"16",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
],
|
||||
need_warmup=True,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_online_latency_eagle\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
f"accept_length: {res['accept_length']:.2f} \n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_e2e_latency_ms"], 1800)
|
||||
else:
|
||||
self.assertLess(res["median_e2e_latency_ms"], 900)
|
||||
self.assertGreater(res["accept_length"], 3.0)
|
||||
|
||||
def test_moe_offline_throughput_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=300,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--tp", "2"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_moe_offline_throughput_default\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 2100)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 2200)
|
||||
|
||||
def test_moe_offline_throughput_without_radix_cache(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=300,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--tp", "2", "--disable-radix-cache"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_moe_offline_throughput_without_radix_cache\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 2100)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 2200)
|
||||
|
||||
def test_pp_offline_throughput_default_decode(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=1000,
|
||||
request_rate=float("inf"),
|
||||
random_input_len=1,
|
||||
random_output_len=1024,
|
||||
other_server_args=["--pp-size", "2"],
|
||||
need_warmup=True,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_pp_offline_throughput_default_decode\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(res["output_throughput"], 6700)
|
||||
|
||||
def test_pp_long_context_prefill(self):
|
||||
res = run_bench_serving(
|
||||
model="meta-llama/Llama-3.3-70B-Instruct",
|
||||
num_prompts=4,
|
||||
request_rate=float("inf"),
|
||||
random_input_len=128000,
|
||||
random_output_len=1,
|
||||
dataset_name="random",
|
||||
other_server_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--pp-size",
|
||||
"2",
|
||||
]
|
||||
+ (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []),
|
||||
need_warmup=False,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_pp_long_context_latency_prefill\n"
|
||||
f"input_throughput: {res['input_throughput']:.2f} ms\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["input_throughput"], 3000)
|
||||
else:
|
||||
self.assertGreater(res["input_throughput"], 4000)
|
||||
|
||||
def test_score_api_latency_throughput(self):
|
||||
"""Test score API latency and throughput performance"""
|
||||
res = run_score_benchmark(
|
||||
model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
num_requests=1000,
|
||||
batch_size=10,
|
||||
other_server_args=[],
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_score_api_throughput\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Score API throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
self.assertLess(res["avg_latency_ms"], 48)
|
||||
self.assertLess(res["p95_latency_ms"], 50)
|
||||
self.assertGreater(res["throughput"], 20)
|
||||
|
||||
def test_score_api_batch_scaling(self):
|
||||
"""Test score API performance with different batch sizes"""
|
||||
batch_sizes = [10, 25, 50]
|
||||
|
||||
for batch_size in batch_sizes:
|
||||
res = run_score_benchmark(
|
||||
model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
num_requests=500,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_score_api_batch_scaling_size_{batch_size}\n"
|
||||
f"Batch size: {batch_size}\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
bounds = {
|
||||
10: (45, 50),
|
||||
25: (50, 60),
|
||||
50: (60, 65),
|
||||
}
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (60, 65))
|
||||
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
|
||||
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
|
||||
|
||||
def test_embeddings_api_latency_throughput(self):
|
||||
"""Test embeddings API latency and throughput performance"""
|
||||
res = run_embeddings_benchmark(
|
||||
model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
num_requests=1000,
|
||||
batch_size=1,
|
||||
input_tokens=500,
|
||||
other_server_args=[],
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_embeddings_api_throughput\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Embeddings API throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
# Bounds based on actual performance on 1xH100: avg=15ms, p95=15ms, throughput=67req/s
|
||||
self.assertLess(res["avg_latency_ms"], 20)
|
||||
self.assertLess(res["p95_latency_ms"], 25)
|
||||
self.assertGreater(res["throughput"], 60)
|
||||
|
||||
def test_embeddings_api_batch_scaling(self):
|
||||
"""Test embeddings API performance with different batch sizes"""
|
||||
batch_sizes = [10, 25, 50]
|
||||
|
||||
for batch_size in batch_sizes:
|
||||
res = run_embeddings_benchmark(
|
||||
model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
num_requests=500,
|
||||
batch_size=batch_size,
|
||||
input_tokens=500,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_embeddings_api_batch_scaling_size_{batch_size}\n"
|
||||
f"Batch size: {batch_size}\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
bounds = {
|
||||
10: (60, 65),
|
||||
25: (115, 120),
|
||||
50: (190, 195),
|
||||
}
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (250, 250))
|
||||
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
|
||||
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,307 +0,0 @@
|
||||
"""
|
||||
Usage:
|
||||
python3 -m unittest test_bnb.TestVisionModel.test_vlm
|
||||
python3 -m unittest test_bnb.TestLanguageModel.test_mmlu
|
||||
"""
|
||||
|
||||
import multiprocessing as mp
|
||||
import random
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
VISION_MODELS = [
|
||||
"unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit",
|
||||
"unsloth/Qwen2-VL-7B-Instruct-bnb-4bit",
|
||||
"unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",
|
||||
"unsloth/Llama-3.2-11B-Vision-bnb-4bit",
|
||||
"unsloth/gemma-3-4b-it-bnb-4bit",
|
||||
"unsloth/gemma-3-4b-it-unsloth-bnb-4bit",
|
||||
]
|
||||
LANGUAGE_MODELS = [
|
||||
"unsloth/Qwen2.5-7B-Instruct-bnb-4bit",
|
||||
"unsloth/Qwen2-7B-Instruct-bnb-4bit",
|
||||
"unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
|
||||
"unsloth/gemma-3-1b-it-bnb-4bit",
|
||||
]
|
||||
|
||||
# image
|
||||
IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png"
|
||||
IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png"
|
||||
|
||||
# video
|
||||
VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4"
|
||||
|
||||
# audio
|
||||
AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3"
|
||||
AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3"
|
||||
|
||||
|
||||
def popen_launch_server_wrapper(base_url, model, other_args):
|
||||
process = popen_launch_server(
|
||||
model,
|
||||
base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
)
|
||||
return process
|
||||
|
||||
|
||||
class TestVisionModel(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
mp.set_start_method("spawn", force=True)
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.base_url += "/v1"
|
||||
cls.api_key = "sk-123456"
|
||||
|
||||
def _run_single_image_chat_completion(self):
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="default",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_MAN_IRONING_URL},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image in a very short sentence.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
assert response.choices[0].message.role == "assistant"
|
||||
text = response.choices[0].message.content
|
||||
assert isinstance(text, str)
|
||||
# `driver` is for gemma-3-it
|
||||
assert "man" in text or "person" or "driver" in text, text
|
||||
assert "cab" in text or "taxi" in text or "SUV" in text, text
|
||||
# MiniCPMO fails to recognize `iron`, but `hanging`
|
||||
assert "iron" in text or "hang" in text, text
|
||||
assert response.id
|
||||
assert response.created
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.usage.total_tokens > 0
|
||||
|
||||
def _run_multi_turn_chat_completion(self):
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="default",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_MAN_IRONING_URL},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image in a very short sentence.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "There is a man at the back of a yellow cab ironing his clothes.",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Repeat your previous answer."}
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
assert response.choices[0].message.role == "assistant"
|
||||
text = response.choices[0].message.content
|
||||
assert isinstance(text, str)
|
||||
assert "man" in text or "cab" in text, text
|
||||
assert response.id
|
||||
assert response.created
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.usage.total_tokens > 0
|
||||
|
||||
def _run_multi_images_chat_completion(self):
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
response = client.chat.completions.create(
|
||||
model="default",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_MAN_IRONING_URL},
|
||||
"modalities": "multi-images",
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_SGL_LOGO_URL},
|
||||
"modalities": "multi-images",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I have two very different images. They are not related at all. "
|
||||
"Please describe the first image in one sentence, and then describe the second image in another sentence.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
assert response.choices[0].message.role == "assistant"
|
||||
text = response.choices[0].message.content
|
||||
assert isinstance(text, str)
|
||||
print("-" * 30)
|
||||
print(f"Multi images response:\n{text}")
|
||||
print("-" * 30)
|
||||
assert "man" in text or "cab" in text or "SUV" in text or "taxi" in text, text
|
||||
assert "logo" in text or '"S"' in text or "SG" in text, text
|
||||
assert response.id
|
||||
assert response.created
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.usage.total_tokens > 0
|
||||
|
||||
def run_decode_with_image(self, image_id):
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
content = []
|
||||
if image_id == 0:
|
||||
content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_MAN_IRONING_URL},
|
||||
}
|
||||
)
|
||||
elif image_id == 1:
|
||||
content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": IMAGE_SGL_LOGO_URL},
|
||||
}
|
||||
)
|
||||
else:
|
||||
pass
|
||||
|
||||
content.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image in a very short sentence.",
|
||||
}
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="default",
|
||||
messages=[
|
||||
{"role": "user", "content": content},
|
||||
],
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
assert response.choices[0].message.role == "assistant"
|
||||
text = response.choices[0].message.content
|
||||
assert isinstance(text, str)
|
||||
|
||||
def _run_test_mixed_batch(self):
|
||||
image_ids = [0, 1, 2] * 4
|
||||
with ThreadPoolExecutor(4) as executor:
|
||||
list(executor.map(self.run_decode_with_image, image_ids))
|
||||
|
||||
def test_vlm(self):
|
||||
models_to_test = VISION_MODELS
|
||||
|
||||
if is_in_ci():
|
||||
models_to_test = [random.choice(VISION_MODELS)]
|
||||
|
||||
for model in models_to_test:
|
||||
with self.subTest(model=model):
|
||||
other_args = [
|
||||
"--mem-fraction-static",
|
||||
"0.6",
|
||||
"--load-format",
|
||||
"bitsandbytes",
|
||||
"--enable-multimodal",
|
||||
]
|
||||
try:
|
||||
process = popen_launch_server_wrapper(
|
||||
DEFAULT_URL_FOR_TEST, model, other_args
|
||||
)
|
||||
self._run_test_mixed_batch()
|
||||
self._run_multi_images_chat_completion()
|
||||
self._run_multi_turn_chat_completion()
|
||||
self._run_single_image_chat_completion()
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
|
||||
class TestLanguageModel(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
mp.set_start_method("spawn", force=True)
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
# cls.base_url += "/v1"
|
||||
cls.api_key = "sk-123456"
|
||||
|
||||
def test_mmlu(self):
|
||||
models_to_test = LANGUAGE_MODELS
|
||||
|
||||
if is_in_ci():
|
||||
models_to_test = [random.choice(LANGUAGE_MODELS)]
|
||||
|
||||
for model in models_to_test:
|
||||
with self.subTest(model=model):
|
||||
other_args = [
|
||||
"--mem-fraction-static",
|
||||
"0.6",
|
||||
"--load-format",
|
||||
"bitsandbytes",
|
||||
]
|
||||
try:
|
||||
process = popen_launch_server_wrapper(
|
||||
DEFAULT_URL_FOR_TEST, model, other_args
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=model,
|
||||
eval_name="mmlu",
|
||||
num_examples=32,
|
||||
num_threads=16,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
self.assertGreater(metrics["score"], 0.3)
|
||||
finally:
|
||||
kill_process_tree(process.pid)
|
||||
@@ -1,92 +0,0 @@
|
||||
"""
|
||||
Usage:
|
||||
python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
class TestEvalAccuracyLarge(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--log-level-http", "warning"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_mmlu(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mmlu",
|
||||
num_examples=5000,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n')
|
||||
|
||||
self.assertGreater(metrics["score"], 0.70)
|
||||
|
||||
def test_human_eval(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="humaneval",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n'
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["score"], 0.64)
|
||||
|
||||
def test_mgsm_en(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mgsm_en",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n'
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["score"], 0.835)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,27 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
import sglang as sgl
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestGGUF(CustomTestCase):
|
||||
def test_models(self):
|
||||
prompt = "Today is a sunny day and I like"
|
||||
sampling_params = {"temperature": 0, "max_new_tokens": 8}
|
||||
|
||||
model_path = hf_hub_download(
|
||||
"Qwen/Qwen2-1.5B-Instruct-GGUF",
|
||||
filename="qwen2-1_5b-instruct-q4_k_m.gguf",
|
||||
)
|
||||
|
||||
engine = sgl.Engine(model_path=model_path, random_seed=42, cuda_graph_max_bs=2)
|
||||
outputs = engine.generate(prompt, sampling_params)["text"]
|
||||
engine.shutdown()
|
||||
|
||||
self.assertEqual(outputs, " it. I have a lot of work")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,204 +0,0 @@
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
import torch
|
||||
|
||||
from sglang.srt.server_args import set_global_server_args_for_scheduler
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
def check_quant_method(model_path: str, use_marlin_kernel: bool):
|
||||
from sglang.srt.configs.device_config import DeviceConfig
|
||||
from sglang.srt.configs.load_config import LoadConfig
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import monkey_patch_vllm_parallel_state
|
||||
from sglang.srt.layers.quantization.utils import get_dynamic_override
|
||||
from sglang.srt.model_loader import get_model
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
try:
|
||||
init_distributed_environment(
|
||||
backend="nccl",
|
||||
world_size=1,
|
||||
rank=0,
|
||||
local_rank=0,
|
||||
distributed_init_method="tcp://127.0.0.1:2646",
|
||||
)
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
monkey_patch_vllm_parallel_state()
|
||||
except AssertionError:
|
||||
# ignore this error: tensor model parallel group is already initialized
|
||||
pass
|
||||
|
||||
server_args = ServerArgs(model_path=model_path, dtype=torch.float16)
|
||||
set_global_server_args_for_scheduler(server_args)
|
||||
model_config = ModelConfig.from_server_args(server_args)
|
||||
|
||||
load_config = LoadConfig()
|
||||
device_config = DeviceConfig("cuda")
|
||||
model = get_model(
|
||||
model_config=model_config, load_config=load_config, device_config=device_config
|
||||
)
|
||||
|
||||
from sglang.srt.layers.linear import UnquantizedLinearMethod
|
||||
from sglang.srt.layers.quantization.gptq import (
|
||||
GPTQLinearMethod,
|
||||
GPTQMarlinLinearMethod,
|
||||
)
|
||||
|
||||
linear_method_cls = (
|
||||
GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod)
|
||||
)
|
||||
|
||||
for name, submodule in model.named_modules():
|
||||
if name == "lm_head":
|
||||
assert isinstance(submodule.quant_method, linear_method_cls)
|
||||
elif name == "model.layers.0.self_attn.qkv_proj":
|
||||
# The first layer is quantized using bits=4, group_size=128
|
||||
# desc_act=True
|
||||
assert isinstance(submodule.quant_method, linear_method_cls)
|
||||
config = submodule.quant_method.quant_config
|
||||
assert config.weight_bits == 4
|
||||
assert config.group_size == 128
|
||||
assert config.desc_act
|
||||
elif name == "model.layers.1.self_attn.qkv_proj":
|
||||
# The second layer is quantized using bits=8, group_size=32
|
||||
# desc_act=False
|
||||
assert isinstance(submodule.quant_method, linear_method_cls)
|
||||
config = submodule.quant_method.quant_config
|
||||
assert get_dynamic_override(config, layer_name=name, key="bits") == 8
|
||||
assert get_dynamic_override(config, layer_name=name, key="group_size") == 32
|
||||
assert not get_dynamic_override(config, layer_name=name, key="desc_act")
|
||||
elif (
|
||||
name == "model.layers.2.self_attn.qkv_proj"
|
||||
or name == "model.layers.2.mlp.gate_up_proj"
|
||||
):
|
||||
# All other layers (layer index >= 2) are not quantized
|
||||
assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
|
||||
|
||||
del model
|
||||
|
||||
|
||||
# GPTQ with Dynamic Per/Module Quantization Control
|
||||
# Leverages GPTQModel (pypi) to produce the `dynamic` models
|
||||
# Test GPTQ fallback kernel that is not Marlin
|
||||
class TestGPTQModelDynamic(CustomTestCase):
|
||||
MODEL_PATH = (
|
||||
"ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = cls.MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--dtype", "float16"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def run_decode(self, max_new_tokens):
|
||||
response = requests.post(
|
||||
self.base_url + "/generate",
|
||||
json={
|
||||
"text": "The capital of France is",
|
||||
"sampling_params": {
|
||||
"max_new_tokens": max_new_tokens,
|
||||
"temperature": 0.001,
|
||||
},
|
||||
},
|
||||
)
|
||||
return response.json()
|
||||
|
||||
def test_throughput(self):
|
||||
max_tokens = 256
|
||||
|
||||
tic = time.perf_counter()
|
||||
result = self.run_decode(max_tokens)
|
||||
tok = time.perf_counter()
|
||||
|
||||
print(f"result = `{result}`")
|
||||
|
||||
self.assertIn("paris", result["text"].lower())
|
||||
|
||||
throughput = max_tokens / (tok - tic)
|
||||
print(f"Throughput: {throughput} tokens/s")
|
||||
self.assertGreaterEqual(throughput, 140)
|
||||
|
||||
def test_gptq_module(self):
|
||||
check_quant_method(self.MODEL_PATH, use_marlin_kernel=False)
|
||||
|
||||
|
||||
# GPTQ with Dynamic Per/Module Quantization Control
|
||||
# Leverages GPTQModel (pypi) to produce the `dynamic` models
|
||||
# Test Marlin kernel
|
||||
class TestGPTQModelDynamicWithMarlin(CustomTestCase):
|
||||
MODEL_PATH = (
|
||||
"ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = cls.MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--dtype", "bfloat16"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def run_decode(self, max_new_tokens):
|
||||
response = requests.post(
|
||||
self.base_url + "/generate",
|
||||
json={
|
||||
"text": "The capital of France is",
|
||||
"sampling_params": {
|
||||
"max_new_tokens": max_new_tokens,
|
||||
"temperature": 0.001,
|
||||
},
|
||||
},
|
||||
)
|
||||
return response.json()
|
||||
|
||||
def test_throughput(self):
|
||||
max_tokens = 256
|
||||
|
||||
tic = time.perf_counter()
|
||||
result = self.run_decode(max_tokens)
|
||||
tok = time.perf_counter()
|
||||
|
||||
print(f"result = `{result}`")
|
||||
|
||||
assert "paris" in result["text"].lower()
|
||||
|
||||
throughput = max_tokens / (tok - tic)
|
||||
print(f"Throughput: {throughput} tokens/s")
|
||||
assert throughput >= 140
|
||||
|
||||
def test_gptq_marlin_module(self):
|
||||
check_quant_method(self.MODEL_PATH, use_marlin_kernel=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,94 +0,0 @@
|
||||
"""
|
||||
Usage:
|
||||
python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mmlu
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
|
||||
class TestMoEEvalAccuracyLarge(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=[
|
||||
"--log-level-http",
|
||||
"warning",
|
||||
"--tp",
|
||||
"2",
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_mmlu(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mmlu",
|
||||
num_examples=5000,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(metrics["score"], 0.62)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n')
|
||||
|
||||
def test_human_eval(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="humaneval",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(metrics["score"], 0.40)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n'
|
||||
)
|
||||
|
||||
def test_mgsm_en(self):
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mgsm_en",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
self.assertGreater(metrics["score"], 0.61)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n'
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,138 +0,0 @@
|
||||
import json
|
||||
import unittest
|
||||
import warnings
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
write_github_step_summary,
|
||||
write_results_to_json,
|
||||
)
|
||||
|
||||
MODEL_SCORE_THRESHOLDS = {
|
||||
"hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.825,
|
||||
"hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.825,
|
||||
"hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.615,
|
||||
}
|
||||
|
||||
|
||||
def parse_models(model_string):
|
||||
return [model.strip() for model in model_string.split(",") if model.strip()]
|
||||
|
||||
|
||||
def popen_launch_server_wrapper(base_url, model, is_fp8, is_tp2):
|
||||
other_args = ["--log-level-http", "warning", "--trust-remote-code"]
|
||||
if is_fp8:
|
||||
if "Llama-3" in model or "gemma-2" in model:
|
||||
other_args.extend(["--kv-cache-dtype", "fp8_e5m2"])
|
||||
elif "Qwen2-72B-Instruct-FP8" in model:
|
||||
other_args.extend(["--quantization", "fp8"])
|
||||
elif "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" in model:
|
||||
other_args.extend([])
|
||||
else:
|
||||
other_args.extend(["--quantization", "fp8", "--kv-cache-dtype", "fp8_e5m2"])
|
||||
if is_tp2:
|
||||
other_args.extend(["--tp", "2"])
|
||||
if "DeepSeek" in model:
|
||||
other_args.extend(["--mem-frac", "0.85"])
|
||||
|
||||
process = popen_launch_server(
|
||||
model,
|
||||
base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
)
|
||||
return process
|
||||
|
||||
|
||||
def check_model_scores(results):
|
||||
failed_models = []
|
||||
summary = " | model | score | threshold |\n"
|
||||
summary += "| ----- | ----- | --------- |\n"
|
||||
|
||||
for model, score in results:
|
||||
threshold = MODEL_SCORE_THRESHOLDS.get(model)
|
||||
if threshold is None:
|
||||
print(f"Warning: No threshold defined for model {model}")
|
||||
continue
|
||||
|
||||
if score < threshold:
|
||||
failed_models.append(
|
||||
f"\nScore Check Failed: {model}\n"
|
||||
f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})"
|
||||
)
|
||||
|
||||
line = f"| {model} | {score} | {threshold} |\n"
|
||||
summary += line
|
||||
|
||||
print(summary)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### TestNightlyGsm8KEval for awq, gptq, gguf\n{summary}"
|
||||
)
|
||||
|
||||
if failed_models:
|
||||
raise AssertionError("\n".join(failed_models))
|
||||
|
||||
|
||||
class TestNightlyGsm8KEval(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model_groups = [
|
||||
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1), False, False),
|
||||
]
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
def test_mgsm_en_all_models(self):
|
||||
warnings.filterwarnings(
|
||||
"ignore", category=ResourceWarning, message="unclosed.*socket"
|
||||
)
|
||||
is_first = True
|
||||
all_results = []
|
||||
|
||||
for model_group, is_fp8, is_tp2 in self.model_groups:
|
||||
for model in model_group:
|
||||
with self.subTest(model=model):
|
||||
process = popen_launch_server_wrapper(
|
||||
self.base_url, model, is_fp8, is_tp2
|
||||
)
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=model,
|
||||
eval_name="mgsm_en",
|
||||
num_examples=None,
|
||||
num_threads=1024,
|
||||
)
|
||||
|
||||
metrics = run_eval(args)
|
||||
print(
|
||||
f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
|
||||
)
|
||||
|
||||
write_results_to_json(model, metrics, "w" if is_first else "a")
|
||||
is_first = False
|
||||
|
||||
all_results.append((model, metrics["score"]))
|
||||
kill_process_tree(process.pid)
|
||||
|
||||
try:
|
||||
with open("results.json", "r") as f:
|
||||
print("\nFinal Results from results.json:")
|
||||
print(json.dumps(json.load(f), indent=2))
|
||||
except Exception as e:
|
||||
print(f"Error reading results.json: {e}")
|
||||
|
||||
# Check all scores after collecting all results
|
||||
check_model_scores(all_results)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user