From beae3f961cd517a03ca92ccbca1a030d4f549026 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Mon, 22 Dec 2025 14:19:35 +0800 Subject: [PATCH] Adapt fixture-kit to gsm8k mixin (#15599) --- python/sglang/test/gsm8k_mixin.py | 44 ------------ python/sglang/test/kits/gsm8k_accuracy_kit.py | 22 ++++++ .../mmmu_vlm_kit.py} | 5 +- .../test/server_fixtures/default_fixture.py | 67 +++++++++++++++++++ test/srt/models/test_ministral3_models.py | 10 +-- .../models/test_nvidia_nemotron_nano_v2.py | 16 +++-- .../models/test_nvidia_nemotron_nano_v2_vl.py | 10 +-- test/srt/models/test_vlm_models.py | 6 +- 8 files changed, 114 insertions(+), 66 deletions(-) delete mode 100644 python/sglang/test/gsm8k_mixin.py create mode 100644 python/sglang/test/kits/gsm8k_accuracy_kit.py rename python/sglang/test/{mmmu_vlm_mixin.py => kits/mmmu_vlm_kit.py} (98%) create mode 100644 python/sglang/test/server_fixtures/default_fixture.py diff --git a/python/sglang/test/gsm8k_mixin.py b/python/sglang/test/gsm8k_mixin.py deleted file mode 100644 index dc09d8a55..000000000 --- a/python/sglang/test/gsm8k_mixin.py +++ /dev/null @@ -1,44 +0,0 @@ -from abc import ABC -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - popen_launch_server, -) - - -class GSM8KMixin(ABC): - accuracy: float - model: str - other_args: list[str] = [] - - @classmethod - def setUpClass(cls): - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=cls.other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], self.accuracy) diff --git a/python/sglang/test/kits/gsm8k_accuracy_kit.py b/python/sglang/test/kits/gsm8k_accuracy_kit.py new file mode 100644 index 000000000..55099450d --- /dev/null +++ b/python/sglang/test/kits/gsm8k_accuracy_kit.py @@ -0,0 +1,22 @@ +from types import SimpleNamespace + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k +from sglang.test.test_utils import CustomTestCase + + +class GSM8KMixin: + accuracy: float + + def test_gsm8k(self: CustomTestCase): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_gsm8k(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["accuracy"], self.accuracy) diff --git a/python/sglang/test/mmmu_vlm_mixin.py b/python/sglang/test/kits/mmmu_vlm_kit.py similarity index 98% rename from python/sglang/test/mmmu_vlm_mixin.py rename to python/sglang/test/kits/mmmu_vlm_kit.py index c4c739f15..9f890e421 100644 --- a/python/sglang/test/mmmu_vlm_mixin.py +++ b/python/sglang/test/kits/mmmu_vlm_kit.py @@ -2,13 +2,13 @@ import glob import json import os import subprocess -from abc import ABC from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, + CustomTestCase, popen_launch_server, ) @@ -16,7 +16,8 @@ from sglang.test.test_utils import ( DEFAULT_MEM_FRACTION_STATIC = 0.8 -class MMMUVLMMixin(ABC): +class MMMUVLMTestBase(CustomTestCase): + # TODO: split the MMMUVLMTestBase into a fixture and a mixin parsed_args = None # Class variable to store args other_args = [] mmmu_args = [] diff --git a/python/sglang/test/server_fixtures/default_fixture.py b/python/sglang/test/server_fixtures/default_fixture.py new file mode 100644 index 000000000..f5f355046 --- /dev/null +++ b/python/sglang/test/server_fixtures/default_fixture.py @@ -0,0 +1,67 @@ +import logging +import time +from contextlib import contextmanager + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +logger = logging.getLogger(__name__) + + +@contextmanager +def openai_api_env(api_key: str): + """Context manager to set OpenAI API environment variables.""" + import os + + original_api_key = os.environ.get("OPENAI_API_KEY") + original_base_url = os.environ.get("OPENAI_API_BASE") + + os.environ["OPENAI_API_KEY"] = api_key + os.environ["OPENAI_API_BASE"] = f"{DEFAULT_URL_FOR_TEST}/v1" + + try: + yield + finally: + if original_api_key is not None: + os.environ["OPENAI_API_KEY"] = original_api_key + else: + del os.environ["OPENAI_API_KEY"] + + if original_base_url is not None: + os.environ["OPENAI_API_BASE"] = original_base_url + else: + del os.environ["OPENAI_API_BASE"] + + +class DefaultServerBase(CustomTestCase): + model = None + base_url = DEFAULT_URL_FOR_TEST + timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + other_args: list[str] = [] + + # For OpenAI API settings + api_key = "sk-123456" + + @classmethod + def setUpClass(cls): + assert cls.model is not None, "Please set cls.model in subclass" + + # Set OpenAI API key and base URL environment variables. + # Needed for lmm-evals to work. + with openai_api_env(cls.api_key): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=cls.timeout, + other_args=cls.other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + time.sleep(2) diff --git a/test/srt/models/test_ministral3_models.py b/test/srt/models/test_ministral3_models.py index 6f6ca51b4..cd12594ea 100644 --- a/test/srt/models/test_ministral3_models.py +++ b/test/srt/models/test_ministral3_models.py @@ -1,20 +1,20 @@ import unittest from types import SimpleNamespace -from sglang.test.gsm8k_mixin import GSM8KMixin -from sglang.test.mmmu_vlm_mixin import MMMUVLMMixin -from sglang.test.test_utils import CustomTestCase +from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase +from sglang.test.server_fixtures.default_fixture import DefaultServerBase MODEL = "mistralai/Ministral-3-3B-Instruct-2512" -class TestMinistral3TextOnly(GSM8KMixin, CustomTestCase): +class TestMinistral3TextOnly(GSM8KMixin, DefaultServerBase): accuracy = 0.6 model = MODEL other_args = ["--trust-remote-code"] -class TestMinistral3MMMU(MMMUVLMMixin, CustomTestCase): +class TestMinistral3MMMU(MMMUVLMTestBase): accuracy = 0.3 model = MODEL other_args = ["--trust-remote-code"] diff --git a/test/srt/models/test_nvidia_nemotron_nano_v2.py b/test/srt/models/test_nvidia_nemotron_nano_v2.py index ab8c2abdc..274e077fe 100644 --- a/test/srt/models/test_nvidia_nemotron_nano_v2.py +++ b/test/srt/models/test_nvidia_nemotron_nano_v2.py @@ -1,24 +1,24 @@ import unittest from sglang.srt.utils import is_blackwell -from sglang.test.gsm8k_mixin import GSM8KMixin -from sglang.test.test_utils import CustomTestCase +from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase -class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, DefaultServerBase): model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" accuracy = 0.87 other_args = ["--max-mamba-cache-size", "256"] -class TestNvidiaNemotronNanoV2FP8(GSM8KMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2FP8(GSM8KMixin, DefaultServerBase): accuracy = 0.87 model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2-FP8" other_args = ["--max-mamba-cache-size", "256"] @unittest.skipIf(not is_blackwell(), "NVFP4 only supported on blackwell") -class TestNvidiaNemotronNanoV2NVFP4(GSM8KMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2NVFP4(GSM8KMixin, DefaultServerBase): accuracy = 0.855 model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2-NVFP4" other_args = ["--max-mamba-cache-size", "256"] @@ -28,7 +28,7 @@ class TestNvidiaNemotronNanoV2NVFP4(GSM8KMixin, CustomTestCase): "STANDALONE speculative decoding does not yet support target and draft models " "with different hidden sizes (Nemotron-9B: 4480, Llama-3.2-1B: 2048)" ) -class TestNvidiaNemotronNanoV2SpeculativeDecoding(GSM8KMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2SpeculativeDecoding(GSM8KMixin, DefaultServerBase): accuracy = 0.87 model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" other_args = [ @@ -57,7 +57,9 @@ class TestNvidiaNemotronNanoV2SpeculativeDecoding(GSM8KMixin, CustomTestCase): "STANDALONE speculative decoding does not yet support target and draft models " "with different hidden sizes (Nemotron-9B: 4480, Llama-3.2-1B: 2048)" ) -class TestNvidiaNemotronNanoV2SpeculativeDecodingBF16Cache(GSM8KMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2SpeculativeDecodingBF16Cache( + GSM8KMixin, DefaultServerBase +): accuracy = 0.87 model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" other_args = [ diff --git a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py b/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py index a36218410..6fbb0e276 100644 --- a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py +++ b/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py @@ -1,20 +1,20 @@ import unittest from types import SimpleNamespace -from sglang.test.gsm8k_mixin import GSM8KMixin -from sglang.test.mmmu_vlm_mixin import MMMUVLMMixin -from sglang.test.test_utils import CustomTestCase +from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase +from sglang.test.server_fixtures.default_fixture import DefaultServerBase MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16" -class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, DefaultServerBase): accuracy = 0.87 model = MODEL other_args = ["--max-mamba-cache-size", "256", "--trust-remote-code"] -class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMMixin, CustomTestCase): +class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMTestBase): accuracy = 0.454 model = MODEL other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"] diff --git a/test/srt/models/test_vlm_models.py b/test/srt/models/test_vlm_models.py index 3650621a5..c335f7f20 100644 --- a/test/srt/models/test_vlm_models.py +++ b/test/srt/models/test_vlm_models.py @@ -5,8 +5,8 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import is_hip -from sglang.test.mmmu_vlm_mixin import DEFAULT_MEM_FRACTION_STATIC, MMMUVLMMixin -from sglang.test.test_utils import CustomTestCase, is_in_ci +from sglang.test.kits.mmmu_vlm_kit import DEFAULT_MEM_FRACTION_STATIC, MMMUVLMTestBase +from sglang.test.test_utils import is_in_ci _is_hip = is_hip() # VLM models for testing @@ -20,7 +20,7 @@ else: ] -class TestVLMModels(MMMUVLMMixin, CustomTestCase): +class TestVLMModels(MMMUVLMTestBase): def test_vlm_mmmu_benchmark(self): """Test VLM models against MMMU benchmark.""" models_to_test = MODELS