diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 870af8b6c..eb2868bb1 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -6,6 +6,29 @@ from enum import IntEnum from typing import Any +@contextmanager +def temp_set_env(**env_vars: dict[str, Any]): + """Temporarily set non-sglang environment variables, e.g. OPENAI_API_KEY""" + for key in env_vars: + if key.startswith("SGLANG_") or key.startswith("SGL_"): + raise ValueError("temp_set_env should not be used for sglang env vars") + + backup = {key: os.environ.get(key) for key in env_vars} + try: + for key, value in env_vars.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = str(value) + yield + finally: + for key, value in backup.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + class EnvField: _allow_set_name = True diff --git a/python/sglang/test/kits/mmmu_vlm_kit.py b/python/sglang/test/kits/mmmu_vlm_kit.py index 9f890e421..b7415b0d4 100644 --- a/python/sglang/test/kits/mmmu_vlm_kit.py +++ b/python/sglang/test/kits/mmmu_vlm_kit.py @@ -2,8 +2,10 @@ import glob import json import os import subprocess +import tempfile from types import SimpleNamespace +from sglang.srt.environ import temp_set_env from sglang.srt.utils import kill_process_tree from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -16,8 +18,116 @@ from sglang.test.test_utils import ( DEFAULT_MEM_FRACTION_STATIC = 0.8 -class MMMUVLMTestBase(CustomTestCase): - # TODO: split the MMMUVLMTestBase into a fixture and a mixin +class MMMUMixin: + """Mixin for MMMU evaluation. + + Use with MMMUServerBase for single-model tests: + class TestMyModel(MMMUMixin, MMMUServerBase): + model = "my/model" + accuracy = 0.4 + """ + + accuracy: float + mmmu_args: list[str] = [] + + # For OpenAI API settings + api_key = "sk-123456" + + def run_mmmu_eval( + self: CustomTestCase, + model_version: str, + output_path: str, + ): + """ + Evaluate a VLM on the MMMU validation set with lmms-eval. + Only `model_version` (checkpoint) and `chat_template` vary; + We are focusing only on the validation set due to resource constraints. + """ + # -------- fixed settings -------- + model = "openai_compatible" + tp = 1 + tasks = "mmmu_val" + batch_size = 64 + log_suffix = "openai_compatible" + os.makedirs(output_path, exist_ok=True) + + # -------- compose --model_args -------- + model_args = f'model_version="{model_version}",' f"tp={tp}" + + # -------- build command list -------- + cmd = [ + "python3", + "-m", + "lmms_eval", + "--model", + model, + "--model_args", + model_args, + "--tasks", + tasks, + "--batch_size", + str(batch_size), + "--log_samples", + "--log_samples_suffix", + log_suffix, + "--output_path", + str(output_path), + *self.mmmu_args, + ] + + # Set OpenAI API key and base URL environment variables. + # Needed for lmms-eval to work. + with temp_set_env( + OPENAI_API_KEY=self.api_key, + OPENAI_API_BASE=f"{self.base_url}/v1", + ): + subprocess.run( + cmd, + check=True, + timeout=3600, + ) + + def test_mmmu(self: CustomTestCase): + """Run MMMU evaluation test.""" + with tempfile.TemporaryDirectory() as output_path: + # Run evaluation + self.run_mmmu_eval(self.model, output_path) + + # Get the result file + # Search recursively for JSON result files (lmms-eval v0.4.1+ creates subdirectories) + result_files = glob.glob(f"{output_path}/**/*.json", recursive=True) + if not result_files: + result_files = glob.glob(f"{output_path}/*.json") + + if not result_files: + raise FileNotFoundError(f"No JSON result files found in {output_path}") + + result_file_path = result_files[0] + + with open(result_file_path, "r") as f: + result = json.load(f) + print(f"Result: {result}") + + # Process the result + mmmu_accuracy = result["results"]["mmmu_val"]["mmmu_acc,none"] + print(f"Model {self.model} achieved accuracy: {mmmu_accuracy:.4f}") + + # Assert performance meets expected threshold + self.assertGreaterEqual( + mmmu_accuracy, + self.accuracy, + f"Model {self.model} accuracy ({mmmu_accuracy:.4f}) below expected threshold ({self.accuracy:.4f})", + ) + + +class MMMUMultiModelTestBase(CustomTestCase): + """Base class for multi-model MMMU tests. + + This class is for tests that need to evaluate multiple models, + starting and stopping a server for each model within the test method. + For single-model tests, use MMMUMixin with MMMUServerBase instead. + """ + parsed_args = None # Class variable to store args other_args = [] mmmu_args = [] @@ -34,10 +144,27 @@ class MMMUVLMTestBase(CustomTestCase): mem_fraction_static=DEFAULT_MEM_FRACTION_STATIC ) + # Save original environment variables for restoration in tearDownClass + cls._original_openai_api_key = os.environ.get("OPENAI_API_KEY") + cls._original_openai_api_base = os.environ.get("OPENAI_API_BASE") + # Set OpenAI API key and base URL environment variables. Needed for lmm-evals to work. os.environ["OPENAI_API_KEY"] = cls.api_key os.environ["OPENAI_API_BASE"] = f"{cls.base_url}/v1" + @classmethod + def tearDownClass(cls): + # Restore original environment variables + if cls._original_openai_api_key is not None: + os.environ["OPENAI_API_KEY"] = cls._original_openai_api_key + elif "OPENAI_API_KEY" in os.environ: + del os.environ["OPENAI_API_KEY"] + + if cls._original_openai_api_base is not None: + os.environ["OPENAI_API_BASE"] = cls._original_openai_api_base + elif "OPENAI_API_BASE" in os.environ: + del os.environ["OPENAI_API_BASE"] + def run_mmmu_eval( self, model_version: str, @@ -46,7 +173,7 @@ class MMMUVLMTestBase(CustomTestCase): env: dict | None = None, ): """ - Evaluate a VLM on the MMMU validation set with lmms‑eval. + Evaluate a VLM on the MMMU validation set with lmms-eval. Only `model_version` (checkpoint) and `chat_template` vary; We are focusing only on the validation set due to resource constraints. """ @@ -232,3 +359,7 @@ class MMMUVLMTestBase(CustomTestCase): print(f"Error reading {tag.lower()} file: {e}") return "\n".join(output_lines) + + +# Backward compatibility alias +MMMUVLMTestBase = MMMUMultiModelTestBase diff --git a/python/sglang/test/server_fixtures/mmmu_fixture.py b/python/sglang/test/server_fixtures/mmmu_fixture.py new file mode 100644 index 000000000..097dbff76 --- /dev/null +++ b/python/sglang/test/server_fixtures/mmmu_fixture.py @@ -0,0 +1,68 @@ +import logging +import os +import time + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +logger = logging.getLogger(__name__) + +# Set default mem_fraction_static to 0.8 +DEFAULT_MEM_FRACTION_STATIC = 0.8 + + +class MMMUServerBase(CustomTestCase): + """Server fixture for MMMU VLM tests. + + This fixture handles server lifecycle for single-model MMMU tests. + For multi-model tests that need to start/stop servers within test methods, + use MMMUMultiModelTestBase instead. + """ + + model = None + base_url = DEFAULT_URL_FOR_TEST + timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + other_args: list[str] = [] + mem_fraction_static: float = DEFAULT_MEM_FRACTION_STATIC + + @classmethod + def setUpClass(cls): + assert cls.model is not None, "Please set cls.model in subclass" + + # Prepare environment variables + process_env = os.environ.copy() + process_env["SGLANG_USE_CUDA_IPC_TRANSPORT"] = "1" + + # Build server args with MMMU-specific settings + server_args = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "64", + "--enable-multimodal", + "--mem-fraction-static", + str(cls.mem_fraction_static), + *cls.other_args, + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=cls.timeout, + api_key=cls.api_key, + other_args=server_args, + env=process_env, + ) + + @classmethod + def tearDownClass(cls): + if cls.process is not None and cls.process.poll() is None: + try: + kill_process_tree(cls.process.pid) + except Exception as e: + logger.error(f"Error killing process: {e}") + time.sleep(2) diff --git a/test/srt/models/test_ministral3_models.py b/test/srt/models/test_ministral3_models.py index 96626545c..b7715855f 100644 --- a/test/srt/models/test_ministral3_models.py +++ b/test/srt/models/test_ministral3_models.py @@ -1,9 +1,9 @@ import unittest -from types import SimpleNamespace from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin -from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase +from sglang.test.kits.mmmu_vlm_kit import MMMUMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase MODEL = "mistralai/Ministral-3-3B-Instruct-2512" @@ -14,18 +14,13 @@ class TestMinistral3TextOnly(GSM8KMixin, DefaultServerBase): other_args = ["--trust-remote-code"] -class TestMinistral3MMMU(MMMUVLMTestBase): +class TestMinistral3MMMU(MMMUMixin, MMMUServerBase): accuracy = 0.3 model = MODEL other_args = ["--trust-remote-code"] mmmu_args = ["--limit=0.1"] """`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions.""" - def test_vlm_mmmu_benchmark(self): - self._run_vlm_mmmu_test( - SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs" - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py b/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py index 9ba105a15..e41aedd67 100644 --- a/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py +++ b/test/srt/models/test_nvidia_nemotron_nano_v2_vl.py @@ -1,9 +1,9 @@ import unittest -from types import SimpleNamespace from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin -from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase +from sglang.test.kits.mmmu_vlm_kit import MMMUMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16" @@ -14,18 +14,13 @@ class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, DefaultServerBase): other_args = ["--max-mamba-cache-size", "256", "--trust-remote-code"] -class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMTestBase): +class TestNvidiaNemotronNanoV2VLMMMU(MMMUMixin, MMMUServerBase): accuracy = 0.444 model = MODEL other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"] mmmu_args = ["--limit=0.1"] """`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions.""" - def test_vlm_mmmu_benchmark(self): - self._run_vlm_mmmu_test( - SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs" - ) - if __name__ == "__main__": unittest.main() diff --git a/test/srt/models/test_vlm_models.py b/test/srt/models/test_vlm_models.py index c335f7f20..c195bbd81 100644 --- a/test/srt/models/test_vlm_models.py +++ b/test/srt/models/test_vlm_models.py @@ -5,7 +5,10 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import is_hip -from sglang.test.kits.mmmu_vlm_kit import DEFAULT_MEM_FRACTION_STATIC, MMMUVLMTestBase +from sglang.test.kits.mmmu_vlm_kit import ( + DEFAULT_MEM_FRACTION_STATIC, + MMMUMultiModelTestBase, +) from sglang.test.test_utils import is_in_ci _is_hip = is_hip() @@ -20,7 +23,7 @@ else: ] -class TestVLMModels(MMMUVLMTestBase): +class TestVLMModels(MMMUMultiModelTestBase): def test_vlm_mmmu_benchmark(self): """Test VLM models against MMMU benchmark.""" models_to_test = MODELS