Apply fixture-kit mode to MMMUVLMMixin (#15615)

This commit is contained in:
lif
2025-12-28 17:22:05 +08:00
committed by GitHub
parent 8fab48952e
commit 5969be2f06
6 changed files with 236 additions and 21 deletions
+3 -8
View File
@@ -1,9 +1,9 @@
import unittest
from types import SimpleNamespace
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
MODEL = "mistralai/Ministral-3-3B-Instruct-2512"
@@ -14,18 +14,13 @@ class TestMinistral3TextOnly(GSM8KMixin, DefaultServerBase):
other_args = ["--trust-remote-code"]
class TestMinistral3MMMU(MMMUVLMTestBase):
class TestMinistral3MMMU(MMMUMixin, MMMUServerBase):
accuracy = 0.3
model = MODEL
other_args = ["--trust-remote-code"]
mmmu_args = ["--limit=0.1"]
"""`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions."""
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test(
SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs"
)
if __name__ == "__main__":
unittest.main()
@@ -1,9 +1,9 @@
import unittest
from types import SimpleNamespace
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16"
@@ -14,18 +14,13 @@ class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, DefaultServerBase):
other_args = ["--max-mamba-cache-size", "256", "--trust-remote-code"]
class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMTestBase):
class TestNvidiaNemotronNanoV2VLMMMU(MMMUMixin, MMMUServerBase):
accuracy = 0.444
model = MODEL
other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"]
mmmu_args = ["--limit=0.1"]
"""`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions."""
def test_vlm_mmmu_benchmark(self):
self._run_vlm_mmmu_test(
SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs"
)
if __name__ == "__main__":
unittest.main()
+5 -2
View File
@@ -5,7 +5,10 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import is_hip
from sglang.test.kits.mmmu_vlm_kit import DEFAULT_MEM_FRACTION_STATIC, MMMUVLMTestBase
from sglang.test.kits.mmmu_vlm_kit import (
DEFAULT_MEM_FRACTION_STATIC,
MMMUMultiModelTestBase,
)
from sglang.test.test_utils import is_in_ci
_is_hip = is_hip()
@@ -20,7 +23,7 @@ else:
]
class TestVLMModels(MMMUVLMTestBase):
class TestVLMModels(MMMUMultiModelTestBase):
def test_vlm_mmmu_benchmark(self):
"""Test VLM models against MMMU benchmark."""
models_to_test = MODELS