Apply fixture-kit mode to MMMUVLMMixin (#15615)
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase
|
||||
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
|
||||
|
||||
MODEL = "mistralai/Ministral-3-3B-Instruct-2512"
|
||||
|
||||
@@ -14,18 +14,13 @@ class TestMinistral3TextOnly(GSM8KMixin, DefaultServerBase):
|
||||
other_args = ["--trust-remote-code"]
|
||||
|
||||
|
||||
class TestMinistral3MMMU(MMMUVLMTestBase):
|
||||
class TestMinistral3MMMU(MMMUMixin, MMMUServerBase):
|
||||
accuracy = 0.3
|
||||
model = MODEL
|
||||
other_args = ["--trust-remote-code"]
|
||||
mmmu_args = ["--limit=0.1"]
|
||||
"""`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions."""
|
||||
|
||||
def test_vlm_mmmu_benchmark(self):
|
||||
self._run_vlm_mmmu_test(
|
||||
SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.kits.mmmu_vlm_kit import MMMUVLMTestBase
|
||||
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
|
||||
|
||||
MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16"
|
||||
|
||||
@@ -14,18 +14,13 @@ class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, DefaultServerBase):
|
||||
other_args = ["--max-mamba-cache-size", "256", "--trust-remote-code"]
|
||||
|
||||
|
||||
class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMTestBase):
|
||||
class TestNvidiaNemotronNanoV2VLMMMU(MMMUMixin, MMMUServerBase):
|
||||
accuracy = 0.444
|
||||
model = MODEL
|
||||
other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"]
|
||||
mmmu_args = ["--limit=0.1"]
|
||||
"""`--limit=0.1`: 10 percent of each task - this is fine for testing since the nominal result isn't interesting - this run is just to prevent relative regressions."""
|
||||
|
||||
def test_vlm_mmmu_benchmark(self):
|
||||
self._run_vlm_mmmu_test(
|
||||
SimpleNamespace(model=self.model, mmmu_accuracy=self.accuracy), "./logs"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -5,7 +5,10 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.test.kits.mmmu_vlm_kit import DEFAULT_MEM_FRACTION_STATIC, MMMUVLMTestBase
|
||||
from sglang.test.kits.mmmu_vlm_kit import (
|
||||
DEFAULT_MEM_FRACTION_STATIC,
|
||||
MMMUMultiModelTestBase,
|
||||
)
|
||||
from sglang.test.test_utils import is_in_ci
|
||||
|
||||
_is_hip = is_hip()
|
||||
@@ -20,7 +23,7 @@ else:
|
||||
]
|
||||
|
||||
|
||||
class TestVLMModels(MMMUVLMTestBase):
|
||||
class TestVLMModels(MMMUMultiModelTestBase):
|
||||
def test_vlm_mmmu_benchmark(self):
|
||||
"""Test VLM models against MMMU benchmark."""
|
||||
models_to_test = MODELS
|
||||
|
||||
Reference in New Issue
Block a user