From c1282da236062b774a27de269914ab6351e2c222 Mon Sep 17 00:00:00 2001 From: Hudson Xing <1277646412@qq.com> Date: Tue, 20 Jan 2026 02:00:29 +0800 Subject: [PATCH] fix(ci): apply MMMU retry logic to all affected test files (#17329) --- python/sglang/test/kits/mmmu_vlm_kit.py | 46 ++++++++++++++++--- .../nightly/test_vlms_piecewise_cuda_graph.py | 8 +--- .../nightly/test_vlms_vit_cuda_graph.py | 8 +--- test/registered/vlm/test_encoder_dp.py | 8 +--- test/srt/test_epd_disaggregation.py | 6 +-- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/python/sglang/test/kits/mmmu_vlm_kit.py b/python/sglang/test/kits/mmmu_vlm_kit.py index a1ca28fed..cb4e5fcea 100644 --- a/python/sglang/test/kits/mmmu_vlm_kit.py +++ b/python/sglang/test/kits/mmmu_vlm_kit.py @@ -64,11 +64,38 @@ def _run_lmms_eval_with_retry(cmd: list[str], timeout: int = 3600) -> None: capture_output=True, text=True, ) - # Print captured output to maintain visibility of successful runs - if result.stdout: - print(result.stdout, end="") - if result.stderr: - print(result.stderr, end="") + # Check for errors in output even if exit code is 0 + # lmms_eval sometimes returns 0 even when errors occur + combined_output = result.stdout + result.stderr + if _is_mmmu_parquet_corruption(combined_output): + print( + "Detected MMMU parquet corruption error in output. Attempting recovery..." + ) + if _cleanup_mmmu_dataset_cache(): + print("Retrying lmms_eval with fresh download...") + with temp_set_env( + HF_HUB_OFFLINE="0", + HF_DATASETS_DOWNLOAD_MODE="force_redownload", + ): + retry_result = subprocess.run( + cmd, check=True, timeout=timeout, capture_output=True, text=True + ) + # Print retry output + if retry_result.stdout: + print(retry_result.stdout, end="") + if retry_result.stderr: + print(retry_result.stderr, end="") + else: + print( + f"Failed to cleanup corrupted MMMU cache. Output from lmms_eval:\nStdout:\n{result.stdout}\nStderr:\n{result.stderr}" + ) + raise RuntimeError("Failed to cleanup corrupted MMMU cache") + else: + # Print captured output to maintain visibility of successful runs + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="") except subprocess.CalledProcessError as e: error_output = e.stderr + e.stdout if _is_mmmu_parquet_corruption(error_output): @@ -79,7 +106,14 @@ def _run_lmms_eval_with_retry(cmd: list[str], timeout: int = 3600) -> None: HF_HUB_OFFLINE="0", HF_DATASETS_DOWNLOAD_MODE="force_redownload", ): - subprocess.run(cmd, check=True, timeout=timeout) + retry_result = subprocess.run( + cmd, check=True, timeout=timeout, capture_output=True, text=True + ) + # Print retry output + if retry_result.stdout: + print(retry_result.stdout, end="") + if retry_result.stderr: + print(retry_result.stderr, end="") else: print( f"Failed to cleanup corrupted MMMU cache. Error from lmms_eval:\nStdout:\n{e.stdout}\nStderr:\n{e.stderr}" diff --git a/test/manual/nightly/test_vlms_piecewise_cuda_graph.py b/test/manual/nightly/test_vlms_piecewise_cuda_graph.py index 0001b917a..7a72dd3fa 100644 --- a/test/manual/nightly/test_vlms_piecewise_cuda_graph.py +++ b/test/manual/nightly/test_vlms_piecewise_cuda_graph.py @@ -3,12 +3,12 @@ import glob import json import os import random -import subprocess import sys import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.kits.mmmu_vlm_kit import _run_lmms_eval_with_retry from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -85,11 +85,7 @@ class TestVLMPiecewiseCudaGraph(CustomTestCase): str(output_path), ] - subprocess.run( - cmd, - check=True, - timeout=3600, - ) + _run_lmms_eval_with_retry(cmd, timeout=3600) def _run_vlm_mmmu_test( self, diff --git a/test/manual/nightly/test_vlms_vit_cuda_graph.py b/test/manual/nightly/test_vlms_vit_cuda_graph.py index 50e601126..d0f519848 100644 --- a/test/manual/nightly/test_vlms_vit_cuda_graph.py +++ b/test/manual/nightly/test_vlms_vit_cuda_graph.py @@ -3,12 +3,12 @@ import glob import json import os import random -import subprocess import sys import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.kits.mmmu_vlm_kit import _run_lmms_eval_with_retry from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -88,11 +88,7 @@ class TestVLMViTCudaGraph(CustomTestCase): str(output_path), ] - subprocess.run( - cmd, - check=True, - timeout=3600, - ) + _run_lmms_eval_with_retry(cmd, timeout=3600) def _run_vlm_mmmu_test( self, diff --git a/test/registered/vlm/test_encoder_dp.py b/test/registered/vlm/test_encoder_dp.py index a18075f71..fe44cdd93 100644 --- a/test/registered/vlm/test_encoder_dp.py +++ b/test/registered/vlm/test_encoder_dp.py @@ -3,13 +3,13 @@ import glob import json import os import random -import subprocess import sys import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.mmmu_vlm_kit import _run_lmms_eval_with_retry from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -94,11 +94,7 @@ class TestVLMEncoderDP(CustomTestCase): str(output_path), ] - subprocess.run( - cmd, - check=True, - timeout=3600, - ) + _run_lmms_eval_with_retry(cmd, timeout=3600) def _run_vlm_mmmu_test( self, diff --git a/test/srt/test_epd_disaggregation.py b/test/srt/test_epd_disaggregation.py index 2f2387c33..6733bc569 100644 --- a/test/srt/test_epd_disaggregation.py +++ b/test/srt/test_epd_disaggregation.py @@ -1,9 +1,9 @@ import os -import subprocess import threading import unittest from sglang.srt.utils import kill_process_tree +from sglang.test.kits.mmmu_vlm_kit import _run_lmms_eval_with_retry from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -177,7 +177,7 @@ class TestEPDDisaggregationOneEncoder(PDDisaggregationServerBase): limit, ] - subprocess.run(cmd, check=True, timeout=3600) + _run_lmms_eval_with_retry(cmd, timeout=3600) def test_mmmu(self): """Test MMMU evaluation with EPD disaggregation""" @@ -393,7 +393,7 @@ class TestEPDDisaggregationMultiEncoders(PDDisaggregationServerBase): limit, ] - subprocess.run(cmd, check=True, timeout=3600) + _run_lmms_eval_with_retry(cmd, timeout=3600) def test_mmmu(self): """Test MMMU evaluation with EPD disaggregation (multiple encoders)"""