[CI] Migrate nightly tests to test/registered/ (#15582)

2025-12-22 22:16:32 -08:00
parent bc3ca30023
commit 989d4b3012
64 changed files with 168 additions and 140 deletions
@@ -0,0 +1,138 @@
+import json
+import unittest
+import warnings
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
+    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
+    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
+    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    ModelLaunchSettings,
+    check_evaluation_test_results,
+    parse_models,
+    popen_launch_server,
+    write_results_to_json,
+)
+
+register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True)
+
+MODEL_SCORE_THRESHOLDS = {
+    "meta-llama/Llama-3.1-8B-Instruct": 0.82,
+    "mistralai/Mistral-7B-Instruct-v0.3": 0.58,
+    "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
+    "google/gemma-2-27b-it": 0.91,
+    "meta-llama/Llama-3.1-70B-Instruct": 0.95,
+    "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.616,
+    "Qwen/Qwen2-57B-A14B-Instruct": 0.86,
+    "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83,
+    "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
+    "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.835,
+    "zai-org/GLM-4.5-Air-FP8": 0.75,
+    # The threshold of neuralmagic/gemma-2-2b-it-FP8 should be 0.6, but this model has some accuracy regression.
+    # The fix is tracked at https://github.com/sgl-project/sglang/issues/4324, we set it to 0.50, for now, to make CI green.
+    "neuralmagic/gemma-2-2b-it-FP8": 0.50,
+    "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
+    "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65,
+    "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
+    "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82,
+}
+
+
+# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry
+class TestNightlyGsm8KEval(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.models = []
+        models_tp1 = parse_models(
+            DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
+        ) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1)
+        for model_path in models_tp1:
+            cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
+
+        models_tp2 = parse_models(
+            DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2
+        ) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2)
+        for model_path in models_tp2:
+            cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
+
+        cls.base_url = DEFAULT_URL_FOR_TEST
+
+    def test_mgsm_en_all_models(self):
+        warnings.filterwarnings(
+            "ignore", category=ResourceWarning, message="unclosed.*socket"
+        )
+        is_first = True
+        all_results = []
+        for model_setup in self.models:
+            with self.subTest(model=model_setup.model_path):
+                other_args = list(model_setup.extra_args)
+                error_message = None
+
+                if model_setup.model_path == "meta-llama/Llama-3.1-70B-Instruct":
+                    other_args.extend(["--mem-fraction-static", "0.9"])
+
+                process = popen_launch_server(
+                    model=model_setup.model_path,
+                    other_args=other_args,
+                    base_url=self.base_url,
+                    timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                )
+
+                try:
+                    args = SimpleNamespace(
+                        base_url=self.base_url,
+                        model=model_setup.model_path,
+                        eval_name="mgsm_en",
+                        num_examples=None,
+                        num_threads=1024,
+                    )
+
+                    metrics = run_eval(args)
+                    print(
+                        f"{'=' * 42}\n{model_setup.model_path} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
+                    )
+
+                    write_results_to_json(
+                        model_setup.model_path, metrics, "w" if is_first else "a"
+                    )
+                    is_first = False
+
+                    # 0.0 for empty latency, None for no error
+                    all_results.append(
+                        (model_setup.model_path, metrics["score"], 0.0, error_message)
+                    )
+                except Exception as e:
+                    # Capture error message for the summary table
+                    error_message = str(e)
+                    # Still append result with error info (use None for N/A metrics to match else clause)
+                    all_results.append(
+                        (model_setup.model_path, None, None, error_message)
+                    )
+                    print(f"Error evaluating {model_setup.model_path}: {error_message}")
+                finally:
+                    kill_process_tree(process.pid)
+
+        try:
+            with open("results.json", "r") as f:
+                print("\nFinal Results from results.json:")
+                print(json.dumps(json.load(f), indent=2))
+        except Exception as e:
+            print(f"Error reading results.json: {e}")
+
+        # Check all scores after collecting all results
+        check_evaluation_test_results(
+            all_results,
+            self.__class__.__name__,
+            model_accuracy_thresholds=MODEL_SCORE_THRESHOLDS,
+            model_count=len(self.models),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,145 @@
+import json
+import unittest
+import warnings
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    ModelEvalMetrics,
+    ModelLaunchSettings,
+    check_evaluation_test_results,
+    popen_launch_server,
+    write_results_to_json,
+)
+
+register_cuda_ci(est_time=7200, suite="nightly-eval-vlm-2-gpu", nightly=True)
+
+MODEL_THRESHOLDS = {
+    # Conservative thresholds on 100 MMMU samples, especially for latency thresholds
+    ModelLaunchSettings("deepseek-ai/deepseek-vl2-small"): ModelEvalMetrics(
+        0.320, 56.1
+    ),
+    ModelLaunchSettings("deepseek-ai/Janus-Pro-7B"): ModelEvalMetrics(0.285, 40.3),
+    ModelLaunchSettings("Efficient-Large-Model/NVILA-8B-hf"): ModelEvalMetrics(
+        0.270, 56.7
+    ),
+    ModelLaunchSettings("Efficient-Large-Model/NVILA-Lite-2B-hf"): ModelEvalMetrics(
+        0.270, 23.8
+    ),
+    ModelLaunchSettings("google/gemma-3-4b-it"): ModelEvalMetrics(0.360, 10.9),
+    ModelLaunchSettings("google/gemma-3n-E4B-it"): ModelEvalMetrics(0.270, 17.7),
+    ModelLaunchSettings("mistral-community/pixtral-12b"): ModelEvalMetrics(0.360, 16.6),
+    ModelLaunchSettings("moonshotai/Kimi-VL-A3B-Instruct"): ModelEvalMetrics(
+        0.330, 22.3
+    ),
+    ModelLaunchSettings("openbmb/MiniCPM-o-2_6"): ModelEvalMetrics(0.330, 29.3),
+    ModelLaunchSettings("openbmb/MiniCPM-v-2_6"): ModelEvalMetrics(0.259, 36.3),
+    ModelLaunchSettings("OpenGVLab/InternVL2_5-2B"): ModelEvalMetrics(0.300, 17.0),
+    ModelLaunchSettings("Qwen/Qwen2-VL-7B-Instruct"): ModelEvalMetrics(0.310, 83.3),
+    ModelLaunchSettings("Qwen/Qwen2.5-VL-7B-Instruct"): ModelEvalMetrics(0.340, 31.9),
+    ModelLaunchSettings(
+        "Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]
+    ): ModelEvalMetrics(0.29, 37.0),
+    ModelLaunchSettings(
+        "unsloth/Mistral-Small-3.1-24B-Instruct-2503"
+    ): ModelEvalMetrics(0.310, 16.7),
+    ModelLaunchSettings("XiaomiMiMo/MiMo-VL-7B-RL"): ModelEvalMetrics(0.28, 32.0),
+    ModelLaunchSettings("zai-org/GLM-4.1V-9B-Thinking"): ModelEvalMetrics(0.280, 30.4),
+    ModelLaunchSettings(
+        "zai-org/GLM-4.5V-FP8", extra_args=["--tp=2"]
+    ): ModelEvalMetrics(0.26, 32.0),
+}
+
+
+class TestNightlyVLMMmmuEval(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.models = list(MODEL_THRESHOLDS.keys())
+        cls.base_url = DEFAULT_URL_FOR_TEST
+
+    def test_mmmu_vlm_models(self):
+        warnings.filterwarnings(
+            "ignore", category=ResourceWarning, message="unclosed.*socket"
+        )
+        is_first = True
+        all_results = []
+
+        for model in self.models:
+            model_path = model.model_path
+            error_message = None
+            with self.subTest(model=model_path):
+                process = popen_launch_server(
+                    model=model_path,
+                    base_url=self.base_url,
+                    other_args=model.extra_args,
+                    timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+                )
+                try:
+                    args = SimpleNamespace(
+                        base_url=self.base_url,
+                        model=model_path,
+                        eval_name="mmmu",
+                        num_examples=100,
+                        num_threads=64,
+                        max_tokens=30,
+                    )
+
+                    args.return_latency = True
+
+                    metrics, latency = run_eval(args)
+
+                    metrics["score"] = round(metrics["score"], 4)
+                    metrics["latency"] = round(latency, 4)
+                    print(
+                        f"{'=' * 42}\n{model_path} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
+                    )
+
+                    write_results_to_json(model_path, metrics, "w" if is_first else "a")
+                    is_first = False
+
+                    all_results.append(
+                        (
+                            model_path,
+                            metrics["score"],
+                            metrics["latency"],
+                            error_message,
+                        )
+                    )
+                except Exception as e:
+                    # Capture error message for the summary table
+                    error_message = str(e)
+                    # Still append result with error info (use None for N/A metrics to match else clause)
+                    all_results.append((model_path, None, None, error_message))
+                    print(f"Error evaluating {model_path}: {error_message}")
+                finally:
+                    kill_process_tree(process.pid)
+
+        try:
+            with open("results.json", "r") as f:
+                print("\nFinal Results from results.json:")
+                print(json.dumps(json.load(f), indent=2))
+        except Exception as e:
+            print(f"Error reading results: {e}")
+
+        model_accuracy_thresholds = {
+            model.model_path: threshold.accuracy
+            for model, threshold in MODEL_THRESHOLDS.items()
+        }
+        model_latency_thresholds = {
+            model.model_path: threshold.eval_time
+            for model, threshold in MODEL_THRESHOLDS.items()
+        }
+        check_evaluation_test_results(
+            all_results,
+            self.__class__.__name__,
+            model_accuracy_thresholds=model_accuracy_thresholds,
+            model_latency_thresholds=model_latency_thresholds,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()