diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py index 158275d4e..6ea30a03b 100644 --- a/python/sglang/test/accuracy_test_runner.py +++ b/python/sglang/test/accuracy_test_runner.py @@ -26,6 +26,7 @@ class AccuracyTestParams: # Extended parameters for special evaluations (e.g., GPQA with thinking mode) thinking_mode: Optional[str] = None # e.g., "deepseek-v3" temperature: Optional[float] = None + top_p: Optional[float] = None repeat: Optional[int] = None @@ -81,6 +82,7 @@ def _run_simple_eval( return_latency: bool = False, thinking_mode: Optional[str] = None, temperature: Optional[float] = None, + top_p: Optional[float] = None, repeat: Optional[int] = None, ) -> Tuple[bool, Optional[str], Optional[dict]]: """Run evaluation using simple_eval backend (run_eval.py). @@ -117,6 +119,9 @@ def _run_simple_eval( if temperature is not None: args.temperature = temperature + if top_p is not None: + args.top_p = top_p + if repeat is not None: args.repeat = repeat @@ -212,7 +217,13 @@ def run_accuracy_test( print(f"{'='*60}\n") # Run evaluation based on dataset type - if params.dataset == "gsm8k": + # Use few_shot_eval for gsm8k by default for backward compatibility. + # Use simple_eval when any extended params are set that few_shot_eval doesn't support. + has_extended_params = any( + getattr(params, field) is not None + for field in ("thinking_mode", "temperature", "top_p", "repeat") + ) + if params.dataset == "gsm8k" and not has_extended_params: success, error, metrics = _run_few_shot_eval( model=model, base_url=base_url, @@ -230,6 +241,7 @@ def run_accuracy_test( return_latency=params.return_latency, thinking_mode=params.thinking_mode, temperature=params.temperature, + top_p=params.top_p, repeat=params.repeat, ) diff --git a/test/registered/8-gpu-models/test_ring_2_5_1t.py b/test/registered/8-gpu-models/test_ring_2_5_1t.py new file mode 100644 index 000000000..b686aa51c --- /dev/null +++ b/test/registered/8-gpu-models/test_ring_2_5_1t.py @@ -0,0 +1,53 @@ +import unittest + +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.test_utils import ModelLaunchSettings + +# register_cuda_ci(est_time=1000, suite="nightly-8-gpu-common", nightly=True) +register_cuda_ci(est_time=1000, suite="stage-c-test-8-gpu-h200") + +RING_2_5_1T_MODEL_PATH = "inclusionAI/Ring-2.5-1T" + + +class TestRing2_5_1T(unittest.TestCase): + """Accuracy test for Ring-2.5-1T. + + Ring-2.5-1T is a ~1T MoE model with linear attention layers. + Uses TP=8 for GSM8K evaluation. + """ + + def test_ring_2_5_1t(self): + base_args = [ + "--tp=8", + "--trust-remote-code", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] + + variants = [ + ModelLaunchSettings( + RING_2_5_1T_MODEL_PATH, + tp_size=8, + extra_args=base_args, + variant="TP8", + ), + ] + + run_combined_tests( + models=variants, + test_name="Ring-2.5-1T", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", + num_examples=200, + baseline_accuracy=0.88, + temperature=1.2, + top_p=0.8, + max_tokens=4096, + ), + ) + + +if __name__ == "__main__": + unittest.main()