diff --git a/test/srt/hicache/test_hicache.py b/test/srt/hicache/test_hicache.py deleted file mode 100644 index f7616d098..000000000 --- a/test/srt/hicache/test_hicache.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -_is_hip = is_hip() - - -class TestHiCache(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-hierarchical-cache", - "--mem-fraction-static", - 0.7, - "--hicache-size", - 100 if not _is_hip else 200, - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/srt/hicache/test_hicache_eagle.py b/test/srt/hicache/test_hicache_eagle.py deleted file mode 100644 index f6265b9c1..000000000 --- a/test/srt/hicache/test_hicache_eagle.py +++ /dev/null @@ -1,78 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests - -from sglang.bench_serving import get_tokenizer -from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3, - DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - - -class TestHiCacheEagle(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3 - cls.base_url = DEFAULT_URL_FOR_TEST - cls.tokenizer = get_tokenizer(cls.model) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-hierarchical-cache", - "--hicache-ratio", - 1.2, - "--mem-fraction-static", - 0.7, - "--speculative-algorithm", - "EAGLE3", - "--speculative-draft-model-path", - DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, - "--speculative-num-steps", - 2, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 3, - "--dtype", - "float16", - "--chunked-prefill-size", - 1024, - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.72) - - server_info = requests.get(self.base_url + "/get_server_info") - print(f"{server_info=}") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 2.26) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/srt/hicache/test_hicache_mla.py b/test/srt/hicache/test_hicache_mla.py deleted file mode 100644 index c5db0f74a..000000000 --- a/test/srt/hicache/test_hicache_mla.py +++ /dev/null @@ -1,67 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_MLA_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -_is_hip = is_hip() -if _is_hip: - hicache_args = ["--hicache-size", 200] -else: - hicache_args = ["--hicache-ratio", 2] - - -class TestHierarchicalMLA(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--enable-hierarchical-cache", - ] - + hicache_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.5) - - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.8) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/srt/hicache/test_hicache_page.py b/test/srt/hicache/test_hicache_page.py deleted file mode 100644 index b1e1459c2..000000000 --- a/test/srt/hicache/test_hicache_page.py +++ /dev/null @@ -1,51 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - - -class TestHiCachePage(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enable-hierarchical-cache", - "--page-size", - 32, - "--hicache-write-policy", - "write_back", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/srt/hicache/test_hicache_variants.py b/test/srt/hicache/test_hicache_variants.py new file mode 100644 index 000000000..98706878c --- /dev/null +++ b/test/srt/hicache/test_hicache_variants.py @@ -0,0 +1,179 @@ +""" +Consolidated HiCache variant tests. +Tests HiCache with different configurations: standard, MLA, EAGLE, and page size variants. +""" + +import unittest +from types import SimpleNamespace + +import requests + +from sglang.bench_serving import get_tokenizer +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3, + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +_is_hip = is_hip() + + +class HiCacheEvalMixin: + """Mixin class containing common HiCache evaluation test methods""" + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) + + +class HiCacheMGSMEvalMixin: + """Mixin for tests that also run MGSM evaluation""" + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +class HiCacheBaseServer(CustomTestCase): + """Base class for HiCache tests with configurable server setup""" + + model_name = DEFAULT_MODEL_NAME_FOR_TEST + hicache_args = [] + expected_mmlu_score = 0.65 + + @classmethod + def setUpClass(cls): + cls.model = cls.model_name + cls.base_url = DEFAULT_URL_FOR_TEST + + # Setup tokenizer if needed by subclass + if hasattr(cls, "needs_tokenizer") and cls.needs_tokenizer: + cls.tokenizer = get_tokenizer(cls.model) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.hicache_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestHiCacheStandard(HiCacheBaseServer, HiCacheEvalMixin): + """Standard HiCache configuration tests""" + + model_name = DEFAULT_MODEL_NAME_FOR_TEST + hicache_args = [ + "--enable-hierarchical-cache", + "--mem-fraction-static", + 0.7, + "--hicache-size", + 100 if not _is_hip else 200, + ] + expected_mmlu_score = 0.65 + + +class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin): + """HiCache with MLA model tests""" + + model_name = DEFAULT_MLA_MODEL_NAME_FOR_TEST + hicache_args = [ + "--trust-remote-code", + "--enable-hierarchical-cache", + ] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2]) + expected_mmlu_score = 0.5 + + +class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin): + """HiCache with EAGLE speculative decoding tests""" + + model_name = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST_EAGLE3 + needs_tokenizer = True + hicache_args = [ + "--enable-hierarchical-cache", + "--hicache-ratio", + 1.2, + "--mem-fraction-static", + 0.7, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "--speculative-num-steps", + 2, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 3, + "--dtype", + "float16", + "--chunked-prefill-size", + 1024, + ] + expected_mmlu_score = 0.72 + + def test_mmlu(self): + """Override to add EAGLE-specific assertions""" + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) + + # EAGLE-specific check + server_info = requests.get(self.base_url + "/get_server_info") + print(f"{server_info=}") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.26) + + +class TestHiCachePage(HiCacheBaseServer, HiCacheEvalMixin): + """HiCache with custom page size tests""" + + model_name = DEFAULT_MODEL_NAME_FOR_TEST + hicache_args = [ + "--enable-hierarchical-cache", + "--page-size", + 32, + "--hicache-write-policy", + "write_back", + ] + expected_mmlu_score = 0.65 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/features/test_json_mode.py b/test/srt/openai_server/features/test_json_mode.py index d74eccf0f..dd5062d4c 100644 --- a/test/srt/openai_server/features/test_json_mode.py +++ b/test/srt/openai_server/features/test_json_mode.py @@ -1,14 +1,3 @@ -""" -python3 -m unittest openai_server.features.test_json_mode.TestJSONModeOutlines.test_json_mode_response -python3 -m unittest openai_server.features.test_json_mode.TestJSONModeOutlines.test_json_mode_with_streaming - -python3 -m unittest openai_server.features.test_json_mode.TestJSONModeXGrammar.test_json_mode_response -python3 -m unittest openai_server.features.test_json_mode.TestJSONModeXGrammar.test_json_mode_with_streaming - -python3 -m unittest openai_server.features.test_json_mode.TestJSONModeLLGuidance.test_json_mode_response -python3 -m unittest openai_server.features.test_json_mode.TestJSONModeLLGuidance.test_json_mode_with_streaming -""" - import json import unittest @@ -19,38 +8,13 @@ from sglang.test.test_utils import ( DEFAULT_SMALL_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, + CustomTestCase, popen_launch_server, ) -def setup_class(cls, backend): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - - other_args = [ - "--max-running-requests", - "10", - "--grammar-backend", - backend, - ] - - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") - - -class TestJSONModeOutlines(unittest.TestCase): - @classmethod - def setUpClass(cls): - setup_class(cls, "outlines") - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) +class TestJSONModeMixin: + """Mixin class containing JSON mode test methods""" def test_json_mode_response(self): """Test that response_format json_object (also known as "json mode") produces valid JSON, even without a system prompt that mentions JSON.""" @@ -121,16 +85,46 @@ class TestJSONModeOutlines(unittest.TestCase): self.assertIsInstance(js_obj, dict) -class TestJSONModeXGrammar(TestJSONModeOutlines): +class ServerWithGrammarBackend(CustomTestCase): + """Base class for tests requiring a grammar backend server""" + + backend = "xgrammar" + @classmethod def setUpClass(cls): - setup_class(cls, backend="xgrammar") + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + cls.backend, + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") -class TestJSONModeLLGuidance(TestJSONModeOutlines): @classmethod - def setUpClass(cls): - setup_class(cls, backend="llguidance") + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestJSONModeXGrammar(ServerWithGrammarBackend, TestJSONModeMixin): + backend = "xgrammar" + + +class TestJSONModeOutlines(ServerWithGrammarBackend, TestJSONModeMixin): + backend = "outlines" + + +class TestJSONModeLLGuidance(ServerWithGrammarBackend, TestJSONModeMixin): + backend = "llguidance" if __name__ == "__main__": diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 359688931..a649a569b 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -17,10 +17,8 @@ suites = { "per-commit-1-gpu": [ TestFile("debug_utils/test_tensor_dump_forward_hook.py", 15), TestFile("function_call/test_json_schema_constraint.py", 1), - TestFile("hicache/test_hicache.py", 116), - TestFile("hicache/test_hicache_eagle.py", 150), - TestFile("hicache/test_hicache_mla.py", 127), TestFile("hicache/test_hicache_storage.py", 127), + TestFile("hicache/test_hicache_variants.py", 393), TestFile("layers/attention/mamba/test_causal_conv1d.py", 25), TestFile("layers/attention/mamba/test_mamba_ssm.py", 50), TestFile("layers/attention/mamba/test_mamba_ssm_ssd.py", 20), @@ -246,7 +244,6 @@ suites = { TestFile("ep/test_moe_deepep_eval_accuracy_large.py"), TestFile("function_call/test_unknown_tool_name.py"), TestFile("hicache/test_disaggregation_hicache.py"), - TestFile("hicache/test_hicache_page.py"), TestFile("hicache/test_hicache_storage_benchmark.py"), TestFile("hicache/test_hicache_storage_e2e.py"), TestFile("layers/attention/nsa/test_act_quant_triton.py"), @@ -313,10 +310,8 @@ suites = { TestFile("test_get_weights_by_name.py"), TestFile("test_gpt_oss_common.py"), TestFile("test_health_check.py"), - TestFile("test_hicache.py"), - TestFile("test_hicache_mla.py"), - TestFile("test_hicache_page.py"), TestFile("test_hicache_storage.py"), + TestFile("test_hicache_variants.py"), TestFile("test_hybrid_dp_ep_tp_mtp.py"), TestFile("test_int4_kernel.py"), TestFile("test_int8_kernel.py"),