diff --git a/test/srt/ep/test_deepep_large.py b/test/srt/ep/test_deepep_large.py index 2f33c5fa9..688e801cb 100644 --- a/test/srt/ep/test_deepep_large.py +++ b/test/srt/ep/test_deepep_large.py @@ -174,6 +174,8 @@ class TestDeepseekV32TBO(CustomTestCase): "deepep", "--cuda-graph-max-bs", "256", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, diff --git a/test/srt/test_deepseek_v32_basic.py b/test/srt/test_deepseek_v32_basic.py index bec74b96e..edae36131 100644 --- a/test/srt/test_deepseek_v32_basic.py +++ b/test/srt/test_deepseek_v32_basic.py @@ -28,6 +28,8 @@ class TestDeepseekV32DP(CustomTestCase): "--dp", "8", "--enable-dp-attention", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, @@ -83,6 +85,8 @@ class TestDeepseekV32TP(CustomTestCase): "--trust-remote-code", "--tp", "8", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, diff --git a/test/srt/test_deepseek_v32_cp_single_node.py b/test/srt/test_deepseek_v32_cp_single_node.py index 7fc898e01..1dec64966 100644 --- a/test/srt/test_deepseek_v32_cp_single_node.py +++ b/test/srt/test_deepseek_v32_cp_single_node.py @@ -52,6 +52,8 @@ class TestDeepseekV32CP(CustomTestCase): "--enable-nsa-prefill-context-parallel", "--nsa-prefill-cp-mode", "in-seq-split", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, @@ -123,6 +125,8 @@ class TestDeepseekV32CPMode1(CustomTestCase): "--enable-nsa-prefill-context-parallel", "--nsa-prefill-cp-mode", "round-robin-split", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, diff --git a/test/srt/test_deepseek_v32_mtp.py b/test/srt/test_deepseek_v32_mtp.py index 89b065ea4..cb022e8ea 100644 --- a/test/srt/test_deepseek_v32_mtp.py +++ b/test/srt/test_deepseek_v32_mtp.py @@ -40,6 +40,8 @@ class TestDeepseekV32DPMTP(CustomTestCase): "4", "--mem-frac", "0.7", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, @@ -120,6 +122,8 @@ class TestDeepseekV32TPMTP(CustomTestCase): "4", "--mem-frac", "0.7", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] cls.process = popen_launch_server( cls.model, diff --git a/test/srt/test_deepseek_v3_basic.py b/test/srt/test_deepseek_v3_basic.py index e8c4b2d08..3bb67baf6 100644 --- a/test/srt/test_deepseek_v3_basic.py +++ b/test/srt/test_deepseek_v3_basic.py @@ -22,7 +22,13 @@ class TestDeepseekV3Basic(CustomTestCase): def setUpClass(cls): cls.model = FULL_DEEPSEEK_V3_MODEL_PATH cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code", "--tp", "8"] + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] cls.process = popen_launch_server( cls.model, cls.base_url, diff --git a/test/srt/test_deepseek_v3_mtp.py b/test/srt/test_deepseek_v3_mtp.py index 89b9e9e6b..b82a67bcd 100644 --- a/test/srt/test_deepseek_v3_mtp.py +++ b/test/srt/test_deepseek_v3_mtp.py @@ -36,6 +36,8 @@ class TestDeepseekV3MTP(CustomTestCase): "1", "--speculative-num-draft-tokens", "4", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', ] if not is_in_amd_ci(): other_args += ["--mem-frac", "0.7"]