diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index d1e4a0899..5c0dcd7f2 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -1076,7 +1076,7 @@ jobs: strategy: fail-fast: false matrix: - part: [0, 1, 2] + part: [0, 1, 2, 3] steps: - name: Checkout code uses: actions/checkout@v4 @@ -1113,7 +1113,7 @@ jobs: if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then CONTINUE_ON_ERROR_FLAG="--continue-on-error" fi - python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 4 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG unit-test-backend-8-gpu-h20: needs: [check-changes, call-gate, unit-test-backend-1-gpu, stage-b-test-4-gpu-b200] diff --git a/docs/basic_usage/deepseek_v32.md b/docs/basic_usage/deepseek_v32.md index fd81f2535..faf3d1b63 100644 --- a/docs/basic_usage/deepseek_v32.md +++ b/docs/basic_usage/deepseek_v32.md @@ -274,31 +274,40 @@ DeepSeek-V3.2-Speciale: ## DSA long sequence context parallel optimization(experimental) -Accuracy benchmark on long context can be tested on GPQA-diamond dataset with long output tokens and thinking enabled: +**Note: This feature is only verified on Hopper machines** + +For context parallel in DeepSeek V3.2 model, we provide two different modes of splitting tokens, which can be controlled with argument `--nsa-prefill-cp-mode`. + +### In sequence splitting (default setting) + +The first mode can be enabled by `--nsa-prefill-cp-mode in-seq-split`. This mode implements context parallel for DSA by splitting the sequence uniformly between context parallel ranks. At attention stage, each cp rank computes the indexer results of sharded sequence, and collects the whole kv cache through all gather operator. + +The communication group for context parallel reuses the one for attention tp, thus `cp_size` equals `atten_tp_size = tp_size / dp_size`. + +Note that in sequence splitting mode has the following restrictions: +- The batch size is restricted to 1 for prefill batches +- Multi-node/PD disaggregation is still not supported +- `moe_dense_tp_size=1`, `kv_cache_dtype = "bf16"`, `moe_a2a_backend = "deepep"` +- To ensure `cp_size > 1`, the passed in `tp_size` must be larger than `dp_size` + +For more details, please refer to PR https://github.com/sgl-project/sglang/pull/12065. + +Example: +```bash +# In-seq splitting mode launched with EP + DP +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-nsa-prefill-context-parallel --nsa-prefill-cp-mode in-seq-split --max-running-requests 32 +``` + +### Round robin splitting + +This mode can be enabled by specifying the parameter `--nsa-prefill-cp-mode round-robin-split`, which distributes tokens across ranks based on `token_idx % cp_size`. + +In this scenario, compared with the aforementioned method, it additionally supports the fused MoE backend (the fused MoE backend may deliver better performance than DeepEP in single-machine scenarios), FP8 KV-cache, and multi-batch prefill inference. But it cannot be enabled with dp attention together. + +For more details, please refer to PR https://github.com/sgl-project/sglang/pull/13959. Example usage: ```bash -# Launch with EP + DP -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --ep 8 --dp 2 --enable-dp-attention --enable-nsa-prefill-context-parallel --max-running-requests 32 -``` -### Context-parallel Tips -`CP_size` reuses `atten_tp_size`, which is equal to `TP_size` / `DP_size`. -Some features are still not supported at present. -- **Multi-batch prefill**: Currently, only single-request processing is supported during the prefill process. -- **disaggregation**: P/D disaggregation. -- **Cross-machine support**: - Currently only tested on a single machine (TP=8,EP=8). -- **Other Args**: Currently only supports moe_dense_tp_size=1, kv_cache_dtype = "bf16", moe_a2a_backend = "deepep", -- **DP_size**: `CP_size` reuses `atten_tp_size`, which is equal to `TP_size` / `DP_size`. For the cp function to work correctly, `TP_size` must be divisible by `DP_size`, and TP_size / DP_size > 1 (to ensure CP_size > 1). -- **Detailed design reference**: https://github.com/sgl-project/sglang/pull/12065 - -### Alternative context parallel mode - -You can switch the CP token splitting mode for prefill by specifying the parameter `--nsa-prefill-cp-mode round-robin-split`. It distributes tokens across ranks based on `token_idx % cp_size`. -In this scenario, compared with the aforementioned method, it additionally supports the fused MoE backend (the fused MoE backend may deliver better performance than DeepEP in single-machine scenarios), -FP8 KV-cache, and multi-batch prefill inference. For more details, please refer to PR https://github.com/sgl-project/sglang/pull/13959. - -Example usage: -```bash -# Launch with FusedMoe + CP8 + DP1 -python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --dp 1 --enable-dp-attention --enable-nsa-prefill-context-parallel --nsa-prefill-cp-mode round-robin-split --max-running-requests 32 +# Launch with FusedMoe + CP8 +python -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.2-Exp --tp 8 --enable-nsa-prefill-context-parallel --nsa-prefill-cp-mode round-robin-split --max-running-requests 32 ``` diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index b2a09577b..54d4e415a 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1065,15 +1065,26 @@ class ServerArgs: logger.info("Use nsa attention backend for DeepSeek NSA.") if not is_npu(): # CUDA GPU - self.enable_dp_attention = True - logger.warning("DP attention is enabled for DeepSeek NSA.") if self.enable_nsa_prefill_context_parallel: - # TODO Supports moe_dense_tp_size != 1, kv cache dtype = "fp8",moe_a2a_backend non-deepep and cross-machine operation . - self.moe_dense_tp_size = 1 - if self.nsa_prefill_cp_mode != "round-robin-split": + logger.warning( + f"Context parallel feature is still under experiment. It has only been verified on Hopper platform." + ) + if self.nsa_prefill_cp_mode == "in-seq-split": + # TODO Supports moe_dense_tp_size != 1, kv cache dtype = "fp8",moe_a2a_backend non-deepep and cross-machine operation . + self.enable_dp_attention = True + self.moe_dense_tp_size = 1 self.moe_a2a_backend = "deepep" self.ep_size = self.tp_size self.kv_cache_dtype = "bf16" + logger.warning( + f"For in-seq split mode, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, kv_cache_dtype == bf16, batch_size == 1" + ) + else: + self.enable_dp_attention = True + self.moe_dense_tp_size = 1 + assert ( + self.dp_size == 1 + ), "For round-robin split mode, dp attention is not supported." assert ( self.tp_size == 8 ), "Current multi-machine CP support suffers from precision issues. So context parallel only support Single machine(tp_size == 8)" diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index e130bd9ca..4553714f0 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -133,8 +133,9 @@ suites = { TestFile("test_deepseek_v3_mtp.py", 275), TestFile("test_disaggregation_hybrid_attention.py", 600), TestFile("models/test_kimi_k2_models.py", 200), - TestFile("test_deepseek_v32_basic.py", 275), - TestFile("test_deepseek_v32_mtp.py", 275), + TestFile("test_deepseek_v32_basic.py", 360), + TestFile("test_deepseek_v32_mtp.py", 360), + TestFile("test_deepseek_v32_cp_single_node.py", 360), TestFile("models/test_mimo_models.py", 200), ], "per-commit-8-gpu-h20": [ diff --git a/test/srt/test_deepseek_v32_basic.py b/test/srt/test_deepseek_v32_basic.py index ce2d5afae..bec74b96e 100644 --- a/test/srt/test_deepseek_v32_basic.py +++ b/test/srt/test_deepseek_v32_basic.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" -class TestDeepseekV32Basic(CustomTestCase): +class TestDeepseekV32DP(CustomTestCase): @classmethod def setUpClass(cls): cls.model = DEEPSEEK_V32_MODEL_PATH @@ -74,5 +74,60 @@ class TestDeepseekV32Basic(CustomTestCase): self.assertGreater(speed, 50) +class TestDeepseekV32TP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n" + ) + self.assertGreater(speed, 70) + + if __name__ == "__main__": unittest.main() diff --git a/test/manual/test_deepseek_v32_cp_single_node.py b/test/srt/test_deepseek_v32_cp_single_node.py similarity index 98% rename from test/manual/test_deepseek_v32_cp_single_node.py rename to test/srt/test_deepseek_v32_cp_single_node.py index 4fc526ce2..7fc898e01 100644 --- a/test/manual/test_deepseek_v32_cp_single_node.py +++ b/test/srt/test_deepseek_v32_cp_single_node.py @@ -50,6 +50,8 @@ class TestDeepseekV32CP(CustomTestCase): "--max-running-requests", "32", "--enable-nsa-prefill-context-parallel", + "--nsa-prefill-cp-mode", + "in-seq-split", ] cls.process = popen_launch_server( cls.model, @@ -104,9 +106,6 @@ class TestDeepseekV32CPMode1(CustomTestCase): "--trust-remote-code", "--tp", "8", - "--dp", - "1", - "--enable-dp-attention", "--speculative-algorithm", "EAGLE", "--speculative-num-steps", diff --git a/test/srt/test_deepseek_v32_mtp.py b/test/srt/test_deepseek_v32_mtp.py index 41e3aa78d..89b065ea4 100644 --- a/test/srt/test_deepseek_v32_mtp.py +++ b/test/srt/test_deepseek_v32_mtp.py @@ -18,7 +18,7 @@ from sglang.test.test_utils import ( FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" -class TestDeepseekV32MTP(CustomTestCase): +class TestDeepseekV32DPMTP(CustomTestCase): @classmethod def setUpClass(cls): cls.model = FULL_DEEPSEEK_V32_MODEL_PATH @@ -101,5 +101,85 @@ class TestDeepseekV32MTP(CustomTestCase): self.assertGreater(speed, 75) +class TestDeepseekV32TPMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v32 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(avg_spec_accept_length, 2.7) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + + self.assertGreater(acc_length, 2.7) + self.assertGreater(speed, 130) + + if __name__ == "__main__": unittest.main()