From 93cf60fc64032b6dce11d0466fe6de556c99ec67 Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Sun, 9 Nov 2025 00:26:06 -0800 Subject: [PATCH] Fix Deepseek nightly tests (#12906) --- .github/workflows/nightly-test-b200.yml | 4 +- scripts/ci/publish_traces.py | 133 ++++++++++++--- test/srt/nightly/test_deepseek_v31_perf.py | 127 +++++++------- test/srt/nightly/test_deepseek_v32_perf.py | 188 +++++++++------------ test/srt/run_suite.py | 2 +- 5 files changed, 250 insertions(+), 204 deletions(-) diff --git a/.github/workflows/nightly-test-b200.yml b/.github/workflows/nightly-test-b200.yml index 71147a689..49f2c64e4 100644 --- a/.github/workflows/nightly-test-b200.yml +++ b/.github/workflows/nightly-test-b200.yml @@ -53,7 +53,7 @@ jobs: run: | rm -rf test/srt/performance_profiles_deepseek_v31/ cd test/srt - IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py --continue-on-error + IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py - name: Publish DeepSeek v3.1 traces to storage repo env: @@ -71,7 +71,7 @@ jobs: run: | rm -rf test/srt/performance_profiles_deepseek_v32/ cd test/srt - IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py --continue-on-error + IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py - name: Publish DeepSeek v3.2 traces to storage repo env: diff --git a/scripts/ci/publish_traces.py b/scripts/ci/publish_traces.py index 735cc345f..e024c7bba 100644 --- a/scripts/ci/publish_traces.py +++ b/scripts/ci/publish_traces.py @@ -88,7 +88,7 @@ def get_tree_sha(repo_owner, repo_name, commit_sha, token): return data["tree"]["sha"] -def create_blob(repo_owner, repo_name, content, token): +def create_blob(repo_owner, repo_name, content, token, max_retries=3): """Create a blob with file content""" url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs" @@ -97,16 +97,27 @@ def create_blob(repo_owner, repo_name, content, token): data = {"content": content_b64, "encoding": "base64"} - response = make_github_request(url, token, method="POST", data=data) - return json.loads(response)["sha"] + for attempt in range(max_retries): + try: + response = make_github_request(url, token, method="POST", data=data) + return json.loads(response)["sha"] + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s + print( + f"Blob creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise -def create_tree(repo_owner, repo_name, base_tree_sha, files, token): +def create_tree(repo_owner, repo_name, base_tree_sha, files, token, max_retries=3): """Create a new tree with files""" url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/trees" tree_items = [] - for file_path, content in files: + for i, (file_path, content) in enumerate(files): # Create blob first to get SHA blob_sha = create_blob(repo_owner, repo_name, content, token) tree_items.append( @@ -117,24 +128,62 @@ def create_tree(repo_owner, repo_name, base_tree_sha, files, token): "sha": blob_sha, } ) + # Progress indicator for large uploads + if (i + 1) % 10 == 0 or (i + 1) == len(files): + print(f"Created {i + 1}/{len(files)} blobs...") data = {"base_tree": base_tree_sha, "tree": tree_items} - response = make_github_request(url, token, method="POST", data=data) - return json.loads(response)["sha"] + for attempt in range(max_retries): + try: + response = make_github_request(url, token, method="POST", data=data) + return json.loads(response)["sha"] + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt + print( + f"Tree creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise -def create_commit(repo_owner, repo_name, tree_sha, parent_sha, message, token): +def create_commit( + repo_owner, repo_name, tree_sha, parent_sha, message, token, max_retries=3 +): """Create a new commit""" url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits" data = {"tree": tree_sha, "parents": [parent_sha], "message": message} - response = make_github_request(url, token, method="POST", data=data) - return json.loads(response)["sha"] + for attempt in range(max_retries): + try: + response = make_github_request(url, token, method="POST", data=data) + commit_sha = json.loads(response)["sha"] + + # Verify the commit was actually created + verify_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}" + verify_response = make_github_request(verify_url, token) + verify_data = json.loads(verify_response) + if verify_data["sha"] != commit_sha: + raise Exception( + f"Commit verification failed: expected {commit_sha}, got {verify_data['sha']}" + ) + + return commit_sha + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt + print( + f"Commit creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise -def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token): +def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retries=3): """Update branch reference to point to new commit""" url = ( f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}" @@ -142,7 +191,39 @@ def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token): data = {"sha": commit_sha} - make_github_request(url, token, method="PATCH", data=data) + for attempt in range(max_retries): + try: + make_github_request(url, token, method="PATCH", data=data) + return + except HTTPError as e: + # Check if this is an "Object does not exist" error + is_object_not_exist = False + if hasattr(e, "error_body"): + try: + error_data = json.loads(e.error_body) + if "Object does not exist" in error_data.get("message", ""): + is_object_not_exist = True + except Exception: + pass + + if is_object_not_exist and attempt < max_retries - 1: + # This might be a transient consistency issue - wait and retry + wait_time = 2**attempt + print( + f"Branch update failed with 'Object does not exist' (attempt {attempt + 1}/{max_retries}), waiting {wait_time}s for consistency..." + ) + time.sleep(wait_time) + else: + raise + except Exception as e: + if attempt < max_retries - 1: + wait_time = 2**attempt + print( + f"Branch update failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..." + ) + time.sleep(wait_time) + else: + raise def copy_trace_files(source_dir, target_base_path): @@ -240,20 +321,30 @@ def publish_traces(traces_dir, run_id, run_number): return except Exception as e: - is_ff_error = False - if ( - hasattr(e, "error_body") - and "Update is not a fast forward" in e.error_body - ): - is_ff_error = True + # Check for retryable errors + is_retryable = False + error_type = "unknown" - if is_ff_error and attempt < max_retries - 1: + if hasattr(e, "error_body"): + if "Update is not a fast forward" in e.error_body: + is_retryable = True + error_type = "fast-forward conflict" + elif "Object does not exist" in e.error_body: + is_retryable = True + error_type = "object consistency" + + # Also retry on HTTP errors that might be transient + if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]: + is_retryable = True + error_type = f"HTTP {e.code}" + + if is_retryable and attempt < max_retries - 1: print( - f"Attempt {attempt + 1} failed: not a fast-forward update. Retrying in {retry_delay} seconds..." + f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..." ) time.sleep(retry_delay) else: - print(f"Failed to publish traces: {e}") + print(f"Failed to publish traces after {attempt + 1} attempts: {e}") raise diff --git a/test/srt/nightly/test_deepseek_v31_perf.py b/test/srt/nightly/test_deepseek_v31_perf.py index b1ceeaa99..4d51c338c 100644 --- a/test/srt/nightly/test_deepseek_v31_perf.py +++ b/test/srt/nightly/test_deepseek_v31_perf.py @@ -8,7 +8,7 @@ DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1" PROFILE_DIR = "performance_profiles_deepseek_v31" -class TestNightlyDeepseekV31Basic(unittest.TestCase): +class TestNightlyDeepseekV31Performance(unittest.TestCase): @classmethod def setUpClass(cls): cls.model = DEEPSEEK_V31_MODEL_PATH @@ -16,81 +16,72 @@ class TestNightlyDeepseekV31Basic(unittest.TestCase): cls.batch_sizes = [1, 1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - cls.other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", + + # Define variant configurations + cls.variants = [ + { + "name": "basic", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + ], + }, + { + "name": "mtp", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ], + }, ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) cls.runner.setup_profile_directory() def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - variant="basic", - ) + failed_variants = [] - self.runner.add_report(results) - self.runner.write_final_report() + try: + for variant_config in self.variants: + with self.subTest(variant=variant_config["name"]): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=variant_config["other_args"], + variant=variant_config["name"], + ) - if not success: + if not success: + failed_variants.append(variant_config["name"]) + + self.runner.add_report(results) + finally: + self.runner.write_final_report() + + if failed_variants: raise AssertionError( - f"Benchmark failed for {self.model} with basic configuration" - ) - - -class TestNightlyDeepseekV31MTP(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V31_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - cls.other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-frac", - "0.7", - ] - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - variant="mtp", - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model} with MTP configuration" + f"Benchmark failed for {self.model} with the following variants: " + f"{', '.join(failed_variants)}" ) diff --git a/test/srt/nightly/test_deepseek_v32_perf.py b/test/srt/nightly/test_deepseek_v32_perf.py index 623d2d35b..c2f1cb359 100644 --- a/test/srt/nightly/test_deepseek_v32_perf.py +++ b/test/srt/nightly/test_deepseek_v32_perf.py @@ -8,7 +8,7 @@ DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" PROFILE_DIR = "performance_profiles_deepseek_v32" -class TestNightlyDeepseekV32Basic(unittest.TestCase): +class TestNightlyDeepseekV32Performance(unittest.TestCase): @classmethod def setUpClass(cls): cls.model = DEEPSEEK_V32_MODEL_PATH @@ -16,125 +16,89 @@ class TestNightlyDeepseekV32Basic(unittest.TestCase): cls.batch_sizes = [1, 1, 8, 16, 64] cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - cls.other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", + + # Define variant configurations + cls.variants = [ + { + "name": "basic", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + ], + }, + { + "name": "mtp", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + ], + }, + { + "name": "nsa", + "other_args": [ + "--trust-remote-code", + "--tp", + "8", + "--dp", + "8", + "--enable-dp-attention", + "--attention-backend", + "nsa", + "--nsa-prefill-backend", + "flashmla_sparse", + "--nsa-decode-backend", + "flashmla_kv", + ], + }, ] + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) cls.runner.setup_profile_directory() def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - variant="basic", - ) + failed_variants = [] - self.runner.add_report(results) - self.runner.write_final_report() + try: + for variant_config in self.variants: + with self.subTest(variant=variant_config["name"]): + results, success = self.runner.run_benchmark_for_model( + model_path=self.model, + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=variant_config["other_args"], + variant=variant_config["name"], + ) - if not success: + if not success: + failed_variants.append(variant_config["name"]) + + self.runner.add_report(results) + finally: + self.runner.write_final_report() + + if failed_variants: raise AssertionError( - f"Benchmark failed for {self.model} with basic configuration" - ) - - -class TestNightlyDeepseekV32MTP(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - cls.other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-frac", - "0.7", - ] - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - variant="mtp", - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model} with MTP configuration" - ) - - -class TestNightlyDeepseekV32NSA(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = DEEPSEEK_V32_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.batch_sizes = [1, 1, 8, 16, 64] - cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) - cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) - cls.other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--dp", - "8", - "--enable-dp-attention", - "--attention-backend", - "nsa", - "--nsa-prefill-backend", - "flashmla_sparse", - "--nsa-decode-backend", - "flashmla_kv", - ] - cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) - cls.runner.setup_profile_directory() - - def test_bench_one_batch(self): - results, success = self.runner.run_benchmark_for_model( - model_path=self.model, - batch_sizes=self.batch_sizes, - input_lens=self.input_lens, - output_lens=self.output_lens, - other_args=self.other_args, - variant="nsa", - ) - - self.runner.add_report(results) - self.runner.write_final_report() - - if not success: - raise AssertionError( - f"Benchmark failed for {self.model} with NSA configuration" + f"Benchmark failed for {self.model} with the following variants: " + f"{', '.join(failed_variants)}" ) diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 80e632fb5..40739b4bb 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -220,7 +220,7 @@ suites = { ], "nightly-4-gpu-b200": [ TestFile("test_fp4_moe.py", 300), - TestFile("nightly/test_nightly_gpt_oss_4gpu_perf.py", 600), + TestFile("nightly/test_gpt_oss_4gpu_perf.py", 600), ], "nightly-8-gpu-b200": [], "nightly-4-gpu": [],