Fix Deepseek nightly tests (#12906)
This commit is contained in:
Kangyan-Zhou
4
.github/workflows/nightly-test-b200.yml
vendored
4
.github/workflows/nightly-test-b200.yml
vendored
@@ -53,7 +53,7 @@ jobs:
|
||||
run: |
|
||||
rm -rf test/srt/performance_profiles_deepseek_v31/
|
||||
cd test/srt
|
||||
IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py --continue-on-error
|
||||
IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
|
||||
|
||||
- name: Publish DeepSeek v3.1 traces to storage repo
|
||||
env:
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
run: |
|
||||
rm -rf test/srt/performance_profiles_deepseek_v32/
|
||||
cd test/srt
|
||||
IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py --continue-on-error
|
||||
IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
|
||||
|
||||
- name: Publish DeepSeek v3.2 traces to storage repo
|
||||
env:
|
||||
|
||||
@@ -88,7 +88,7 @@ def get_tree_sha(repo_owner, repo_name, commit_sha, token):
|
||||
return data["tree"]["sha"]
|
||||
|
||||
|
||||
def create_blob(repo_owner, repo_name, content, token):
|
||||
def create_blob(repo_owner, repo_name, content, token, max_retries=3):
|
||||
"""Create a blob with file content"""
|
||||
url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/blobs"
|
||||
|
||||
@@ -97,16 +97,27 @@ def create_blob(repo_owner, repo_name, content, token):
|
||||
|
||||
data = {"content": content_b64, "encoding": "base64"}
|
||||
|
||||
response = make_github_request(url, token, method="POST", data=data)
|
||||
return json.loads(response)["sha"]
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = make_github_request(url, token, method="POST", data=data)
|
||||
return json.loads(response)["sha"]
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s
|
||||
print(
|
||||
f"Blob creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def create_tree(repo_owner, repo_name, base_tree_sha, files, token):
|
||||
def create_tree(repo_owner, repo_name, base_tree_sha, files, token, max_retries=3):
|
||||
"""Create a new tree with files"""
|
||||
url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/trees"
|
||||
|
||||
tree_items = []
|
||||
for file_path, content in files:
|
||||
for i, (file_path, content) in enumerate(files):
|
||||
# Create blob first to get SHA
|
||||
blob_sha = create_blob(repo_owner, repo_name, content, token)
|
||||
tree_items.append(
|
||||
@@ -117,24 +128,62 @@ def create_tree(repo_owner, repo_name, base_tree_sha, files, token):
|
||||
"sha": blob_sha,
|
||||
}
|
||||
)
|
||||
# Progress indicator for large uploads
|
||||
if (i + 1) % 10 == 0 or (i + 1) == len(files):
|
||||
print(f"Created {i + 1}/{len(files)} blobs...")
|
||||
|
||||
data = {"base_tree": base_tree_sha, "tree": tree_items}
|
||||
|
||||
response = make_github_request(url, token, method="POST", data=data)
|
||||
return json.loads(response)["sha"]
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = make_github_request(url, token, method="POST", data=data)
|
||||
return json.loads(response)["sha"]
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2**attempt
|
||||
print(
|
||||
f"Tree creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def create_commit(repo_owner, repo_name, tree_sha, parent_sha, message, token):
|
||||
def create_commit(
|
||||
repo_owner, repo_name, tree_sha, parent_sha, message, token, max_retries=3
|
||||
):
|
||||
"""Create a new commit"""
|
||||
url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits"
|
||||
|
||||
data = {"tree": tree_sha, "parents": [parent_sha], "message": message}
|
||||
|
||||
response = make_github_request(url, token, method="POST", data=data)
|
||||
return json.loads(response)["sha"]
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = make_github_request(url, token, method="POST", data=data)
|
||||
commit_sha = json.loads(response)["sha"]
|
||||
|
||||
# Verify the commit was actually created
|
||||
verify_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/commits/{commit_sha}"
|
||||
verify_response = make_github_request(verify_url, token)
|
||||
verify_data = json.loads(verify_response)
|
||||
if verify_data["sha"] != commit_sha:
|
||||
raise Exception(
|
||||
f"Commit verification failed: expected {commit_sha}, got {verify_data['sha']}"
|
||||
)
|
||||
|
||||
return commit_sha
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2**attempt
|
||||
print(
|
||||
f"Commit creation failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token):
|
||||
def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token, max_retries=3):
|
||||
"""Update branch reference to point to new commit"""
|
||||
url = (
|
||||
f"https://api.github.com/repos/{repo_owner}/{repo_name}/git/refs/heads/{branch}"
|
||||
@@ -142,7 +191,39 @@ def update_branch_ref(repo_owner, repo_name, branch, commit_sha, token):
|
||||
|
||||
data = {"sha": commit_sha}
|
||||
|
||||
make_github_request(url, token, method="PATCH", data=data)
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
make_github_request(url, token, method="PATCH", data=data)
|
||||
return
|
||||
except HTTPError as e:
|
||||
# Check if this is an "Object does not exist" error
|
||||
is_object_not_exist = False
|
||||
if hasattr(e, "error_body"):
|
||||
try:
|
||||
error_data = json.loads(e.error_body)
|
||||
if "Object does not exist" in error_data.get("message", ""):
|
||||
is_object_not_exist = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if is_object_not_exist and attempt < max_retries - 1:
|
||||
# This might be a transient consistency issue - wait and retry
|
||||
wait_time = 2**attempt
|
||||
print(
|
||||
f"Branch update failed with 'Object does not exist' (attempt {attempt + 1}/{max_retries}), waiting {wait_time}s for consistency..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2**attempt
|
||||
print(
|
||||
f"Branch update failed (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s..."
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def copy_trace_files(source_dir, target_base_path):
|
||||
@@ -240,20 +321,30 @@ def publish_traces(traces_dir, run_id, run_number):
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
is_ff_error = False
|
||||
if (
|
||||
hasattr(e, "error_body")
|
||||
and "Update is not a fast forward" in e.error_body
|
||||
):
|
||||
is_ff_error = True
|
||||
# Check for retryable errors
|
||||
is_retryable = False
|
||||
error_type = "unknown"
|
||||
|
||||
if is_ff_error and attempt < max_retries - 1:
|
||||
if hasattr(e, "error_body"):
|
||||
if "Update is not a fast forward" in e.error_body:
|
||||
is_retryable = True
|
||||
error_type = "fast-forward conflict"
|
||||
elif "Object does not exist" in e.error_body:
|
||||
is_retryable = True
|
||||
error_type = "object consistency"
|
||||
|
||||
# Also retry on HTTP errors that might be transient
|
||||
if isinstance(e, HTTPError) and e.code in [422, 500, 502, 503, 504]:
|
||||
is_retryable = True
|
||||
error_type = f"HTTP {e.code}"
|
||||
|
||||
if is_retryable and attempt < max_retries - 1:
|
||||
print(
|
||||
f"Attempt {attempt + 1} failed: not a fast-forward update. Retrying in {retry_delay} seconds..."
|
||||
f"Attempt {attempt + 1}/{max_retries} failed ({error_type}). Retrying in {retry_delay} seconds..."
|
||||
)
|
||||
time.sleep(retry_delay)
|
||||
else:
|
||||
print(f"Failed to publish traces: {e}")
|
||||
print(f"Failed to publish traces after {attempt + 1} attempts: {e}")
|
||||
raise
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
|
||||
PROFILE_DIR = "performance_profiles_deepseek_v31"
|
||||
|
||||
|
||||
class TestNightlyDeepseekV31Basic(unittest.TestCase):
|
||||
class TestNightlyDeepseekV31Performance(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V31_MODEL_PATH
|
||||
@@ -16,81 +16,72 @@ class TestNightlyDeepseekV31Basic(unittest.TestCase):
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
|
||||
# Define variant configurations
|
||||
cls.variants = [
|
||||
{
|
||||
"name": "basic",
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "mtp",
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-frac",
|
||||
"0.7",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="basic",
|
||||
)
|
||||
failed_variants = []
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
try:
|
||||
for variant_config in self.variants:
|
||||
with self.subTest(variant=variant_config["name"]):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=variant_config["other_args"],
|
||||
variant=variant_config["name"],
|
||||
)
|
||||
|
||||
if not success:
|
||||
if not success:
|
||||
failed_variants.append(variant_config["name"])
|
||||
|
||||
self.runner.add_report(results)
|
||||
finally:
|
||||
self.runner.write_final_report()
|
||||
|
||||
if failed_variants:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with basic configuration"
|
||||
)
|
||||
|
||||
|
||||
class TestNightlyDeepseekV31MTP(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V31_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-frac",
|
||||
"0.7",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="mtp",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with MTP configuration"
|
||||
f"Benchmark failed for {self.model} with the following variants: "
|
||||
f"{', '.join(failed_variants)}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
|
||||
PROFILE_DIR = "performance_profiles_deepseek_v32"
|
||||
|
||||
|
||||
class TestNightlyDeepseekV32Basic(unittest.TestCase):
|
||||
class TestNightlyDeepseekV32Performance(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V32_MODEL_PATH
|
||||
@@ -16,125 +16,89 @@ class TestNightlyDeepseekV32Basic(unittest.TestCase):
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
|
||||
# Define variant configurations
|
||||
cls.variants = [
|
||||
{
|
||||
"name": "basic",
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "mtp",
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-frac",
|
||||
"0.7",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "nsa",
|
||||
"other_args": [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--attention-backend",
|
||||
"nsa",
|
||||
"--nsa-prefill-backend",
|
||||
"flashmla_sparse",
|
||||
"--nsa-decode-backend",
|
||||
"flashmla_kv",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="basic",
|
||||
)
|
||||
failed_variants = []
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
try:
|
||||
for variant_config in self.variants:
|
||||
with self.subTest(variant=variant_config["name"]):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=variant_config["other_args"],
|
||||
variant=variant_config["name"],
|
||||
)
|
||||
|
||||
if not success:
|
||||
if not success:
|
||||
failed_variants.append(variant_config["name"])
|
||||
|
||||
self.runner.add_report(results)
|
||||
finally:
|
||||
self.runner.write_final_report()
|
||||
|
||||
if failed_variants:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with basic configuration"
|
||||
)
|
||||
|
||||
|
||||
class TestNightlyDeepseekV32MTP(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V32_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-num-steps",
|
||||
"3",
|
||||
"--speculative-eagle-topk",
|
||||
"1",
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
"--mem-frac",
|
||||
"0.7",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="mtp",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with MTP configuration"
|
||||
)
|
||||
|
||||
|
||||
class TestNightlyDeepseekV32NSA(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEEPSEEK_V32_MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.batch_sizes = [1, 1, 8, 16, 64]
|
||||
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
|
||||
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
|
||||
cls.other_args = [
|
||||
"--trust-remote-code",
|
||||
"--tp",
|
||||
"8",
|
||||
"--dp",
|
||||
"8",
|
||||
"--enable-dp-attention",
|
||||
"--attention-backend",
|
||||
"nsa",
|
||||
"--nsa-prefill-backend",
|
||||
"flashmla_sparse",
|
||||
"--nsa-decode-backend",
|
||||
"flashmla_kv",
|
||||
]
|
||||
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
|
||||
cls.runner.setup_profile_directory()
|
||||
|
||||
def test_bench_one_batch(self):
|
||||
results, success = self.runner.run_benchmark_for_model(
|
||||
model_path=self.model,
|
||||
batch_sizes=self.batch_sizes,
|
||||
input_lens=self.input_lens,
|
||||
output_lens=self.output_lens,
|
||||
other_args=self.other_args,
|
||||
variant="nsa",
|
||||
)
|
||||
|
||||
self.runner.add_report(results)
|
||||
self.runner.write_final_report()
|
||||
|
||||
if not success:
|
||||
raise AssertionError(
|
||||
f"Benchmark failed for {self.model} with NSA configuration"
|
||||
f"Benchmark failed for {self.model} with the following variants: "
|
||||
f"{', '.join(failed_variants)}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -220,7 +220,7 @@ suites = {
|
||||
],
|
||||
"nightly-4-gpu-b200": [
|
||||
TestFile("test_fp4_moe.py", 300),
|
||||
TestFile("nightly/test_nightly_gpt_oss_4gpu_perf.py", 600),
|
||||
TestFile("nightly/test_gpt_oss_4gpu_perf.py", 600),
|
||||
],
|
||||
"nightly-8-gpu-b200": [],
|
||||
"nightly-4-gpu": [],
|
||||
|
||||
Reference in New Issue
Block a user