From 7b44bc923e9afd3cb0ed3727a4f2520924853b7f Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Wed, 11 Mar 2026 12:35:21 -0700 Subject: [PATCH] Relax flaky B200 GSM8K accuracy thresholds (#20304) Co-authored-by: Alison Shao --- test/registered/quant/test_deepseek_v3_fp4_4gpu.py | 8 ++++---- .../spec/eagle/test_eagle_infer_beta_dp_attention.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py index 6dd9ca13c..f67b6f0ae 100644 --- a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py +++ b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py @@ -70,7 +70,7 @@ class TestDeepseekV3FP4(CustomTestCase): f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["accuracy"], 0.93) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) @@ -136,7 +136,7 @@ class TestDeepseekV3FP4PiecewiseCudaGraph(CustomTestCase): f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["accuracy"], 0.93) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) @@ -206,7 +206,7 @@ class TestDeepseekV3FP4CutlassMoE(CustomTestCase): f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n" f'{metrics["accuracy"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["accuracy"], 0.93) class TestDeepseekV3FP4SymmetricMemory(CustomTestCase): @@ -260,7 +260,7 @@ class TestDeepseekV3FP4SymmetricMemory(CustomTestCase): f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["accuracy"], 0.93) if __name__ == "__main__": diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index b990e5990..8a6e9779f 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -85,7 +85,7 @@ class TestEagleDPAttnServerSmall(CustomTestCase): def test_a_gsm8k(self): metrics, avg_spec_accept_length = test_gsm8k(self.base_url) - self.assertGreater(metrics["accuracy"], 0.64) + self.assertGreater(metrics["accuracy"], 0.62) self.assertGreater(avg_spec_accept_length, 2.7)