From d22163eb8c02169f5e39de1f5f3d09023c1e3c25 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Thu, 5 Feb 2026 10:24:18 -0800 Subject: [PATCH] Fix flaky test_frequency_penalty_reduces_word_repetition by using deterministic seeds (#18285) --- test/registered/sampling/test_penalty.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/registered/sampling/test_penalty.py b/test/registered/sampling/test_penalty.py index d41e5bfd2..876790c95 100644 --- a/test/registered/sampling/test_penalty.py +++ b/test/registered/sampling/test_penalty.py @@ -62,10 +62,15 @@ class TestPenalty(CustomTestCase): print(json.dumps(response.json())) print("=" * 100) - def run_generate_with_prompt(self, prompt, sampling_params, max_tokens=100): + def run_generate_with_prompt( + self, prompt, sampling_params, max_tokens=100, seed=None + ): """Helper method to generate text with a specific prompt and parameters.""" + sampling_params = sampling_params.copy() sampling_params.setdefault("temperature", 0.05) sampling_params.setdefault("top_p", 1.0) + if seed is not None: + sampling_params["seed"] = seed response = requests.post( self.base_url + "/v1/chat/completions", @@ -96,15 +101,20 @@ class TestPenalty(CustomTestCase): ): """Generic test for penalty effects.""" # Run multiple iterations to get more reliable results + # Use fixed seeds for deterministic behavior + base_seed = 42 baseline_counts = [] penalty_counts = [] for i in range(5): + # Use same seed for both baseline and penalty in each iteration + # to ensure fair comparison with identical starting conditions + seed = base_seed + i baseline_output = self.run_generate_with_prompt( - prompt, baseline_params, max_tokens + prompt, baseline_params, max_tokens, seed=seed ) penalty_output = self.run_generate_with_prompt( - prompt, penalty_params, max_tokens + prompt, penalty_params, max_tokens, seed=seed ) baseline_count = self.count_word_repetitions(baseline_output, target_word)