Minor fixes for server_args, parallel_state, and test_deterministic.py (#11159)

2025-10-02 12:12:49 -07:00
parent 7ff740a6ce
commit 6a261aaca5
4 changed files with 17 additions and 6 deletions
@@ -809,13 +809,13 @@ class TokenizerMetricsCollector:
    def check_time_to_first_token_straggler(self, value: float) -> bool:
        his = self.histogram_time_to_first_token.labels(**self.labels)
        total_observations = sum(bucket._value for bucket in his._buckets)
-        if total_observations < 100:
+        if total_observations < 1000:
            return False
-        p99_threshold = total_observations * 0.99
+        p999_threshold = total_observations * 0.999
        cumulative_count = 0
        for i, bucket in enumerate(his._buckets):
            cumulative_count += bucket._value
-            if cumulative_count > p99_threshold:
+            if cumulative_count > p999_threshold:
                return value >= his._upper_bounds[i]
        return False