Replace LLM cleanup with deterministic profiling

2026-08-18 17:36:36 +08:00
parent d48cce3f81
commit 4a062134d3
47 changed files with 1371 additions and 3846 deletions
@@ -0,0 +1,61 @@
+"""Tests for dataset aggregation and derived thresholds."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from swe_data_processing.summary import build_summary, percentile
+
+
+def _metric(sample_id: str, tokens: int, errors: int) -> dict:
+    return {
+        "sample_id": sample_id,
+        "resolved": 1,
+        "source_dataset": "test",
+        "length": {
+            "turn_count": 10,
+            "canonical_chars": 100,
+            "token_count": tokens,
+        },
+        "tools": {
+            "tool_call_count": 10,
+            "failed_tool_call_count": errors,
+            "failed_tool_call_rate": errors / 10,
+            "longest_consecutive_failure_run": 0,
+            "error_type_counts": {"nonzero_exit": errors} if errors else {},
+            "error_tool_counts": {"bash": errors} if errors else {},
+            "error_positions": {
+                "early_count": 0,
+                "early_fraction": 0.0,
+                "occupied_bins_5": 0,
+            },
+        },
+        "structure": {
+            "invalid_turn_count": 0,
+            "malformed_tool_definition_count": 0,
+            "malformed_tool_call_count": 0,
+            "unknown_tool_call_count": 0,
+            "missing_tool_result_count": 0,
+            "orphan_tool_result_count": 0,
+        },
+    }
+
+
+def test_percentile_interpolates() -> None:
+    assert percentile([0, 10], 0.5) == 5
+
+
+def test_summary_reports_token_tail_and_p99_threshold(tmp_path: Path) -> None:
+    path = tmp_path / "metrics.jsonl"
+    rows = [_metric(f"s-{index}", 10_000, index) for index in range(10)]
+    rows.append(_metric("long", 300_000, 0))
+    path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
+    summary, threshold = build_summary(path)
+    assert summary["sample_count"] == 11
+    assert summary["length"]["buckets"]["GT_262144"]["samples"] == 1
+    assert summary["length"]["longest_sample_token_contribution"]["0.001"][
+        "token_share"
+    ] > 0.7
+    assert threshold >= 8
+