62 lines
2.0 KiB
Python
62 lines
2.0 KiB
Python
"""Tests for dataset aggregation and derived thresholds."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from swe_data_processing.summary import build_summary, percentile
|
|
|
|
|
|
def _metric(sample_id: str, tokens: int, errors: int) -> dict:
|
|
return {
|
|
"sample_id": sample_id,
|
|
"resolved": 1,
|
|
"source_dataset": "test",
|
|
"length": {
|
|
"turn_count": 10,
|
|
"canonical_chars": 100,
|
|
"token_count": tokens,
|
|
},
|
|
"tools": {
|
|
"tool_call_count": 10,
|
|
"failed_tool_call_count": errors,
|
|
"failed_tool_call_rate": errors / 10,
|
|
"longest_consecutive_failure_run": 0,
|
|
"error_type_counts": {"nonzero_exit": errors} if errors else {},
|
|
"error_tool_counts": {"bash": errors} if errors else {},
|
|
"error_positions": {
|
|
"early_count": 0,
|
|
"early_fraction": 0.0,
|
|
"occupied_bins_5": 0,
|
|
},
|
|
},
|
|
"structure": {
|
|
"invalid_turn_count": 0,
|
|
"malformed_tool_definition_count": 0,
|
|
"malformed_tool_call_count": 0,
|
|
"unknown_tool_call_count": 0,
|
|
"missing_tool_result_count": 0,
|
|
"orphan_tool_result_count": 0,
|
|
},
|
|
}
|
|
|
|
|
|
def test_percentile_interpolates() -> None:
|
|
assert percentile([0, 10], 0.5) == 5
|
|
|
|
|
|
def test_summary_reports_token_tail_and_p99_threshold(tmp_path: Path) -> None:
|
|
path = tmp_path / "metrics.jsonl"
|
|
rows = [_metric(f"s-{index}", 10_000, index) for index in range(10)]
|
|
rows.append(_metric("long", 300_000, 0))
|
|
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
|
|
summary, threshold = build_summary(path)
|
|
assert summary["sample_count"] == 11
|
|
assert summary["length"]["buckets"]["GT_262144"]["samples"] == 1
|
|
assert summary["length"]["longest_sample_token_contribution"]["0.001"][
|
|
"token_share"
|
|
] > 0.7
|
|
assert threshold >= 8
|
|
|