yi_lu/OpenSWETraces_cleanup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace LLM cleanup with deterministic profiling

Browse Source
This commit is contained in:
jiachun
2026-08-18 17:36:36 +08:00
parent d48cce3f81
commit 4a062134d3
47 changed files with 1371 additions and 3846 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_audit.py
-216
View File
@@ -1,216 +0,0 @@
"""Tests for the two-call boundary and prefix-quality policy."""
from __future__ import annotations
import json
import pytest
from swe_data_processing.audit import (
compute_prefix_quality,
derive_prefix_safety,
effective_boundary_policy,
materialize_prefix,
validate_boundary,
validate_prefix_quality,
)
from swe_data_processing.policy import PolicyViolation
from swe_data_processing.workflow import (
prepare_boundary_payload,
prepare_prefix_quality_payload,
)
def _record() -> dict:
return {
"trajectory_id": "sample-1",
"resolved": 0,
"tools": [],
"trajectory": [
{"role": "system", "content": "system"},
{"role": "user", "content": "Fix the bug."},
{"role": "assistant", "content": "I will inspect the code."},
{"role": "tool", "content": "relevant.py"},
{"role": "assistant", "content": "I will apply the harmful patch."},
{"role": "tool", "content": "tests failed"},
],
}
def _boundary() -> dict:
return {
"sample_id": "sample-1",
"checks": {
"task_coverage": "COMPLETE",
"final_patch_scope": "CLEAN",
"constraints": "RESPECTED",
"claims_vs_observations": "CONSISTENT",
},
"decision": "TRUNCATE",
"truncate_before_turn": 5,
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
"severity": "MAJOR",
"state_effect": "UNRECOVERED",
"evidence_turns": [5],
"reason": "The patch fails and is not repaired.",
}
def _quality() -> dict:
return {
"sample_id": "sample-1",
"behavior_issues": [],
"dimensions": {
"planning": 14,
"investigation": 15,
"tool_use_and_observation": 14,
"progress": 12,
"clarity_and_efficiency": 10,
},
"evidence_turns": [],
"reason": "Useful investigation.",
}
def test_valid_boundary_materializes_exact_prefix() -> None:
validate_boundary(_record(), _boundary())
prefix = materialize_prefix(_record(), 5)
assert prefix == _record()["trajectory"][:4]
def test_boundary_must_be_an_assistant_turn() -> None:
result = _boundary()
result["truncate_before_turn"] = 4
with pytest.raises(PolicyViolation, match="assistant turn"):
validate_boundary(_record(), result)
def test_boundary_evidence_must_include_boundary_turn() -> None:
result = _boundary()
result["evidence_turns"] = [6]
with pytest.raises(PolicyViolation, match="excluded assistant turn"):
validate_boundary(_record(), result)
def test_keep_full_cannot_contain_a_boundary() -> None:
result = _boundary()
result.update(
decision="KEEP_FULL",
truncate_before_turn=None,
category="NONE",
severity="NONE",
state_effect="NONE",
evidence_turns=[],
)
validate_boundary(_record(), result)
result["truncate_before_turn"] = 5
with pytest.raises(PolicyViolation, match="cannot contain a boundary"):
validate_boundary(_record(), result)
def test_prefix_quality_cannot_reference_suffix() -> None:
prefix = materialize_prefix(_record(), 5)
result = _quality()
result["evidence_turns"] = [5]
with pytest.raises(PolicyViolation, match="outside the prefix"):
validate_prefix_quality("sample-1", prefix, result)
def test_quality_score_and_tier_are_computed_locally() -> None:
score = compute_prefix_quality(_quality())
assert score["educational_quality_score"] == 65
assert score["quality_tier"] == "MEDIUM"
def test_unrecovered_major_issue_is_locally_invalid() -> None:
result = _quality()
result["behavior_issues"] = [
{
"turn_id": 3,
"kind": "ERROR",
"severity": "MAJOR",
"recovered": False,
"reason": "The prefix leaves a known broken edit.",
}
]
result["evidence_turns"] = [3]
validate_prefix_quality("sample-1", materialize_prefix(_record(), 5), result)
assert derive_prefix_safety(result)["prefix_valid"] is False
assert compute_prefix_quality(result)["quality_tier"] == "REJECT"
@pytest.mark.parametrize("resolved", [0, -1])
def test_non_success_outcome_is_not_capped_at_first_stateful_turn(resolved: int) -> None:
record = _record()
record["resolved"] = resolved
record["tools"] = [
{"type": "function", "function": {"name": "str_replace_editor"}}
]
record["trajectory"][2]["tool_calls"] = [
{
"id": "call-1",
"type": "function",
"function": {
"name": "str_replace_editor",
"arguments": '{"command":"str_replace","path":"src/a.py"}',
},
}
]
result = _boundary()
result.update(decision="KEEP_FULL", truncate_before_turn=None)
policy = effective_boundary_policy(record, result)
assert policy["decision"] == "KEEP_FULL"
assert policy["truncate_before_turn"] is None
assert policy["source"] == "MODEL_DECISION"
assert policy["first_stateful_turn"] == 3
def test_semantic_boundary_is_not_replaced_by_earlier_stateful_turn() -> None:
record = _record()
record["resolved"] = 0
record["tools"] = [
{"type": "function", "function": {"name": "str_replace_editor"}}
]
record["trajectory"][2]["tool_calls"] = [
{
"id": "call-1",
"type": "function",
"function": {
"name": "str_replace_editor",
"arguments": '{"command":"str_replace","path":"src/a.py"}',
},
}
]
result = _boundary()
policy = effective_boundary_policy(record, result)
assert policy["decision"] == "TRUNCATE"
assert policy["truncate_before_turn"] == 5
assert policy["source"] == "MODEL_BOUNDARY"
assert policy["first_stateful_turn"] == 3
def test_boundary_payload_hides_outcome_and_patch_metadata() -> None:
record = _record()
record["resolved"] = -1
record["metadata"] = {
"model_patch": {"patch": "MODEL_PATCH_SECRET"},
"reference_patch": {"patch": "REFERENCE_PATCH_SECRET"},
}
serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False)
assert "resolved" not in serialized
assert "MODEL_PATCH_SECRET" in serialized
assert "REFERENCE_PATCH_SECRET" not in serialized
def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None:
first = _record()
second = _record()
second["resolved"] = -1
second["trajectory"][4]["content"] = "DIFFERENT SUFFIX"
second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT"
second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}}
first_prefix = materialize_prefix(first, 5)
second_prefix = materialize_prefix(second, 5)
assert prepare_prefix_quality_payload(
first, first_prefix
) == prepare_prefix_quality_payload(second, second_prefix)
tests/test_audit_workflow.py
-165
View File
@@ -1,165 +0,0 @@
"""End-to-end tests for call isolation in the audit workflow."""
from __future__ import annotations
import json
from swe_data_processing.client import GLMResponse
from swe_data_processing.config import Settings
from swe_data_processing.workflow import audit_trajectory
class FakeClient:
"""Return deterministic responses while recording every API payload."""
def __init__(self, responses: list[dict]) -> None:
self.settings = Settings(api_key="test-secret")
self.responses = iter(responses)
self.calls: list[dict] = []
def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse:
self.calls.append(
{"system_prompt": system_prompt, "payload": payload, "schema": schema}
)
return GLMResponse(
data=next(self.responses),
request_id=f"request-{len(self.calls)}",
usage={"total_tokens": 10},
compatibility_fallback_used=False,
)
def _record() -> dict:
return {
"trajectory_id": "sample-1",
"resolved": 0,
"trajectory": [
{"role": "system", "content": "system"},
{"role": "user", "content": "Fix the bug."},
{"role": "assistant", "content": "Inspect relevant.py"},
{"role": "tool", "content": "relevant code"},
{"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"},
{"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"},
],
}
def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None:
client = FakeClient(
[
{
"sample_id": "sample-1",
"checks": {
"task_coverage": "INCOMPLETE",
"final_patch_scope": "POLLUTED",
"constraints": "RESPECTED",
"claims_vs_observations": "CONTRADICTED",
},
"decision": "TRUNCATE",
"truncate_before_turn": 5,
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
"severity": "MAJOR",
"state_effect": "UNRECOVERED",
"evidence_turns": [5],
"reason": "The patch is not repaired.",
},
{
"sample_id": "sample-1",
"behavior_issues": [],
"dimensions": {
"planning": 15,
"investigation": 15,
"tool_use_and_observation": 14,
"progress": 12,
"clarity_and_efficiency": 10,
},
"evidence_turns": [],
"reason": "The prefix is useful.",
},
]
)
result = audit_trajectory(_record(), client) # type: ignore[arg-type]
assert len(client.calls) == 2
quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False)
assert "BAD_SUFFIX_SENTINEL" not in quality_payload
assert "truncate_before_turn" not in quality_payload
assert result["prefix"]["retained_turn_count"] == 4
assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM"
assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"
def test_hold_boundary_skips_quality_call() -> None:
client = FakeClient(
[
{
"sample_id": "sample-1",
"checks": {
"task_coverage": "COMPLETE",
"final_patch_scope": "CLEAN",
"constraints": "RESPECTED",
"claims_vs_observations": "CONSISTENT",
},
"decision": "HOLD",
"truncate_before_turn": None,
"category": "NONE",
"severity": "NONE",
"state_effect": "UNCLEAR",
"evidence_turns": [],
"reason": "No clear unrepaired defect.",
}
]
)
result = audit_trajectory(_record(), client) # type: ignore[arg-type]
assert len(client.calls) == 1
assert result["recommended_use"] == "HOLD"
assert result["quality"] is None
def test_keep_full_process_trajectory_is_scored() -> None:
client = FakeClient(
[
{
"sample_id": "sample-1",
"checks": {
"task_coverage": "COMPLETE",
"final_patch_scope": "CLEAN",
"constraints": "RESPECTED",
"claims_vs_observations": "CONSISTENT",
},
"decision": "KEEP_FULL",
"truncate_before_turn": None,
"category": "NONE",
"severity": "NONE",
"state_effect": "NONE",
"evidence_turns": [],
"reason": "No unrepaired severe problem is visible.",
},
{
"sample_id": "sample-1",
"behavior_issues": [
{
"turn_id": 3,
"kind": "INEFFICIENCY",
"severity": "MINOR",
"recovered": True,
"reason": "The inspection was somewhat broad.",
}
],
"dimensions": {
"planning": 15,
"investigation": 15,
"tool_use_and_observation": 14,
"progress": 12,
"clarity_and_efficiency": 10,
},
"evidence_turns": [],
"reason": "Useful despite minor inefficiency.",
},
]
)
result = audit_trajectory(_record(), client) # type: ignore[arg-type]
assert len(client.calls) == 2
assert result["prefix"]["retained_turn_count"] == 6
assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1
assert result["recommended_use"] == "HOLD"
tests/test_cli.py
+17 -33
View File
@@ -1,54 +1,38 @@
"""Tests for the streaming CLI runner."""
"""Tests for CLI parsing and streaming profile execution."""
from __future__ import annotations
import json
import threading
import time
from pathlib import Path
from swe_data_processing.cli import _run_streaming_stage
from swe_data_processing.cli import _run_streaming_stage, build_parser
def test_streaming_stage_processes_records_concurrently(tmp_path: Path) -> None:
"""Multiple workers run processors in parallel while one thread writes JSONL."""
def test_streaming_stage_writes_each_record(tmp_path: Path) -> None:
input_path = tmp_path / "input.jsonl"
output_path = tmp_path / "output.jsonl"
input_path.write_text(
"".join(
json.dumps({"trajectory_id": f"sample-{index}"}) + "\n" for index in range(20)
),
"".join(json.dumps({"trajectory_id": f"s-{i}"}) + "\n" for i in range(10)),
encoding="utf-8",
)
lock = threading.Lock()
active = 0
peak_active = 0
def processor(record: dict) -> dict:
nonlocal active, peak_active
with lock:
active += 1
peak_active = max(peak_active, active)
time.sleep(0.01)
with lock:
active -= 1
return {"sample_id": record["trajectory_id"]}
status = _run_streaming_stage(
input_path=input_path,
output_path=output_path,
errors_path=None,
limit=None,
resume=False,
workers=4,
stage_name="test",
processor=processor,
workers=3,
processor=lambda record: {"sample_id": record["trajectory_id"]},
)
written = [json.loads(line) for line in output_path.read_text().splitlines()]
assert status == 0
assert peak_active == 4
assert {value["sample_id"] for value in written} == {
f"sample-{index}" for index in range(20)
}
assert len(output_path.read_text().splitlines()) == 10
def test_cli_has_only_deterministic_pipeline_commands() -> None:
parser = build_parser()
assert parser.parse_args(
["profile", "--input", "in.jsonl", "--output", "out.jsonl"]
).command == "profile"
help_text = parser.format_help()
assert "classify" not in help_text
assert "repair" not in help_text
tests/test_client.py
-103
View File
@@ -1,103 +0,0 @@
"""Tests for API request compatibility and structured-response validation."""
from __future__ import annotations
import json
import httpx
import pytest
from swe_data_processing.client import GLMClient
from swe_data_processing.config import Settings
def test_client_parses_fenced_json_and_never_sends_key_in_body() -> None:
captured_body = {}
def handler(request: httpx.Request) -> httpx.Response:
captured_body.update(json.loads(request.content))
assert request.headers["authorization"] == "Bearer test-secret"
return httpx.Response(
200,
headers={"x-request-id": "request-123"},
json={
"choices": [
{"message": {"role": "assistant", "content": "```json\n{\"status\":\"ok\"}\n```"}}
],
"usage": {"total_tokens": 10},
},
)
schema = {
"type": "object",
"additionalProperties": False,
"required": ["status"],
"properties": {"status": {"const": "ok"}},
}
settings = Settings(api_key="test-secret", max_retries=0)
with GLMClient(settings, transport=httpx.MockTransport(handler)) as client:
response = client.invoke_json(system_prompt="Return JSON.", payload={"ping": True}, schema=schema)
assert response.data == {"status": "ok"}
assert response.request_id == "request-123"
assert response.usage == {"total_tokens": 10}
assert "test-secret" not in json.dumps(captured_body)
assert captured_body["model"] == "glm-5.2"
def test_client_falls_back_when_vendor_extensions_are_rejected() -> None:
calls = []
def handler(request: httpx.Request) -> httpx.Response:
body = json.loads(request.content)
calls.append(body)
if len(calls) == 1:
return httpx.Response(400, json={"error": {"message": "unknown parameter: thinking"}})
return httpx.Response(
200,
json={"choices": [{"message": {"content": "{\"status\":\"ok\"}"}}]},
)
schema = {
"type": "object",
"required": ["status"],
"properties": {"status": {"const": "ok"}},
}
settings = Settings(api_key="test-secret", max_retries=1)
with GLMClient(settings, transport=httpx.MockTransport(handler)) as client:
response = client.invoke_json(system_prompt="Return JSON.", payload={}, schema=schema)
assert "thinking" in calls[0]
assert "thinking" not in calls[1]
assert response.compatibility_fallback_used is True
def test_client_retries_five_failures_before_succeeding(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Five retryable failures are followed by a sixth and final attempt."""
calls = 0
def handler(_: httpx.Request) -> httpx.Response:
nonlocal calls
calls += 1
if calls <= 5:
return httpx.Response(503, json={"error": {"message": "temporarily unavailable"}})
return httpx.Response(
200,
json={"choices": [{"message": {"content": "{\"status\":\"ok\"}"}}]},
)
monkeypatch.setattr("swe_data_processing.client.time.sleep", lambda _: None)
schema = {
"type": "object",
"required": ["status"],
"properties": {"status": {"const": "ok"}},
}
settings = Settings(api_key="test-secret", max_retries=5)
with GLMClient(settings, transport=httpx.MockTransport(handler)) as client:
response = client.invoke_json(system_prompt="Return JSON.", payload={}, schema=schema)
assert calls == 6
assert response.data == {"status": "ok"}
tests/test_config.py
-29
View File
@@ -1,29 +0,0 @@
"""Tests for strict environment configuration and endpoint normalization."""
from __future__ import annotations
import pytest
from swe_data_processing.config import Settings
def test_endpoint_normalization() -> None:
settings = Settings(
api_key="test-secret",
api_base="https://llm-api.cowin.run/",
api_path="v1/chat/completions",
)
assert settings.endpoint == "https://llm-api.cowin.run/v1/chat/completions"
def test_missing_api_key_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("GLM_API_KEY", raising=False)
with pytest.raises(ValueError, match="GLM_API_KEY"):
Settings.from_env()
def test_feature_only_settings_can_omit_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("GLM_API_KEY", raising=False)
settings = Settings.from_env(require_api_key=False)
assert settings.api_key == ""
assert settings.max_retries == 5
tests/test_evidence.py
-73
View File
@@ -1,73 +0,0 @@
"""Tests for prompt-only trajectory compaction."""
from swe_data_processing.evidence import (
BOUNDARY_BLOCK_TURNS,
build_trajectory_blocks,
compact_text,
compact_trajectory,
)
def test_compact_text_preserves_short_values() -> None:
"""Short evidence must remain byte-for-byte identical."""
assert compact_text("short evidence", 100) == "short evidence"
def test_compact_text_hashes_long_values() -> None:
"""Long evidence must expose its size and digest for auditability."""
result = compact_text("a" * 1_000, 200)
assert len(result) == 200
assert "COMPACTED original_chars=1000 sha256=" in result
def test_compact_trajectory_prioritizes_test_observations() -> None:
"""Important test output receives a larger preview than ordinary output."""
long_content = "x" * 3_000
trajectory = [
{"turn_id": 1, "role": "user", "content": "issue"},
{"turn_id": 2, "role": "assistant", "content": "", "tool_calls": []},
{"turn_id": 3, "role": "tool", "content": long_content},
{"turn_id": 4, "role": "assistant", "content": "done"},
]
signals = {
"stateful_turns": [],
"test_events": [{"command_turn": 2, "output_turn": 3}],
"malformed_tool_turns": [],
"unknown_tool_turns": [],
}
compacted, metadata = compact_trajectory(trajectory, signals)
assert compacted[2]["content"] == long_content
assert 3 in metadata["important_turn_ids"]
def test_boundary_blocks_preserve_absolute_turns_without_overlap() -> None:
trajectory = [
{"turn_id": turn, "role": "assistant", "content": f"turn {turn}"}
for turn in range(1, BOUNDARY_BLOCK_TURNS + 3)
]
blocks, metadata = build_trajectory_blocks(trajectory, {})
assert len(blocks) == 2
assert blocks[0]["start_turn"] == 1
assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS
assert blocks[1]["start_turn"] == BOUNDARY_BLOCK_TURNS + 1
assert blocks[1]["end_turn"] == BOUNDARY_BLOCK_TURNS + 2
turn_ids = [
message["turn_id"] for block in blocks for message in block["messages"]
]
assert turn_ids == list(range(1, BOUNDARY_BLOCK_TURNS + 3))
assert metadata["block_count"] == 2
def test_boundary_block_keeps_immediate_tool_result_with_assistant() -> None:
trajectory = [
{"turn_id": turn, "role": "user", "content": f"turn {turn}"}
for turn in range(1, BOUNDARY_BLOCK_TURNS + 2)
]
trajectory[BOUNDARY_BLOCK_TURNS - 1]["role"] = "assistant"
trajectory[BOUNDARY_BLOCK_TURNS]["role"] = "tool"
blocks, _ = build_trajectory_blocks(trajectory, {})
assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS + 1
assert len(blocks) == 1
tests/test_features.py
-100
View File
@@ -1,100 +0,0 @@
"""Tests for deterministic static trajectory evidence extraction."""
from __future__ import annotations
import json
from swe_data_processing.features import extract_patch_files, extract_static_signals
def _tool(name: str) -> str:
return json.dumps({"type": "function", "function": {"name": name, "parameters": {}}})
def test_detects_masked_failing_test_after_edit() -> None:
record = {
"trajectory_id": "sample-1",
"resolved": 0,
"tools": [_tool("str_replace_editor"), _tool("execute_bash"), _tool("finish")],
"trajectory": [
{"role": "system", "content": "system"},
{"role": "user", "content": "Do not modify test files."},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "edit-1",
"type": "function",
"function": {
"name": "str_replace_editor",
"arguments": json.dumps(
{"command": "str_replace", "path": "/workspace/src/main.py"}
),
},
}
],
},
{"role": "tool", "content": "The file was edited."},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "test-1",
"type": "function",
"function": {
"name": "execute_bash",
"arguments": json.dumps({"command": "pytest tests | tail -20"}),
},
}
],
},
{
"role": "tool",
"content": "1 failed, 10 passed\n[Command finished with exit code 0]",
},
],
"metadata": {
"model_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"},
"reference_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"},
},
}
signals = extract_static_signals(record)
assert signals["last_stateful_turn"] == 3
assert signals["instruction_constraints"] == ["Do not modify test files."]
assert signals["test_events"][0]["masked_pipeline"] is True
assert signals["test_events"][0]["explicit_failure"] is True
assert signals["test_events"][0]["explicit_exit_codes"] == [0]
def test_detects_malformed_and_unknown_tool_calls() -> None:
record = {
"trajectory_id": "sample-2",
"tools": [_tool("execute_bash")],
"trajectory": [
{"role": "system", "content": "system"},
{"role": "user", "content": "issue"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "bad",
"function": {"name": "made_up_tool", "arguments": "{bad json"},
}
],
},
{"role": "tool", "content": "error"},
],
"metadata": {"model_patch": {}, "reference_patch": {}},
}
signals = extract_static_signals(record)
assert signals["malformed_tool_turns"][0]["turn_id"] == 3
assert signals["unknown_tool_turns"] == [{"turn_id": 3, "tool_name": "made_up_tool"}]
def test_extract_patch_files_is_stable_and_unique() -> None:
patch = "+++ b/src/a.py\n+++ b/src/b.py\n+++ b/src/a.py\n+++ /dev/null\n"
assert extract_patch_files(patch) == ["src/a.py", "src/b.py"]
tests/test_heuristics.py
+84
View File
@@ -0,0 +1,84 @@
"""Tests for transparent filtering rules."""
from __future__ import annotations
from swe_data_processing.heuristics import classify_metrics, length_bucket
def _metrics(*, failed: int = 0, calls: int = 10, streak: int = 0, tokens=50_000):
return {
"sample_id": "sample-1",
"instance_id": "repo-1",
"source_group": "example_group",
"source_parquet": "/data/example.parquet",
"resolved": 1,
"length": {"turn_count": 30, "token_count": tokens},
"tools": {
"tool_call_count": calls,
"failed_tool_call_count": failed,
"failed_tool_call_rate": failed / calls if calls else 0.0,
"longest_consecutive_failure_run": streak,
"error_positions": {
"early_count": failed,
"early_fraction": 1.0 if failed else 0.0,
"occupied_bins_5": min(failed, 5),
},
"error_type_counts": {},
"error_tool_counts": {},
},
"structure": {
"invalid_turn_count": 0,
"malformed_tool_definition_count": 0,
"malformed_tool_call_count": 0,
"unknown_tool_call_count": 0,
"missing_tool_result_count": 0,
"orphan_tool_result_count": 0,
},
}
def test_length_buckets_match_training_limits() -> None:
assert length_bucket(None) == "TOKENIZER_REQUIRED"
assert length_bucket(81_920) == "LE_81920"
assert length_bucket(81_921) == "81921_TO_131072"
assert length_bucket(131_073) == "131073_TO_262144"
assert length_bucket(262_145) == "GT_262144"
def test_five_consecutive_failures_is_hard_reject() -> None:
decision = classify_metrics(_metrics(failed=5, calls=20, streak=5), 10)
assert "FIVE_CONSECUTIVE_TOOL_FAILURES" in decision["hard_reject_reasons"]
assert decision["recommended_action"] == "DROP_DEFINITE_TOOL_PROBLEM"
def test_early_cluster_is_review_only() -> None:
decision = classify_metrics(_metrics(failed=3, calls=20, streak=2), 10)
assert decision["hard_reject_reasons"] == []
assert decision["review_flags"] == ["EARLY_FAILURE_CLUSTER"]
assert decision["recommended_action"] == "REVIEW_HEURISTIC_HIT"
def test_relative_count_outlier_is_not_automatic_drop() -> None:
decision = classify_metrics(_metrics(failed=10, calls=100, streak=2), 10)
assert decision["hard_reject_reasons"] == []
assert "EXTREME_ERROR_COUNT" in decision["review_flags"]
def test_distributed_failures_require_review_instead_of_automatic_drop() -> None:
decision = classify_metrics(_metrics(failed=8, calls=40, streak=2), 20)
assert decision["hard_reject_reasons"] == []
assert "PERSISTENT_DISTRIBUTED_FAILURES" in decision["review_flags"]
def test_failed_outcome_is_never_recommended_for_training() -> None:
metrics = _metrics()
metrics["resolved"] = 0
decision = classify_metrics(metrics, 10)
assert decision["outcome_use"] == "EXCLUDE_FROM_SUCCESS_SFT"
assert decision["recommended_action"] == "EXCLUDE_FAILED_OUTCOME"
def test_decision_keeps_source_provenance_for_targeted_review() -> None:
decision = classify_metrics(_metrics(), 10)
assert decision["source_group"] == "example_group"
assert decision["source_parquet"] == "/data/example.parquet"
tests/test_metrics.py
+123
View File
@@ -0,0 +1,123 @@
"""Tests for deterministic trajectory metrics."""
from __future__ import annotations
import json
from swe_data_processing.metrics import canonical_training_text, extract_metrics
def _record(outputs: list[str]) -> dict:
trajectory = [{"role": "user", "content": "Fix the bug."}]
for index, output in enumerate(outputs, 1):
trajectory.extend(
[
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": f"call-{index}",
"type": "function",
"function": {
"name": "bash",
"arguments": json.dumps({"command": "pytest tests"}),
},
}
],
},
{"role": "tool", "content": output},
]
)
return {
"trajectory_id": "sample-1",
"instance_id": "repo-1",
"resolved": 1,
"tools": [{"type": "function", "function": {"name": "bash"}}],
"trajectory": trajectory,
}
def test_canonical_text_is_stable_and_excludes_metadata() -> None:
record = _record(["exit code 0"])
record["metadata"] = {"reference_patch": {"patch": "SECRET"}}
text = canonical_training_text(record)
assert "SECRET" not in text
assert text == canonical_training_text(record)
def test_profile_keeps_source_parquet_out_of_canonical_text() -> None:
record = _record(["exit code 0"])
record["_source_parquet"] = "/data/openhands/train.parquet"
metrics = extract_metrics(record)
assert metrics["source_group"] == "openhands"
assert "/data/openhands" not in canonical_training_text(record)
def test_extracts_failure_count_type_distribution_and_streak() -> None:
record = _record(
[
"2 failed\n[The command completed with exit code 1.]",
"permission denied\nexit status 2",
"tests passed\nexit code 0",
"timeout while waiting",
]
)
metrics = extract_metrics(record)
tools = metrics["tools"]
assert tools["tool_call_count"] == 4
assert tools["failed_tool_call_count"] == 3
assert tools["longest_consecutive_failure_run"] == 2
assert tools["error_type_counts"]["nonzero_exit"] == 2
assert tools["error_type_counts"]["test_failure"] == 1
assert tools["error_type_counts"]["permission_denied"] == 1
assert tools["error_type_counts"]["timeout"] == 1
assert tools["error_positions"]["bins_5"] == [1, 1, 0, 0, 1]
def test_explicit_success_suppresses_incidental_error_words() -> None:
record = _record(["Read fixture containing 'permission denied'.\nexit code 0"])
assert extract_metrics(record)["tools"]["failed_tool_call_count"] == 0
def test_detects_broken_tool_structure() -> None:
record = _record(["exit code 0"])
call = record["trajectory"][1]["tool_calls"][0]
call["function"]["arguments"] = "{broken"
record["trajectory"].pop()
metrics = extract_metrics(record)
assert metrics["structure"]["malformed_tool_call_count"] == 1
assert metrics["structure"]["missing_tool_result_count"] == 1
assert metrics["tools"]["failed_tool_call_count"] == 1
def test_terminal_finish_without_result_is_valid() -> None:
record = _record([])
record["trajectory"].append(
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"function": {"name": "finish", "arguments": "{}"},
"id": "finish-1",
"type": "function",
}
],
}
)
record["tools"].append({"type": "function", "function": {"name": "finish"}})
metrics = extract_metrics(record)
assert metrics["structure"]["missing_tool_result_count"] == 0
assert metrics["tools"]["failed_tool_call_count"] == 0
def test_editor_file_content_does_not_trigger_shell_patterns() -> None:
record = _record(["source code contains timeout and permission denied"])
call = record["trajectory"][1]["tool_calls"][0]
call["function"]["name"] = "str_replace_editor"
call["function"]["arguments"] = '{"command":"view","path":"src/a.py"}'
record["tools"] = [
{"type": "function", "function": {"name": "str_replace_editor"}}
]
assert extract_metrics(record)["tools"]["failed_tool_call_count"] == 0
tests/test_policy.py
-100
View File
@@ -1,100 +0,0 @@
"""Tests for immutable classification and repair policy enforcement."""
from __future__ import annotations
import pytest
from swe_data_processing.policy import (
PolicyViolation,
enforce_classification_policy,
enforce_repair_policy,
)
def _classification() -> dict:
dimensions = {
"trajectory_integrity": "PASS",
"tool_integrity": "PASS",
"patch_presence": "PASS",
"patch_trajectory_consistency": "PASS",
"instruction_compliance": "PASS",
"verification_consistency": "PASS",
"final_claim_alignment": "PASS",
"patch_hygiene": "PASS",
"issue_patch_alignment": "PASS",
}
return {
"source_outcome_class": "POSITIVE_CANDIDATE",
"qc_decision": "ACCEPT_SILVER_POSITIVE",
"training_use": "SFT_FULL",
"qc_passed": True,
"hard_fail_codes": [],
"verification": {"status": "PASS_RELIABLE"},
"dimensions": dimensions,
}
def test_valid_silver_positive_passes() -> None:
enforce_classification_policy({"resolved": 1}, _classification())
def test_resolved_zero_cannot_be_promoted() -> None:
result = _classification()
result["source_outcome_class"] = "EXPLICIT_NEGATIVE"
with pytest.raises(PolicyViolation, match="resolved=0"):
enforce_classification_policy({"resolved": 0}, result)
def test_unknown_cannot_be_full_sft() -> None:
result = _classification()
result["source_outcome_class"] = "UNVERIFIED"
result["qc_decision"] = "HOLD_UNVERIFIED"
result["qc_passed"] = False
with pytest.raises(PolicyViolation, match="resolved=-1"):
enforce_classification_policy({"resolved": -1}, result)
def _repair_result() -> dict:
"""Return a minimal non-mutating repair result for policy tests."""
return {
"repair_decision": "NO_CHANGE",
"maximum_training_use": "HOLD",
"invariants": {
"resolved_unchanged": True,
"tool_outputs_unchanged": True,
"model_patch_unchanged": True,
"reference_patch_unchanged": True,
"no_synthetic_execution_result": True,
},
"operations": [],
"requires_second_review": True,
}
def test_nonmutating_repair_decision_rejects_operations() -> None:
"""A hold decision cannot smuggle in an operation the applier will ignore."""
result = _repair_result()
result["repair_decision"] = "REQUIRES_EXECUTION"
result["operations"] = [
{
"op": "REWRITE_FINAL_SUMMARY",
"target_turns": [3],
"preconditions": ["Turn 3 is an assistant summary"],
"replacement": "Honest summary",
"reason": "Align the claim with evidence",
}
]
with pytest.raises(PolicyViolation, match="must not contain"):
enforce_repair_policy(result)
def test_step_example_requires_operations() -> None:
"""Step-only salvage must include a concrete correction and truncation."""
result = _repair_result()
result["repair_decision"] = "CREATE_STEP_EXAMPLE"
result["maximum_training_use"] = "SFT_STEP_ONLY"
with pytest.raises(PolicyViolation, match="requires at least one"):
enforce_repair_policy(result)
tests/test_repair.py
-54
View File
@@ -1,54 +0,0 @@
"""Tests for deterministic static repair application and evidence preservation."""
from __future__ import annotations
from swe_data_processing.repair import apply_static_repair
def _plan() -> dict:
return {
"sample_id": "sample-1",
"repair_decision": "APPLY_STATIC_REPAIR",
"maximum_training_use": "ERROR_ANALYSIS",
"invariants": {
"resolved_unchanged": True,
"tool_outputs_unchanged": True,
"model_patch_unchanged": True,
"reference_patch_unchanged": True,
"no_synthetic_execution_result": True,
},
"operations": [
{
"op": "REWRITE_FINAL_SUMMARY",
"target_turns": [5],
"preconditions": ["Turn 5 is an assistant-only summary."],
"replacement": "The build passed, but functional behavior remains unverified.",
"reason": "Remove an unsupported success claim.",
}
],
"requires_second_review": True,
"summary": "Correct the final claim without changing execution evidence.",
}
def test_rewrite_summary_preserves_tool_output_and_patch() -> None:
record = {
"trajectory_id": "sample-1",
"resolved": -1,
"trajectory": [
{"role": "system", "content": "system"},
{"role": "user", "content": "issue"},
{"role": "assistant", "content": "build", "tool_calls": []},
{"role": "tool", "content": "build completed with exit code 0"},
{"role": "assistant", "content": "All tests pass.", "tool_calls": []},
],
"metadata": {
"model_patch": {"patch": "+++ b/src/a.py\n"},
"reference_patch": {"patch": "+++ b/src/a.py\n"},
},
}
repaired, diff = apply_static_repair(record, _plan())
assert repaired["trajectory"][3] == record["trajectory"][3]
assert repaired["trajectory"][4]["content"].startswith("The build passed")
assert repaired["metadata"] == record["metadata"]
assert diff["replaced_original_turn_ids"] == [5]
tests/test_sampling.py
+47
View File
@@ -0,0 +1,47 @@
"""Tests for deterministic review sampling."""
from __future__ import annotations
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
from swe_data_processing.sampling import iter_review_records
def test_review_reads_only_provenance_shards(tmp_path: Path) -> None:
selected_shard = tmp_path / "selected.parquet"
unrelated_shard = tmp_path / "unrelated.parquet"
pq.write_table(
pa.Table.from_pylist(
[
{"trajectory_id": "wanted", "trajectory": "selected"},
{"trajectory_id": "other", "trajectory": "same shard"},
]
),
selected_shard,
)
pq.write_table(
pa.Table.from_pylist(
[{"trajectory_id": "unrelated", "trajectory": "must not be read"}]
),
unrelated_shard,
)
selected = {"wanted": ["hard:FIVE_CONSECUTIVE_TOOL_FAILURES"]}
decision = {
"sample_id": "wanted",
"source_parquet": str(selected_shard),
"hard_reject_reasons": ["FIVE_CONSECUTIVE_TOOL_FAILURES"],
}
records = list(
iter_review_records(
tmp_path,
selected,
{"wanted": decision},
)
)
assert [record["trajectory_id"] for record in records] == ["wanted"]
assert records[0]["_qc_review"]["decision"] == decision
tests/test_summary.py
+61
View File
@@ -0,0 +1,61 @@
"""Tests for dataset aggregation and derived thresholds."""
from __future__ import annotations
import json
from pathlib import Path
from swe_data_processing.summary import build_summary, percentile
def _metric(sample_id: str, tokens: int, errors: int) -> dict:
return {
"sample_id": sample_id,
"resolved": 1,
"source_dataset": "test",
"length": {
"turn_count": 10,
"canonical_chars": 100,
"token_count": tokens,
},
"tools": {
"tool_call_count": 10,
"failed_tool_call_count": errors,
"failed_tool_call_rate": errors / 10,
"longest_consecutive_failure_run": 0,
"error_type_counts": {"nonzero_exit": errors} if errors else {},
"error_tool_counts": {"bash": errors} if errors else {},
"error_positions": {
"early_count": 0,
"early_fraction": 0.0,
"occupied_bins_5": 0,
},
},
"structure": {
"invalid_turn_count": 0,
"malformed_tool_definition_count": 0,
"malformed_tool_call_count": 0,
"unknown_tool_call_count": 0,
"missing_tool_result_count": 0,
"orphan_tool_result_count": 0,
},
}
def test_percentile_interpolates() -> None:
assert percentile([0, 10], 0.5) == 5
def test_summary_reports_token_tail_and_p99_threshold(tmp_path: Path) -> None:
path = tmp_path / "metrics.jsonl"
rows = [_metric(f"s-{index}", 10_000, index) for index in range(10)]
rows.append(_metric("long", 300_000, 0))
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
summary, threshold = build_summary(path)
assert summary["sample_count"] == 11
assert summary["length"]["buckets"]["GT_262144"]["samples"] == 1
assert summary["length"]["longest_sample_token_contribution"]["0.001"][
"token_share"
] > 0.7
assert threshold >= 8