yi_lu/OpenSWETraces_cleanup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden non-success trajectory truncation

Browse Source
This commit is contained in:
yi_lu
2026-08-07 01:40:36 +08:00
parent c6c583dcee
commit ba7d619624
10 changed files with 360 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_audit.py
+65 -34
View File
@@ -8,6 +8,8 @@ import pytest
from swe_data_processing.audit import (
compute_prefix_quality,
derive_prefix_safety,
effective_boundary_policy,
materialize_prefix,
validate_boundary,
validate_prefix_quality,
@@ -38,27 +40,25 @@ def _record() -> dict:
def _boundary() -> dict:
return {
"sample_id": "sample-1",
"checks": {
"task_coverage": "COMPLETE",
"final_patch_scope": "CLEAN",
"constraints": "RESPECTED",
"claims_vs_observations": "CONSISTENT",
},
"decision": "TRUNCATE",
"candidate_block_id": "block-001",
"truncate_before_turn": 5,
"prefix_safe_before_boundary": True,
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
"severity": "MAJOR",
"state_effect": "UNRECOVERED",
"evidence": [{"turn_id": 5, "quote": "apply the harmful patch"}],
"evidence_turns": [5],
"reason": "The patch fails and is not repaired.",
}
def _blocks() -> list[dict]:
return [{"block_id": "block-001", "start_turn": 1, "end_turn": 6}]
def _quality() -> dict:
return {
"sample_id": "sample-1",
"prefix_valid": True,
"unrecovered_major_or_critical": False,
"behavior_issues": [],
"dimensions": {
"planning": 14,
@@ -67,13 +67,13 @@ def _quality() -> dict:
"progress": 12,
"clarity_and_efficiency": 10,
},
"evidence": [],
"evidence_turns": [],
"reason": "Useful investigation.",
}
def test_valid_boundary_materializes_exact_prefix() -> None:
validate_boundary(_record(), _blocks(), _boundary())
validate_boundary(_record(), _boundary())
prefix = materialize_prefix(_record(), 5)
assert prefix == _record()["trajectory"][:4]
@@ -82,47 +82,36 @@ def test_boundary_must_be_an_assistant_turn() -> None:
result = _boundary()
result["truncate_before_turn"] = 4
with pytest.raises(PolicyViolation, match="assistant turn"):
validate_boundary(_record(), _blocks(), result)
validate_boundary(_record(), result)
def test_boundary_evidence_must_quote_boundary_turn() -> None:
def test_boundary_evidence_must_include_boundary_turn() -> None:
result = _boundary()
result["evidence"] = [{"turn_id": 6, "quote": "tests failed"}]
result["evidence_turns"] = [6]
with pytest.raises(PolicyViolation, match="excluded assistant turn"):
validate_boundary(_record(), _blocks(), result)
validate_boundary(_record(), result)
def test_boundary_quote_must_be_grounded() -> None:
result = _boundary()
result["evidence"][0]["quote"] = "invented evidence"
with pytest.raises(PolicyViolation, match="not present"):
validate_boundary(_record(), _blocks(), result)
def test_keep_full_requires_safe_full_trajectory() -> None:
def test_keep_full_cannot_contain_a_boundary() -> None:
result = _boundary()
result.update(
decision="KEEP_FULL",
candidate_block_id=None,
truncate_before_turn=None,
prefix_safe_before_boundary=True,
category="NONE",
severity="NONE",
state_effect="NONE",
evidence=[],
evidence_turns=[],
)
validate_boundary(_record(), _blocks(), result)
result["prefix_safe_before_boundary"] = False
with pytest.raises(PolicyViolation, match="safe full trajectory"):
validate_boundary(_record(), _blocks(), result)
validate_boundary(_record(), result)
result["truncate_before_turn"] = 5
with pytest.raises(PolicyViolation, match="cannot contain a boundary"):
validate_boundary(_record(), result)
def test_prefix_quality_cannot_reference_suffix() -> None:
prefix = materialize_prefix(_record(), 5)
result = _quality()
result["prefix_valid"] = False
result["unrecovered_major_or_critical"] = True
result["evidence"] = [{"turn_id": 5, "quote": "harmful patch"}]
result["evidence_turns"] = [5]
with pytest.raises(PolicyViolation, match="outside the prefix"):
validate_prefix_quality("sample-1", prefix, result)
@@ -133,6 +122,48 @@ def test_quality_score_and_tier_are_computed_locally() -> None:
assert score["quality_tier"] == "MEDIUM"
def test_unrecovered_major_issue_is_locally_invalid() -> None:
result = _quality()
result["behavior_issues"] = [
{
"turn_id": 3,
"kind": "ERROR",
"severity": "MAJOR",
"recovered": False,
"reason": "The prefix leaves a known broken edit.",
}
]
result["evidence_turns"] = [3]
validate_prefix_quality("sample-1", materialize_prefix(_record(), 5), result)
assert derive_prefix_safety(result)["prefix_valid"] is False
assert compute_prefix_quality(result)["quality_tier"] == "REJECT"
@pytest.mark.parametrize("resolved", [0, -1])
def test_non_success_outcome_is_capped_before_first_stateful_turn(resolved: int) -> None:
record = _record()
record["resolved"] = resolved
record["tools"] = [
{"type": "function", "function": {"name": "str_replace_editor"}}
]
record["trajectory"][2]["tool_calls"] = [
{
"id": "call-1",
"type": "function",
"function": {
"name": "str_replace_editor",
"arguments": '{"command":"str_replace","path":"src/a.py"}',
},
}
]
result = _boundary()
result.update(decision="KEEP_FULL", truncate_before_turn=None)
policy = effective_boundary_policy(record, result)
assert policy["decision"] == "TRUNCATE"
assert policy["truncate_before_turn"] == 3
assert policy["source"] == "NON_SUCCESS_FIRST_STATEFUL_CAP"
def test_boundary_payload_hides_outcome_and_patch_metadata() -> None:
record = _record()
record["resolved"] = -1
@@ -142,7 +173,7 @@ def test_boundary_payload_hides_outcome_and_patch_metadata() -> None:
}
serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False)
assert "resolved" not in serialized
assert "MODEL_PATCH_SECRET" not in serialized
assert "MODEL_PATCH_SECRET" in serialized
assert "REFERENCE_PATCH_SECRET" not in serialized
tests/test_audit_workflow.py
+26 -17
View File
@@ -49,20 +49,22 @@ def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None:
[
{
"sample_id": "sample-1",
"checks": {
"task_coverage": "INCOMPLETE",
"final_patch_scope": "POLLUTED",
"constraints": "RESPECTED",
"claims_vs_observations": "CONTRADICTED",
},
"decision": "TRUNCATE",
"candidate_block_id": "block-001",
"truncate_before_turn": 5,
"prefix_safe_before_boundary": True,
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
"severity": "MAJOR",
"state_effect": "UNRECOVERED",
"evidence": [{"turn_id": 5, "quote": "BAD_SUFFIX_SENTINEL patch"}],
"evidence_turns": [5],
"reason": "The patch is not repaired.",
},
{
"sample_id": "sample-1",
"prefix_valid": True,
"unrecovered_major_or_critical": False,
"behavior_issues": [],
"dimensions": {
"planning": 15,
@@ -71,7 +73,7 @@ def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None:
"progress": 12,
"clarity_and_efficiency": 10,
},
"evidence": [],
"evidence_turns": [],
"reason": "The prefix is useful.",
},
]
@@ -92,14 +94,18 @@ def test_hold_boundary_skips_quality_call() -> None:
[
{
"sample_id": "sample-1",
"checks": {
"task_coverage": "COMPLETE",
"final_patch_scope": "CLEAN",
"constraints": "RESPECTED",
"claims_vs_observations": "CONSISTENT",
},
"decision": "HOLD",
"candidate_block_id": None,
"truncate_before_turn": None,
"prefix_safe_before_boundary": False,
"category": "NONE",
"severity": "NONE",
"state_effect": "UNCLEAR",
"evidence": [],
"evidence_turns": [],
"reason": "No clear unrepaired defect.",
}
]
@@ -115,25 +121,28 @@ def test_keep_full_process_trajectory_is_scored() -> None:
[
{
"sample_id": "sample-1",
"checks": {
"task_coverage": "COMPLETE",
"final_patch_scope": "CLEAN",
"constraints": "RESPECTED",
"claims_vs_observations": "CONSISTENT",
},
"decision": "KEEP_FULL",
"candidate_block_id": None,
"truncate_before_turn": None,
"prefix_safe_before_boundary": True,
"category": "NONE",
"severity": "NONE",
"state_effect": "NONE",
"evidence": [],
"evidence_turns": [],
"reason": "No unrepaired severe problem is visible.",
},
{
"sample_id": "sample-1",
"prefix_valid": True,
"unrecovered_major_or_critical": False,
"behavior_issues": [
{
"assistant_turn": 3,
"turn_id": 3,
"kind": "INEFFICIENCY",
"severity": "MINOR",
"recovered": True,
"reason": "The inspection was somewhat broad.",
}
],
@@ -144,7 +153,7 @@ def test_keep_full_process_trajectory_is_scored() -> None:
"progress": 12,
"clarity_and_efficiency": 10,
},
"evidence": [],
"evidence_turns": [],
"reason": "Useful despite minor inefficiency.",
},
]
@@ -153,4 +162,4 @@ def test_keep_full_process_trajectory_is_scored() -> None:
assert len(client.calls) == 2
assert result["prefix"]["retained_turn_count"] == 6
assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1
assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"
assert result["recommended_use"] == "HOLD"