161 lines
5.2 KiB
Python
161 lines
5.2 KiB
Python
"""Tests for the two-call boundary and prefix-quality policy."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from swe_data_processing.audit import (
|
|
compute_prefix_quality,
|
|
materialize_prefix,
|
|
validate_boundary,
|
|
validate_prefix_quality,
|
|
)
|
|
from swe_data_processing.policy import PolicyViolation
|
|
from swe_data_processing.workflow import (
|
|
prepare_boundary_payload,
|
|
prepare_prefix_quality_payload,
|
|
)
|
|
|
|
|
|
def _record() -> dict:
|
|
return {
|
|
"trajectory_id": "sample-1",
|
|
"resolved": 0,
|
|
"tools": [],
|
|
"trajectory": [
|
|
{"role": "system", "content": "system"},
|
|
{"role": "user", "content": "Fix the bug."},
|
|
{"role": "assistant", "content": "I will inspect the code."},
|
|
{"role": "tool", "content": "relevant.py"},
|
|
{"role": "assistant", "content": "I will apply the harmful patch."},
|
|
{"role": "tool", "content": "tests failed"},
|
|
],
|
|
}
|
|
|
|
|
|
def _boundary() -> dict:
|
|
return {
|
|
"sample_id": "sample-1",
|
|
"decision": "TRUNCATE",
|
|
"candidate_block_id": "block-001",
|
|
"truncate_before_turn": 5,
|
|
"prefix_safe_before_boundary": True,
|
|
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
|
|
"severity": "MAJOR",
|
|
"state_effect": "UNRECOVERED",
|
|
"evidence": [{"turn_id": 5, "quote": "apply the harmful patch"}],
|
|
"reason": "The patch fails and is not repaired.",
|
|
}
|
|
|
|
|
|
def _blocks() -> list[dict]:
|
|
return [{"block_id": "block-001", "start_turn": 1, "end_turn": 6}]
|
|
|
|
|
|
def _quality() -> dict:
|
|
return {
|
|
"sample_id": "sample-1",
|
|
"prefix_valid": True,
|
|
"unrecovered_major_or_critical": False,
|
|
"behavior_issues": [],
|
|
"dimensions": {
|
|
"planning": 14,
|
|
"investigation": 15,
|
|
"tool_use_and_observation": 14,
|
|
"progress": 12,
|
|
"clarity_and_efficiency": 10,
|
|
},
|
|
"evidence": [],
|
|
"reason": "Useful investigation.",
|
|
}
|
|
|
|
|
|
def test_valid_boundary_materializes_exact_prefix() -> None:
|
|
validate_boundary(_record(), _blocks(), _boundary())
|
|
prefix = materialize_prefix(_record(), 5)
|
|
assert prefix == _record()["trajectory"][:4]
|
|
|
|
|
|
def test_boundary_must_be_an_assistant_turn() -> None:
|
|
result = _boundary()
|
|
result["truncate_before_turn"] = 4
|
|
with pytest.raises(PolicyViolation, match="assistant turn"):
|
|
validate_boundary(_record(), _blocks(), result)
|
|
|
|
|
|
def test_boundary_evidence_must_quote_boundary_turn() -> None:
|
|
result = _boundary()
|
|
result["evidence"] = [{"turn_id": 6, "quote": "tests failed"}]
|
|
with pytest.raises(PolicyViolation, match="excluded assistant turn"):
|
|
validate_boundary(_record(), _blocks(), result)
|
|
|
|
|
|
def test_boundary_quote_must_be_grounded() -> None:
|
|
result = _boundary()
|
|
result["evidence"][0]["quote"] = "invented evidence"
|
|
with pytest.raises(PolicyViolation, match="not present"):
|
|
validate_boundary(_record(), _blocks(), result)
|
|
|
|
|
|
def test_keep_full_requires_safe_full_trajectory() -> None:
|
|
result = _boundary()
|
|
result.update(
|
|
decision="KEEP_FULL",
|
|
candidate_block_id=None,
|
|
truncate_before_turn=None,
|
|
prefix_safe_before_boundary=True,
|
|
category="NONE",
|
|
severity="NONE",
|
|
state_effect="NONE",
|
|
evidence=[],
|
|
)
|
|
validate_boundary(_record(), _blocks(), result)
|
|
result["prefix_safe_before_boundary"] = False
|
|
with pytest.raises(PolicyViolation, match="safe full trajectory"):
|
|
validate_boundary(_record(), _blocks(), result)
|
|
|
|
|
|
def test_prefix_quality_cannot_reference_suffix() -> None:
|
|
prefix = materialize_prefix(_record(), 5)
|
|
result = _quality()
|
|
result["prefix_valid"] = False
|
|
result["unrecovered_major_or_critical"] = True
|
|
result["evidence"] = [{"turn_id": 5, "quote": "harmful patch"}]
|
|
with pytest.raises(PolicyViolation, match="outside the prefix"):
|
|
validate_prefix_quality("sample-1", prefix, result)
|
|
|
|
|
|
def test_quality_score_and_tier_are_computed_locally() -> None:
|
|
score = compute_prefix_quality(_quality())
|
|
assert score["educational_quality_score"] == 65
|
|
assert score["quality_tier"] == "MEDIUM"
|
|
|
|
|
|
def test_boundary_payload_hides_outcome_and_patch_metadata() -> None:
|
|
record = _record()
|
|
record["resolved"] = -1
|
|
record["metadata"] = {
|
|
"model_patch": {"patch": "MODEL_PATCH_SECRET"},
|
|
"reference_patch": {"patch": "REFERENCE_PATCH_SECRET"},
|
|
}
|
|
serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False)
|
|
assert "resolved" not in serialized
|
|
assert "MODEL_PATCH_SECRET" not in serialized
|
|
assert "REFERENCE_PATCH_SECRET" not in serialized
|
|
|
|
|
|
def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None:
|
|
first = _record()
|
|
second = _record()
|
|
second["resolved"] = -1
|
|
second["trajectory"][4]["content"] = "DIFFERENT SUFFIX"
|
|
second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT"
|
|
second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}}
|
|
first_prefix = materialize_prefix(first, 5)
|
|
second_prefix = materialize_prefix(second, 5)
|
|
assert prepare_prefix_quality_payload(
|
|
first, first_prefix
|
|
) == prepare_prefix_quality_payload(second, second_prefix)
|