"""Tests for the two-call boundary and prefix-quality policy.""" from __future__ import annotations import json import pytest from swe_data_processing.audit import ( compute_prefix_quality, materialize_prefix, validate_boundary, validate_prefix_quality, ) from swe_data_processing.policy import PolicyViolation from swe_data_processing.workflow import ( prepare_boundary_payload, prepare_prefix_quality_payload, ) def _record() -> dict: return { "trajectory_id": "sample-1", "resolved": 0, "tools": [], "trajectory": [ {"role": "system", "content": "system"}, {"role": "user", "content": "Fix the bug."}, {"role": "assistant", "content": "I will inspect the code."}, {"role": "tool", "content": "relevant.py"}, {"role": "assistant", "content": "I will apply the harmful patch."}, {"role": "tool", "content": "tests failed"}, ], } def _boundary() -> dict: return { "sample_id": "sample-1", "decision": "TRUNCATE", "candidate_block_id": "block-001", "truncate_before_turn": 5, "prefix_safe_before_boundary": True, "category": "PERSISTENT_WRONG_IMPLEMENTATION", "severity": "MAJOR", "state_effect": "UNRECOVERED", "evidence": [{"turn_id": 5, "quote": "apply the harmful patch"}], "reason": "The patch fails and is not repaired.", } def _blocks() -> list[dict]: return [{"block_id": "block-001", "start_turn": 1, "end_turn": 6}] def _quality() -> dict: return { "sample_id": "sample-1", "prefix_valid": True, "unrecovered_major_or_critical": False, "behavior_issues": [], "dimensions": { "planning": 14, "investigation": 15, "tool_use_and_observation": 14, "progress": 12, "clarity_and_efficiency": 10, }, "evidence": [], "reason": "Useful investigation.", } def test_valid_boundary_materializes_exact_prefix() -> None: validate_boundary(_record(), _blocks(), _boundary()) prefix = materialize_prefix(_record(), 5) assert prefix == _record()["trajectory"][:4] def test_boundary_must_be_an_assistant_turn() -> None: result = _boundary() result["truncate_before_turn"] = 4 with pytest.raises(PolicyViolation, match="assistant turn"): validate_boundary(_record(), _blocks(), result) def test_boundary_evidence_must_quote_boundary_turn() -> None: result = _boundary() result["evidence"] = [{"turn_id": 6, "quote": "tests failed"}] with pytest.raises(PolicyViolation, match="excluded assistant turn"): validate_boundary(_record(), _blocks(), result) def test_boundary_quote_must_be_grounded() -> None: result = _boundary() result["evidence"][0]["quote"] = "invented evidence" with pytest.raises(PolicyViolation, match="not present"): validate_boundary(_record(), _blocks(), result) def test_keep_full_requires_safe_full_trajectory() -> None: result = _boundary() result.update( decision="KEEP_FULL", candidate_block_id=None, truncate_before_turn=None, prefix_safe_before_boundary=True, category="NONE", severity="NONE", state_effect="NONE", evidence=[], ) validate_boundary(_record(), _blocks(), result) result["prefix_safe_before_boundary"] = False with pytest.raises(PolicyViolation, match="safe full trajectory"): validate_boundary(_record(), _blocks(), result) def test_prefix_quality_cannot_reference_suffix() -> None: prefix = materialize_prefix(_record(), 5) result = _quality() result["prefix_valid"] = False result["unrecovered_major_or_critical"] = True result["evidence"] = [{"turn_id": 5, "quote": "harmful patch"}] with pytest.raises(PolicyViolation, match="outside the prefix"): validate_prefix_quality("sample-1", prefix, result) def test_quality_score_and_tier_are_computed_locally() -> None: score = compute_prefix_quality(_quality()) assert score["educational_quality_score"] == 65 assert score["quality_tier"] == "MEDIUM" def test_boundary_payload_hides_outcome_and_patch_metadata() -> None: record = _record() record["resolved"] = -1 record["metadata"] = { "model_patch": {"patch": "MODEL_PATCH_SECRET"}, "reference_patch": {"patch": "REFERENCE_PATCH_SECRET"}, } serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False) assert "resolved" not in serialized assert "MODEL_PATCH_SECRET" not in serialized assert "REFERENCE_PATCH_SECRET" not in serialized def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None: first = _record() second = _record() second["resolved"] = -1 second["trajectory"][4]["content"] = "DIFFERENT SUFFIX" second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT" second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}} first_prefix = materialize_prefix(first, 5) second_prefix = materialize_prefix(second, 5) assert prepare_prefix_quality_payload( first, first_prefix ) == prepare_prefix_quality_payload(second, second_prefix)