Replace trajectory audit with isolated prefix scoring
This commit is contained in:
+147
-101
@@ -1,114 +1,160 @@
|
||||
"""Tests for local trajectory-audit policy and scoring."""
|
||||
"""Tests for the two-call boundary and prefix-quality policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from swe_data_processing.audit import compute_quality_score, validate_audit
|
||||
from swe_data_processing.audit import (
|
||||
compute_prefix_quality,
|
||||
materialize_prefix,
|
||||
validate_boundary,
|
||||
validate_prefix_quality,
|
||||
)
|
||||
from swe_data_processing.policy import PolicyViolation
|
||||
from swe_data_processing.workflow import prepare_audit_payload
|
||||
|
||||
|
||||
def _audit_result() -> dict:
|
||||
"""Return one internally consistent audit result."""
|
||||
|
||||
return {
|
||||
"evaluation_mode": "PROCESS_SALVAGE",
|
||||
"recommended_use": "PROCESS_PREFIX_CANDIDATE",
|
||||
"truncation": {
|
||||
"first_bad_assistant_turn": 3,
|
||||
"truncate_before_turn": 3,
|
||||
"acceptable_start_turn": 3,
|
||||
"acceptable_end_turn": 3,
|
||||
"prefix_usable": True,
|
||||
},
|
||||
"behavior_issues": [
|
||||
{
|
||||
"assistant_turn": 3,
|
||||
"tool_result_turn": 4,
|
||||
"kind": "ERROR",
|
||||
"severity": "MAJOR",
|
||||
}
|
||||
],
|
||||
"issue_counts": {
|
||||
"errors": 1,
|
||||
"inefficiencies": 0,
|
||||
"critical": 0,
|
||||
"major": 1,
|
||||
"minor": 0,
|
||||
},
|
||||
"quality_dimensions": {
|
||||
"planning": 4,
|
||||
"tool_selection": 4,
|
||||
"observation_use": 3,
|
||||
"efficiency": 4,
|
||||
"verification_discipline": 3,
|
||||
"claim_calibration": 3,
|
||||
},
|
||||
}
|
||||
from swe_data_processing.workflow import (
|
||||
prepare_boundary_payload,
|
||||
prepare_prefix_quality_payload,
|
||||
)
|
||||
|
||||
|
||||
def _record() -> dict:
|
||||
return {
|
||||
"trajectory": [
|
||||
{"role": "user", "content": "issue"},
|
||||
{"role": "tool", "content": "context"},
|
||||
{"role": "assistant", "content": "bad call"},
|
||||
{"role": "tool", "content": "failed"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_valid_audit_and_score() -> None:
|
||||
"""A consistent audit receives a deterministic bounded score."""
|
||||
|
||||
result = _audit_result()
|
||||
validate_audit(_record(), result)
|
||||
score = compute_quality_score(result)
|
||||
assert score["educational_quality_score"] == 65.0
|
||||
assert score["quality_tier"] == "LOW"
|
||||
|
||||
|
||||
def test_first_bad_turn_must_be_assistant() -> None:
|
||||
"""The truncation boundary cannot point at a tool observation."""
|
||||
|
||||
result = _audit_result()
|
||||
result["truncation"]["first_bad_assistant_turn"] = 4
|
||||
result["truncation"]["truncate_before_turn"] = 4
|
||||
with pytest.raises(PolicyViolation, match="assistant turn"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_issue_counts_are_recomputed() -> None:
|
||||
"""GLM cannot under-report the number of issues it listed."""
|
||||
|
||||
result = _audit_result()
|
||||
result["issue_counts"]["errors"] = 0
|
||||
with pytest.raises(PolicyViolation, match="issue_counts"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_process_salvage_cannot_become_full_trajectory_candidate() -> None:
|
||||
"""Outcome routing cannot be overridden by the model recommendation."""
|
||||
|
||||
result = _audit_result()
|
||||
result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE"
|
||||
with pytest.raises(PolicyViolation, match="full-trajectory"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_audit_payload_removes_reference_solution_signals() -> None:
|
||||
"""The causal audit cannot see reference patch text or derived metadata."""
|
||||
|
||||
record = {
|
||||
"trajectory_id": "sample-1",
|
||||
"resolved": 0,
|
||||
"trajectory": [{"role": "user", "content": "Fix the issue"}],
|
||||
"metadata": {
|
||||
"model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"},
|
||||
"reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"},
|
||||
},
|
||||
"tools": [],
|
||||
"trajectory": [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "Fix the bug."},
|
||||
{"role": "assistant", "content": "I will inspect the code."},
|
||||
{"role": "tool", "content": "relevant.py"},
|
||||
{"role": "assistant", "content": "I will apply the harmful patch."},
|
||||
{"role": "tool", "content": "tests failed"},
|
||||
],
|
||||
}
|
||||
payload = prepare_audit_payload(record)
|
||||
assert "reference_patch" not in payload
|
||||
assert not any("reference" in key for key in payload["static_signals"])
|
||||
|
||||
|
||||
def _boundary() -> dict:
|
||||
return {
|
||||
"sample_id": "sample-1",
|
||||
"decision": "TRUNCATE",
|
||||
"candidate_block_id": "block-001",
|
||||
"truncate_before_turn": 5,
|
||||
"prefix_safe_before_boundary": True,
|
||||
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
|
||||
"severity": "MAJOR",
|
||||
"state_effect": "UNRECOVERED",
|
||||
"evidence": [{"turn_id": 5, "quote": "apply the harmful patch"}],
|
||||
"reason": "The patch fails and is not repaired.",
|
||||
}
|
||||
|
||||
|
||||
def _blocks() -> list[dict]:
|
||||
return [{"block_id": "block-001", "start_turn": 1, "end_turn": 6}]
|
||||
|
||||
|
||||
def _quality() -> dict:
|
||||
return {
|
||||
"sample_id": "sample-1",
|
||||
"prefix_valid": True,
|
||||
"unrecovered_major_or_critical": False,
|
||||
"behavior_issues": [],
|
||||
"dimensions": {
|
||||
"planning": 14,
|
||||
"investigation": 15,
|
||||
"tool_use_and_observation": 14,
|
||||
"progress": 12,
|
||||
"clarity_and_efficiency": 10,
|
||||
},
|
||||
"evidence": [],
|
||||
"reason": "Useful investigation.",
|
||||
}
|
||||
|
||||
|
||||
def test_valid_boundary_materializes_exact_prefix() -> None:
|
||||
validate_boundary(_record(), _blocks(), _boundary())
|
||||
prefix = materialize_prefix(_record(), 5)
|
||||
assert prefix == _record()["trajectory"][:4]
|
||||
|
||||
|
||||
def test_boundary_must_be_an_assistant_turn() -> None:
|
||||
result = _boundary()
|
||||
result["truncate_before_turn"] = 4
|
||||
with pytest.raises(PolicyViolation, match="assistant turn"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_boundary_evidence_must_quote_boundary_turn() -> None:
|
||||
result = _boundary()
|
||||
result["evidence"] = [{"turn_id": 6, "quote": "tests failed"}]
|
||||
with pytest.raises(PolicyViolation, match="excluded assistant turn"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_boundary_quote_must_be_grounded() -> None:
|
||||
result = _boundary()
|
||||
result["evidence"][0]["quote"] = "invented evidence"
|
||||
with pytest.raises(PolicyViolation, match="not present"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_keep_full_requires_safe_full_trajectory() -> None:
|
||||
result = _boundary()
|
||||
result.update(
|
||||
decision="KEEP_FULL",
|
||||
candidate_block_id=None,
|
||||
truncate_before_turn=None,
|
||||
prefix_safe_before_boundary=True,
|
||||
category="NONE",
|
||||
severity="NONE",
|
||||
state_effect="NONE",
|
||||
evidence=[],
|
||||
)
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
result["prefix_safe_before_boundary"] = False
|
||||
with pytest.raises(PolicyViolation, match="safe full trajectory"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_prefix_quality_cannot_reference_suffix() -> None:
|
||||
prefix = materialize_prefix(_record(), 5)
|
||||
result = _quality()
|
||||
result["prefix_valid"] = False
|
||||
result["unrecovered_major_or_critical"] = True
|
||||
result["evidence"] = [{"turn_id": 5, "quote": "harmful patch"}]
|
||||
with pytest.raises(PolicyViolation, match="outside the prefix"):
|
||||
validate_prefix_quality("sample-1", prefix, result)
|
||||
|
||||
|
||||
def test_quality_score_and_tier_are_computed_locally() -> None:
|
||||
score = compute_prefix_quality(_quality())
|
||||
assert score["educational_quality_score"] == 65
|
||||
assert score["quality_tier"] == "MEDIUM"
|
||||
|
||||
|
||||
def test_boundary_payload_hides_outcome_and_patch_metadata() -> None:
|
||||
record = _record()
|
||||
record["resolved"] = -1
|
||||
record["metadata"] = {
|
||||
"model_patch": {"patch": "MODEL_PATCH_SECRET"},
|
||||
"reference_patch": {"patch": "REFERENCE_PATCH_SECRET"},
|
||||
}
|
||||
serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False)
|
||||
assert "resolved" not in serialized
|
||||
assert "MODEL_PATCH_SECRET" not in serialized
|
||||
assert "REFERENCE_PATCH_SECRET" not in serialized
|
||||
|
||||
|
||||
def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None:
|
||||
first = _record()
|
||||
second = _record()
|
||||
second["resolved"] = -1
|
||||
second["trajectory"][4]["content"] = "DIFFERENT SUFFIX"
|
||||
second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT"
|
||||
second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}}
|
||||
first_prefix = materialize_prefix(first, 5)
|
||||
second_prefix = materialize_prefix(second, 5)
|
||||
assert prepare_prefix_quality_payload(
|
||||
first, first_prefix
|
||||
) == prepare_prefix_quality_payload(second, second_prefix)
|
||||
|
||||
Reference in New Issue
Block a user