115 lines
3.6 KiB
Python
115 lines
3.6 KiB
Python
"""Tests for local trajectory-audit policy and scoring."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from swe_data_processing.audit import compute_quality_score, validate_audit
|
|
from swe_data_processing.policy import PolicyViolation
|
|
from swe_data_processing.workflow import prepare_audit_payload
|
|
|
|
|
|
def _audit_result() -> dict:
|
|
"""Return one internally consistent audit result."""
|
|
|
|
return {
|
|
"evaluation_mode": "PROCESS_SALVAGE",
|
|
"recommended_use": "PROCESS_PREFIX_CANDIDATE",
|
|
"truncation": {
|
|
"first_bad_assistant_turn": 3,
|
|
"truncate_before_turn": 3,
|
|
"acceptable_start_turn": 3,
|
|
"acceptable_end_turn": 3,
|
|
"prefix_usable": True,
|
|
},
|
|
"behavior_issues": [
|
|
{
|
|
"assistant_turn": 3,
|
|
"tool_result_turn": 4,
|
|
"kind": "ERROR",
|
|
"severity": "MAJOR",
|
|
}
|
|
],
|
|
"issue_counts": {
|
|
"errors": 1,
|
|
"inefficiencies": 0,
|
|
"critical": 0,
|
|
"major": 1,
|
|
"minor": 0,
|
|
},
|
|
"quality_dimensions": {
|
|
"planning": 4,
|
|
"tool_selection": 4,
|
|
"observation_use": 3,
|
|
"efficiency": 4,
|
|
"verification_discipline": 3,
|
|
"claim_calibration": 3,
|
|
},
|
|
}
|
|
|
|
|
|
def _record() -> dict:
|
|
return {
|
|
"trajectory": [
|
|
{"role": "user", "content": "issue"},
|
|
{"role": "tool", "content": "context"},
|
|
{"role": "assistant", "content": "bad call"},
|
|
{"role": "tool", "content": "failed"},
|
|
]
|
|
}
|
|
|
|
|
|
def test_valid_audit_and_score() -> None:
|
|
"""A consistent audit receives a deterministic bounded score."""
|
|
|
|
result = _audit_result()
|
|
validate_audit(_record(), result)
|
|
score = compute_quality_score(result)
|
|
assert score["educational_quality_score"] == 65.0
|
|
assert score["quality_tier"] == "LOW"
|
|
|
|
|
|
def test_first_bad_turn_must_be_assistant() -> None:
|
|
"""The truncation boundary cannot point at a tool observation."""
|
|
|
|
result = _audit_result()
|
|
result["truncation"]["first_bad_assistant_turn"] = 4
|
|
result["truncation"]["truncate_before_turn"] = 4
|
|
with pytest.raises(PolicyViolation, match="assistant turn"):
|
|
validate_audit(_record(), result)
|
|
|
|
|
|
def test_issue_counts_are_recomputed() -> None:
|
|
"""GLM cannot under-report the number of issues it listed."""
|
|
|
|
result = _audit_result()
|
|
result["issue_counts"]["errors"] = 0
|
|
with pytest.raises(PolicyViolation, match="issue_counts"):
|
|
validate_audit(_record(), result)
|
|
|
|
|
|
def test_process_salvage_cannot_become_full_trajectory_candidate() -> None:
|
|
"""Outcome routing cannot be overridden by the model recommendation."""
|
|
|
|
result = _audit_result()
|
|
result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE"
|
|
with pytest.raises(PolicyViolation, match="full-trajectory"):
|
|
validate_audit(_record(), result)
|
|
|
|
|
|
def test_audit_payload_removes_reference_solution_signals() -> None:
|
|
"""The causal audit cannot see reference patch text or derived metadata."""
|
|
|
|
record = {
|
|
"trajectory_id": "sample-1",
|
|
"resolved": 0,
|
|
"trajectory": [{"role": "user", "content": "Fix the issue"}],
|
|
"metadata": {
|
|
"model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"},
|
|
"reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"},
|
|
},
|
|
}
|
|
payload = prepare_audit_payload(record)
|
|
assert "reference_patch" not in payload
|
|
assert not any("reference" in key for key in payload["static_signals"])
|