"""Tests for local trajectory-audit policy and scoring.""" from __future__ import annotations import pytest from swe_data_processing.audit import compute_quality_score, validate_audit from swe_data_processing.policy import PolicyViolation from swe_data_processing.workflow import prepare_audit_payload def _audit_result() -> dict: """Return one internally consistent audit result.""" return { "evaluation_mode": "PROCESS_SALVAGE", "recommended_use": "PROCESS_PREFIX_CANDIDATE", "truncation": { "first_bad_assistant_turn": 3, "truncate_before_turn": 3, "acceptable_start_turn": 3, "acceptable_end_turn": 3, "prefix_usable": True, }, "behavior_issues": [ { "assistant_turn": 3, "tool_result_turn": 4, "kind": "ERROR", "severity": "MAJOR", } ], "issue_counts": { "errors": 1, "inefficiencies": 0, "critical": 0, "major": 1, "minor": 0, }, "quality_dimensions": { "planning": 4, "tool_selection": 4, "observation_use": 3, "efficiency": 4, "verification_discipline": 3, "claim_calibration": 3, }, } def _record() -> dict: return { "trajectory": [ {"role": "user", "content": "issue"}, {"role": "tool", "content": "context"}, {"role": "assistant", "content": "bad call"}, {"role": "tool", "content": "failed"}, ] } def test_valid_audit_and_score() -> None: """A consistent audit receives a deterministic bounded score.""" result = _audit_result() validate_audit(_record(), result) score = compute_quality_score(result) assert score["educational_quality_score"] == 65.0 assert score["quality_tier"] == "LOW" def test_first_bad_turn_must_be_assistant() -> None: """The truncation boundary cannot point at a tool observation.""" result = _audit_result() result["truncation"]["first_bad_assistant_turn"] = 4 result["truncation"]["truncate_before_turn"] = 4 with pytest.raises(PolicyViolation, match="assistant turn"): validate_audit(_record(), result) def test_issue_counts_are_recomputed() -> None: """GLM cannot under-report the number of issues it listed.""" result = _audit_result() result["issue_counts"]["errors"] = 0 with pytest.raises(PolicyViolation, match="issue_counts"): validate_audit(_record(), result) def test_process_salvage_cannot_become_full_trajectory_candidate() -> None: """Outcome routing cannot be overridden by the model recommendation.""" result = _audit_result() result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE" with pytest.raises(PolicyViolation, match="full-trajectory"): validate_audit(_record(), result) def test_audit_payload_removes_reference_solution_signals() -> None: """The causal audit cannot see reference patch text or derived metadata.""" record = { "trajectory_id": "sample-1", "resolved": 0, "trajectory": [{"role": "user", "content": "Fix the issue"}], "metadata": { "model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"}, "reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"}, }, } payload = prepare_audit_payload(record) assert "reference_patch" not in payload assert not any("reference" in key for key in payload["static_signals"])