OpenSWETraces_cleanup/tests/test_audit_workflow.py

"""End-to-end tests for call isolation in the audit workflow."""

from __future__ import annotations

import json

from swe_data_processing.client import GLMResponse
from swe_data_processing.config import Settings
from swe_data_processing.workflow import audit_trajectory


class FakeClient:
    """Return deterministic responses while recording every API payload."""

    def __init__(self, responses: list[dict]) -> None:
        self.settings = Settings(api_key="test-secret")
        self.responses = iter(responses)
        self.calls: list[dict] = []

    def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse:
        self.calls.append(
            {"system_prompt": system_prompt, "payload": payload, "schema": schema}
        )
        return GLMResponse(
            data=next(self.responses),
            request_id=f"request-{len(self.calls)}",
            usage={"total_tokens": 10},
            compatibility_fallback_used=False,
        )


def _record() -> dict:
    return {
        "trajectory_id": "sample-1",
        "resolved": 0,
        "trajectory": [
            {"role": "system", "content": "system"},
            {"role": "user", "content": "Fix the bug."},
            {"role": "assistant", "content": "Inspect relevant.py"},
            {"role": "tool", "content": "relevant code"},
            {"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"},
            {"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"},
        ],
    }


def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None:
    client = FakeClient(
        [
            {
                "sample_id": "sample-1",
                "checks": {
                    "task_coverage": "INCOMPLETE",
                    "final_patch_scope": "POLLUTED",
                    "constraints": "RESPECTED",
                    "claims_vs_observations": "CONTRADICTED",
                },
                "decision": "TRUNCATE",
                "truncate_before_turn": 5,
                "category": "PERSISTENT_WRONG_IMPLEMENTATION",
                "severity": "MAJOR",
                "state_effect": "UNRECOVERED",
                "evidence_turns": [5],
                "reason": "The patch is not repaired.",
            },
            {
                "sample_id": "sample-1",
                "behavior_issues": [],
                "dimensions": {
                    "planning": 15,
                    "investigation": 15,
                    "tool_use_and_observation": 14,
                    "progress": 12,
                    "clarity_and_efficiency": 10,
                },
                "evidence_turns": [],
                "reason": "The prefix is useful.",
            },
        ]
    )
    result = audit_trajectory(_record(), client)  # type: ignore[arg-type]

    assert len(client.calls) == 2
    quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False)
    assert "BAD_SUFFIX_SENTINEL" not in quality_payload
    assert "truncate_before_turn" not in quality_payload
    assert result["prefix"]["retained_turn_count"] == 4
    assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM"
    assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"


def test_hold_boundary_skips_quality_call() -> None:
    client = FakeClient(
        [
            {
                "sample_id": "sample-1",
                "checks": {
                    "task_coverage": "COMPLETE",
                    "final_patch_scope": "CLEAN",
                    "constraints": "RESPECTED",
                    "claims_vs_observations": "CONSISTENT",
                },
                "decision": "HOLD",
                "truncate_before_turn": None,
                "category": "NONE",
                "severity": "NONE",
                "state_effect": "UNCLEAR",
                "evidence_turns": [],
                "reason": "No clear unrepaired defect.",
            }
        ]
    )
    result = audit_trajectory(_record(), client)  # type: ignore[arg-type]
    assert len(client.calls) == 1
    assert result["recommended_use"] == "HOLD"
    assert result["quality"] is None


def test_keep_full_process_trajectory_is_scored() -> None:
    client = FakeClient(
        [
            {
                "sample_id": "sample-1",
                "checks": {
                    "task_coverage": "COMPLETE",
                    "final_patch_scope": "CLEAN",
                    "constraints": "RESPECTED",
                    "claims_vs_observations": "CONSISTENT",
                },
                "decision": "KEEP_FULL",
                "truncate_before_turn": None,
                "category": "NONE",
                "severity": "NONE",
                "state_effect": "NONE",
                "evidence_turns": [],
                "reason": "No unrepaired severe problem is visible.",
            },
            {
                "sample_id": "sample-1",
                "behavior_issues": [
                    {
                        "turn_id": 3,
                        "kind": "INEFFICIENCY",
                        "severity": "MINOR",
                        "recovered": True,
                        "reason": "The inspection was somewhat broad.",
                    }
                ],
                "dimensions": {
                    "planning": 15,
                    "investigation": 15,
                    "tool_use_and_observation": 14,
                    "progress": 12,
                    "clarity_and_efficiency": 10,
                },
                "evidence_turns": [],
                "reason": "Useful despite minor inefficiency.",
            },
        ]
    )
    result = audit_trajectory(_record(), client)  # type: ignore[arg-type]
    assert len(client.calls) == 2
    assert result["prefix"]["retained_turn_count"] == 6
    assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1
    assert result["recommended_use"] == "HOLD"