"""End-to-end tests for call isolation in the audit workflow.""" from __future__ import annotations import json from swe_data_processing.client import GLMResponse from swe_data_processing.config import Settings from swe_data_processing.workflow import audit_trajectory class FakeClient: """Return deterministic responses while recording every API payload.""" def __init__(self, responses: list[dict]) -> None: self.settings = Settings(api_key="test-secret") self.responses = iter(responses) self.calls: list[dict] = [] def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse: self.calls.append( {"system_prompt": system_prompt, "payload": payload, "schema": schema} ) return GLMResponse( data=next(self.responses), request_id=f"request-{len(self.calls)}", usage={"total_tokens": 10}, compatibility_fallback_used=False, ) def _record() -> dict: return { "trajectory_id": "sample-1", "resolved": 0, "trajectory": [ {"role": "system", "content": "system"}, {"role": "user", "content": "Fix the bug."}, {"role": "assistant", "content": "Inspect relevant.py"}, {"role": "tool", "content": "relevant code"}, {"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"}, {"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"}, ], } def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None: client = FakeClient( [ { "sample_id": "sample-1", "decision": "TRUNCATE", "candidate_block_id": "block-001", "truncate_before_turn": 5, "prefix_safe_before_boundary": True, "category": "PERSISTENT_WRONG_IMPLEMENTATION", "severity": "MAJOR", "state_effect": "UNRECOVERED", "evidence": [{"turn_id": 5, "quote": "BAD_SUFFIX_SENTINEL patch"}], "reason": "The patch is not repaired.", }, { "sample_id": "sample-1", "prefix_valid": True, "unrecovered_major_or_critical": False, "behavior_issues": [], "dimensions": { "planning": 15, "investigation": 15, "tool_use_and_observation": 14, "progress": 12, "clarity_and_efficiency": 10, }, "evidence": [], "reason": "The prefix is useful.", }, ] ) result = audit_trajectory(_record(), client) # type: ignore[arg-type] assert len(client.calls) == 2 quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False) assert "BAD_SUFFIX_SENTINEL" not in quality_payload assert "truncate_before_turn" not in quality_payload assert result["prefix"]["retained_turn_count"] == 4 assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM" assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE" def test_hold_boundary_skips_quality_call() -> None: client = FakeClient( [ { "sample_id": "sample-1", "decision": "HOLD", "candidate_block_id": None, "truncate_before_turn": None, "prefix_safe_before_boundary": False, "category": "NONE", "severity": "NONE", "state_effect": "UNCLEAR", "evidence": [], "reason": "No clear unrepaired defect.", } ] ) result = audit_trajectory(_record(), client) # type: ignore[arg-type] assert len(client.calls) == 1 assert result["recommended_use"] == "HOLD" assert result["quality"] is None def test_keep_full_process_trajectory_is_scored() -> None: client = FakeClient( [ { "sample_id": "sample-1", "decision": "KEEP_FULL", "candidate_block_id": None, "truncate_before_turn": None, "prefix_safe_before_boundary": True, "category": "NONE", "severity": "NONE", "state_effect": "NONE", "evidence": [], "reason": "No unrepaired severe problem is visible.", }, { "sample_id": "sample-1", "prefix_valid": True, "unrecovered_major_or_critical": False, "behavior_issues": [ { "assistant_turn": 3, "kind": "INEFFICIENCY", "severity": "MINOR", "reason": "The inspection was somewhat broad.", } ], "dimensions": { "planning": 15, "investigation": 15, "tool_use_and_observation": 14, "progress": 12, "clarity_and_efficiency": 10, }, "evidence": [], "reason": "Useful despite minor inefficiency.", }, ] ) result = audit_trajectory(_record(), client) # type: ignore[arg-type] assert len(client.calls) == 2 assert result["prefix"]["retained_turn_count"] == 6 assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1 assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"