Replace trajectory audit with isolated prefix scoring

2026-08-07 00:47:31 +08:00
parent f1a090e8c4
commit c6c583dcee
18 changed files with 914 additions and 469 deletions
@@ -0,0 +1,156 @@
+"""End-to-end tests for call isolation in the audit workflow."""
+
+from __future__ import annotations
+
+import json
+
+from swe_data_processing.client import GLMResponse
+from swe_data_processing.config import Settings
+from swe_data_processing.workflow import audit_trajectory
+
+
+class FakeClient:
+    """Return deterministic responses while recording every API payload."""
+
+    def __init__(self, responses: list[dict]) -> None:
+        self.settings = Settings(api_key="test-secret")
+        self.responses = iter(responses)
+        self.calls: list[dict] = []
+
+    def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse:
+        self.calls.append(
+            {"system_prompt": system_prompt, "payload": payload, "schema": schema}
+        )
+        return GLMResponse(
+            data=next(self.responses),
+            request_id=f"request-{len(self.calls)}",
+            usage={"total_tokens": 10},
+            compatibility_fallback_used=False,
+        )
+
+
+def _record() -> dict:
+    return {
+        "trajectory_id": "sample-1",
+        "resolved": 0,
+        "trajectory": [
+            {"role": "system", "content": "system"},
+            {"role": "user", "content": "Fix the bug."},
+            {"role": "assistant", "content": "Inspect relevant.py"},
+            {"role": "tool", "content": "relevant code"},
+            {"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"},
+            {"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"},
+        ],
+    }
+
+
+def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None:
+    client = FakeClient(
+        [
+            {
+                "sample_id": "sample-1",
+                "decision": "TRUNCATE",
+                "candidate_block_id": "block-001",
+                "truncate_before_turn": 5,
+                "prefix_safe_before_boundary": True,
+                "category": "PERSISTENT_WRONG_IMPLEMENTATION",
+                "severity": "MAJOR",
+                "state_effect": "UNRECOVERED",
+                "evidence": [{"turn_id": 5, "quote": "BAD_SUFFIX_SENTINEL patch"}],
+                "reason": "The patch is not repaired.",
+            },
+            {
+                "sample_id": "sample-1",
+                "prefix_valid": True,
+                "unrecovered_major_or_critical": False,
+                "behavior_issues": [],
+                "dimensions": {
+                    "planning": 15,
+                    "investigation": 15,
+                    "tool_use_and_observation": 14,
+                    "progress": 12,
+                    "clarity_and_efficiency": 10,
+                },
+                "evidence": [],
+                "reason": "The prefix is useful.",
+            },
+        ]
+    )
+    result = audit_trajectory(_record(), client)  # type: ignore[arg-type]
+
+    assert len(client.calls) == 2
+    quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False)
+    assert "BAD_SUFFIX_SENTINEL" not in quality_payload
+    assert "truncate_before_turn" not in quality_payload
+    assert result["prefix"]["retained_turn_count"] == 4
+    assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM"
+    assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"
+
+
+def test_hold_boundary_skips_quality_call() -> None:
+    client = FakeClient(
+        [
+            {
+                "sample_id": "sample-1",
+                "decision": "HOLD",
+                "candidate_block_id": None,
+                "truncate_before_turn": None,
+                "prefix_safe_before_boundary": False,
+                "category": "NONE",
+                "severity": "NONE",
+                "state_effect": "UNCLEAR",
+                "evidence": [],
+                "reason": "No clear unrepaired defect.",
+            }
+        ]
+    )
+    result = audit_trajectory(_record(), client)  # type: ignore[arg-type]
+    assert len(client.calls) == 1
+    assert result["recommended_use"] == "HOLD"
+    assert result["quality"] is None
+
+
+def test_keep_full_process_trajectory_is_scored() -> None:
+    client = FakeClient(
+        [
+            {
+                "sample_id": "sample-1",
+                "decision": "KEEP_FULL",
+                "candidate_block_id": None,
+                "truncate_before_turn": None,
+                "prefix_safe_before_boundary": True,
+                "category": "NONE",
+                "severity": "NONE",
+                "state_effect": "NONE",
+                "evidence": [],
+                "reason": "No unrepaired severe problem is visible.",
+            },
+            {
+                "sample_id": "sample-1",
+                "prefix_valid": True,
+                "unrecovered_major_or_critical": False,
+                "behavior_issues": [
+                    {
+                        "assistant_turn": 3,
+                        "kind": "INEFFICIENCY",
+                        "severity": "MINOR",
+                        "reason": "The inspection was somewhat broad.",
+                    }
+                ],
+                "dimensions": {
+                    "planning": 15,
+                    "investigation": 15,
+                    "tool_use_and_observation": 14,
+                    "progress": 12,
+                    "clarity_and_efficiency": 10,
+                },
+                "evidence": [],
+                "reason": "Useful despite minor inefficiency.",
+            },
+        ]
+    )
+    result = audit_trajectory(_record(), client)  # type: ignore[arg-type]
+    assert len(client.calls) == 2
+    assert result["prefix"]["retained_turn_count"] == 6
+    assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1
+    assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"