Initial Open-SWE-Traces cleanup pipeline

2026-08-06 22:53:47 +08:00
commit 044bd03f0e
35 changed files with 3638 additions and 0 deletions
@@ -0,0 +1,114 @@
+"""Tests for local trajectory-audit policy and scoring."""
+
+from __future__ import annotations
+
+import pytest
+
+from swe_data_processing.audit import compute_quality_score, validate_audit
+from swe_data_processing.policy import PolicyViolation
+from swe_data_processing.workflow import prepare_audit_payload
+
+
+def _audit_result() -> dict:
+    """Return one internally consistent audit result."""
+
+    return {
+        "evaluation_mode": "PROCESS_SALVAGE",
+        "recommended_use": "PROCESS_PREFIX_CANDIDATE",
+        "truncation": {
+            "first_bad_assistant_turn": 3,
+            "truncate_before_turn": 3,
+            "acceptable_start_turn": 3,
+            "acceptable_end_turn": 3,
+            "prefix_usable": True,
+        },
+        "behavior_issues": [
+            {
+                "assistant_turn": 3,
+                "tool_result_turn": 4,
+                "kind": "ERROR",
+                "severity": "MAJOR",
+            }
+        ],
+        "issue_counts": {
+            "errors": 1,
+            "inefficiencies": 0,
+            "critical": 0,
+            "major": 1,
+            "minor": 0,
+        },
+        "quality_dimensions": {
+            "planning": 4,
+            "tool_selection": 4,
+            "observation_use": 3,
+            "efficiency": 4,
+            "verification_discipline": 3,
+            "claim_calibration": 3,
+        },
+    }
+
+
+def _record() -> dict:
+    return {
+        "trajectory": [
+            {"role": "user", "content": "issue"},
+            {"role": "tool", "content": "context"},
+            {"role": "assistant", "content": "bad call"},
+            {"role": "tool", "content": "failed"},
+        ]
+    }
+
+
+def test_valid_audit_and_score() -> None:
+    """A consistent audit receives a deterministic bounded score."""
+
+    result = _audit_result()
+    validate_audit(_record(), result)
+    score = compute_quality_score(result)
+    assert score["educational_quality_score"] == 65.0
+    assert score["quality_tier"] == "LOW"
+
+
+def test_first_bad_turn_must_be_assistant() -> None:
+    """The truncation boundary cannot point at a tool observation."""
+
+    result = _audit_result()
+    result["truncation"]["first_bad_assistant_turn"] = 4
+    result["truncation"]["truncate_before_turn"] = 4
+    with pytest.raises(PolicyViolation, match="assistant turn"):
+        validate_audit(_record(), result)
+
+
+def test_issue_counts_are_recomputed() -> None:
+    """GLM cannot under-report the number of issues it listed."""
+
+    result = _audit_result()
+    result["issue_counts"]["errors"] = 0
+    with pytest.raises(PolicyViolation, match="issue_counts"):
+        validate_audit(_record(), result)
+
+
+def test_process_salvage_cannot_become_full_trajectory_candidate() -> None:
+    """Outcome routing cannot be overridden by the model recommendation."""
+
+    result = _audit_result()
+    result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE"
+    with pytest.raises(PolicyViolation, match="full-trajectory"):
+        validate_audit(_record(), result)
+
+
+def test_audit_payload_removes_reference_solution_signals() -> None:
+    """The causal audit cannot see reference patch text or derived metadata."""
+
+    record = {
+        "trajectory_id": "sample-1",
+        "resolved": 0,
+        "trajectory": [{"role": "user", "content": "Fix the issue"}],
+        "metadata": {
+            "model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"},
+            "reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"},
+        },
+    }
+    payload = prepare_audit_payload(record)
+    assert "reference_patch" not in payload
+    assert not any("reference" in key for key in payload["static_signals"])