Replace trajectory audit with isolated prefix scoring
This commit is contained in:
+147
-101
@@ -1,114 +1,160 @@
|
||||
"""Tests for local trajectory-audit policy and scoring."""
|
||||
"""Tests for the two-call boundary and prefix-quality policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from swe_data_processing.audit import compute_quality_score, validate_audit
|
||||
from swe_data_processing.audit import (
|
||||
compute_prefix_quality,
|
||||
materialize_prefix,
|
||||
validate_boundary,
|
||||
validate_prefix_quality,
|
||||
)
|
||||
from swe_data_processing.policy import PolicyViolation
|
||||
from swe_data_processing.workflow import prepare_audit_payload
|
||||
|
||||
|
||||
def _audit_result() -> dict:
|
||||
"""Return one internally consistent audit result."""
|
||||
|
||||
return {
|
||||
"evaluation_mode": "PROCESS_SALVAGE",
|
||||
"recommended_use": "PROCESS_PREFIX_CANDIDATE",
|
||||
"truncation": {
|
||||
"first_bad_assistant_turn": 3,
|
||||
"truncate_before_turn": 3,
|
||||
"acceptable_start_turn": 3,
|
||||
"acceptable_end_turn": 3,
|
||||
"prefix_usable": True,
|
||||
},
|
||||
"behavior_issues": [
|
||||
{
|
||||
"assistant_turn": 3,
|
||||
"tool_result_turn": 4,
|
||||
"kind": "ERROR",
|
||||
"severity": "MAJOR",
|
||||
}
|
||||
],
|
||||
"issue_counts": {
|
||||
"errors": 1,
|
||||
"inefficiencies": 0,
|
||||
"critical": 0,
|
||||
"major": 1,
|
||||
"minor": 0,
|
||||
},
|
||||
"quality_dimensions": {
|
||||
"planning": 4,
|
||||
"tool_selection": 4,
|
||||
"observation_use": 3,
|
||||
"efficiency": 4,
|
||||
"verification_discipline": 3,
|
||||
"claim_calibration": 3,
|
||||
},
|
||||
}
|
||||
from swe_data_processing.workflow import (
|
||||
prepare_boundary_payload,
|
||||
prepare_prefix_quality_payload,
|
||||
)
|
||||
|
||||
|
||||
def _record() -> dict:
|
||||
return {
|
||||
"trajectory": [
|
||||
{"role": "user", "content": "issue"},
|
||||
{"role": "tool", "content": "context"},
|
||||
{"role": "assistant", "content": "bad call"},
|
||||
{"role": "tool", "content": "failed"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_valid_audit_and_score() -> None:
|
||||
"""A consistent audit receives a deterministic bounded score."""
|
||||
|
||||
result = _audit_result()
|
||||
validate_audit(_record(), result)
|
||||
score = compute_quality_score(result)
|
||||
assert score["educational_quality_score"] == 65.0
|
||||
assert score["quality_tier"] == "LOW"
|
||||
|
||||
|
||||
def test_first_bad_turn_must_be_assistant() -> None:
|
||||
"""The truncation boundary cannot point at a tool observation."""
|
||||
|
||||
result = _audit_result()
|
||||
result["truncation"]["first_bad_assistant_turn"] = 4
|
||||
result["truncation"]["truncate_before_turn"] = 4
|
||||
with pytest.raises(PolicyViolation, match="assistant turn"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_issue_counts_are_recomputed() -> None:
|
||||
"""GLM cannot under-report the number of issues it listed."""
|
||||
|
||||
result = _audit_result()
|
||||
result["issue_counts"]["errors"] = 0
|
||||
with pytest.raises(PolicyViolation, match="issue_counts"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_process_salvage_cannot_become_full_trajectory_candidate() -> None:
|
||||
"""Outcome routing cannot be overridden by the model recommendation."""
|
||||
|
||||
result = _audit_result()
|
||||
result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE"
|
||||
with pytest.raises(PolicyViolation, match="full-trajectory"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_audit_payload_removes_reference_solution_signals() -> None:
|
||||
"""The causal audit cannot see reference patch text or derived metadata."""
|
||||
|
||||
record = {
|
||||
"trajectory_id": "sample-1",
|
||||
"resolved": 0,
|
||||
"trajectory": [{"role": "user", "content": "Fix the issue"}],
|
||||
"metadata": {
|
||||
"model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"},
|
||||
"reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"},
|
||||
},
|
||||
"tools": [],
|
||||
"trajectory": [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "Fix the bug."},
|
||||
{"role": "assistant", "content": "I will inspect the code."},
|
||||
{"role": "tool", "content": "relevant.py"},
|
||||
{"role": "assistant", "content": "I will apply the harmful patch."},
|
||||
{"role": "tool", "content": "tests failed"},
|
||||
],
|
||||
}
|
||||
payload = prepare_audit_payload(record)
|
||||
assert "reference_patch" not in payload
|
||||
assert not any("reference" in key for key in payload["static_signals"])
|
||||
|
||||
|
||||
def _boundary() -> dict:
|
||||
return {
|
||||
"sample_id": "sample-1",
|
||||
"decision": "TRUNCATE",
|
||||
"candidate_block_id": "block-001",
|
||||
"truncate_before_turn": 5,
|
||||
"prefix_safe_before_boundary": True,
|
||||
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
|
||||
"severity": "MAJOR",
|
||||
"state_effect": "UNRECOVERED",
|
||||
"evidence": [{"turn_id": 5, "quote": "apply the harmful patch"}],
|
||||
"reason": "The patch fails and is not repaired.",
|
||||
}
|
||||
|
||||
|
||||
def _blocks() -> list[dict]:
|
||||
return [{"block_id": "block-001", "start_turn": 1, "end_turn": 6}]
|
||||
|
||||
|
||||
def _quality() -> dict:
|
||||
return {
|
||||
"sample_id": "sample-1",
|
||||
"prefix_valid": True,
|
||||
"unrecovered_major_or_critical": False,
|
||||
"behavior_issues": [],
|
||||
"dimensions": {
|
||||
"planning": 14,
|
||||
"investigation": 15,
|
||||
"tool_use_and_observation": 14,
|
||||
"progress": 12,
|
||||
"clarity_and_efficiency": 10,
|
||||
},
|
||||
"evidence": [],
|
||||
"reason": "Useful investigation.",
|
||||
}
|
||||
|
||||
|
||||
def test_valid_boundary_materializes_exact_prefix() -> None:
|
||||
validate_boundary(_record(), _blocks(), _boundary())
|
||||
prefix = materialize_prefix(_record(), 5)
|
||||
assert prefix == _record()["trajectory"][:4]
|
||||
|
||||
|
||||
def test_boundary_must_be_an_assistant_turn() -> None:
|
||||
result = _boundary()
|
||||
result["truncate_before_turn"] = 4
|
||||
with pytest.raises(PolicyViolation, match="assistant turn"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_boundary_evidence_must_quote_boundary_turn() -> None:
|
||||
result = _boundary()
|
||||
result["evidence"] = [{"turn_id": 6, "quote": "tests failed"}]
|
||||
with pytest.raises(PolicyViolation, match="excluded assistant turn"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_boundary_quote_must_be_grounded() -> None:
|
||||
result = _boundary()
|
||||
result["evidence"][0]["quote"] = "invented evidence"
|
||||
with pytest.raises(PolicyViolation, match="not present"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_keep_full_requires_safe_full_trajectory() -> None:
|
||||
result = _boundary()
|
||||
result.update(
|
||||
decision="KEEP_FULL",
|
||||
candidate_block_id=None,
|
||||
truncate_before_turn=None,
|
||||
prefix_safe_before_boundary=True,
|
||||
category="NONE",
|
||||
severity="NONE",
|
||||
state_effect="NONE",
|
||||
evidence=[],
|
||||
)
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
result["prefix_safe_before_boundary"] = False
|
||||
with pytest.raises(PolicyViolation, match="safe full trajectory"):
|
||||
validate_boundary(_record(), _blocks(), result)
|
||||
|
||||
|
||||
def test_prefix_quality_cannot_reference_suffix() -> None:
|
||||
prefix = materialize_prefix(_record(), 5)
|
||||
result = _quality()
|
||||
result["prefix_valid"] = False
|
||||
result["unrecovered_major_or_critical"] = True
|
||||
result["evidence"] = [{"turn_id": 5, "quote": "harmful patch"}]
|
||||
with pytest.raises(PolicyViolation, match="outside the prefix"):
|
||||
validate_prefix_quality("sample-1", prefix, result)
|
||||
|
||||
|
||||
def test_quality_score_and_tier_are_computed_locally() -> None:
|
||||
score = compute_prefix_quality(_quality())
|
||||
assert score["educational_quality_score"] == 65
|
||||
assert score["quality_tier"] == "MEDIUM"
|
||||
|
||||
|
||||
def test_boundary_payload_hides_outcome_and_patch_metadata() -> None:
|
||||
record = _record()
|
||||
record["resolved"] = -1
|
||||
record["metadata"] = {
|
||||
"model_patch": {"patch": "MODEL_PATCH_SECRET"},
|
||||
"reference_patch": {"patch": "REFERENCE_PATCH_SECRET"},
|
||||
}
|
||||
serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False)
|
||||
assert "resolved" not in serialized
|
||||
assert "MODEL_PATCH_SECRET" not in serialized
|
||||
assert "REFERENCE_PATCH_SECRET" not in serialized
|
||||
|
||||
|
||||
def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None:
|
||||
first = _record()
|
||||
second = _record()
|
||||
second["resolved"] = -1
|
||||
second["trajectory"][4]["content"] = "DIFFERENT SUFFIX"
|
||||
second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT"
|
||||
second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}}
|
||||
first_prefix = materialize_prefix(first, 5)
|
||||
second_prefix = materialize_prefix(second, 5)
|
||||
assert prepare_prefix_quality_payload(
|
||||
first, first_prefix
|
||||
) == prepare_prefix_quality_payload(second, second_prefix)
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""End-to-end tests for call isolation in the audit workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from swe_data_processing.client import GLMResponse
|
||||
from swe_data_processing.config import Settings
|
||||
from swe_data_processing.workflow import audit_trajectory
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""Return deterministic responses while recording every API payload."""
|
||||
|
||||
def __init__(self, responses: list[dict]) -> None:
|
||||
self.settings = Settings(api_key="test-secret")
|
||||
self.responses = iter(responses)
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse:
|
||||
self.calls.append(
|
||||
{"system_prompt": system_prompt, "payload": payload, "schema": schema}
|
||||
)
|
||||
return GLMResponse(
|
||||
data=next(self.responses),
|
||||
request_id=f"request-{len(self.calls)}",
|
||||
usage={"total_tokens": 10},
|
||||
compatibility_fallback_used=False,
|
||||
)
|
||||
|
||||
|
||||
def _record() -> dict:
|
||||
return {
|
||||
"trajectory_id": "sample-1",
|
||||
"resolved": 0,
|
||||
"trajectory": [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "Fix the bug."},
|
||||
{"role": "assistant", "content": "Inspect relevant.py"},
|
||||
{"role": "tool", "content": "relevant code"},
|
||||
{"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"},
|
||||
{"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None:
|
||||
client = FakeClient(
|
||||
[
|
||||
{
|
||||
"sample_id": "sample-1",
|
||||
"decision": "TRUNCATE",
|
||||
"candidate_block_id": "block-001",
|
||||
"truncate_before_turn": 5,
|
||||
"prefix_safe_before_boundary": True,
|
||||
"category": "PERSISTENT_WRONG_IMPLEMENTATION",
|
||||
"severity": "MAJOR",
|
||||
"state_effect": "UNRECOVERED",
|
||||
"evidence": [{"turn_id": 5, "quote": "BAD_SUFFIX_SENTINEL patch"}],
|
||||
"reason": "The patch is not repaired.",
|
||||
},
|
||||
{
|
||||
"sample_id": "sample-1",
|
||||
"prefix_valid": True,
|
||||
"unrecovered_major_or_critical": False,
|
||||
"behavior_issues": [],
|
||||
"dimensions": {
|
||||
"planning": 15,
|
||||
"investigation": 15,
|
||||
"tool_use_and_observation": 14,
|
||||
"progress": 12,
|
||||
"clarity_and_efficiency": 10,
|
||||
},
|
||||
"evidence": [],
|
||||
"reason": "The prefix is useful.",
|
||||
},
|
||||
]
|
||||
)
|
||||
result = audit_trajectory(_record(), client) # type: ignore[arg-type]
|
||||
|
||||
assert len(client.calls) == 2
|
||||
quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False)
|
||||
assert "BAD_SUFFIX_SENTINEL" not in quality_payload
|
||||
assert "truncate_before_turn" not in quality_payload
|
||||
assert result["prefix"]["retained_turn_count"] == 4
|
||||
assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM"
|
||||
assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"
|
||||
|
||||
|
||||
def test_hold_boundary_skips_quality_call() -> None:
|
||||
client = FakeClient(
|
||||
[
|
||||
{
|
||||
"sample_id": "sample-1",
|
||||
"decision": "HOLD",
|
||||
"candidate_block_id": None,
|
||||
"truncate_before_turn": None,
|
||||
"prefix_safe_before_boundary": False,
|
||||
"category": "NONE",
|
||||
"severity": "NONE",
|
||||
"state_effect": "UNCLEAR",
|
||||
"evidence": [],
|
||||
"reason": "No clear unrepaired defect.",
|
||||
}
|
||||
]
|
||||
)
|
||||
result = audit_trajectory(_record(), client) # type: ignore[arg-type]
|
||||
assert len(client.calls) == 1
|
||||
assert result["recommended_use"] == "HOLD"
|
||||
assert result["quality"] is None
|
||||
|
||||
|
||||
def test_keep_full_process_trajectory_is_scored() -> None:
|
||||
client = FakeClient(
|
||||
[
|
||||
{
|
||||
"sample_id": "sample-1",
|
||||
"decision": "KEEP_FULL",
|
||||
"candidate_block_id": None,
|
||||
"truncate_before_turn": None,
|
||||
"prefix_safe_before_boundary": True,
|
||||
"category": "NONE",
|
||||
"severity": "NONE",
|
||||
"state_effect": "NONE",
|
||||
"evidence": [],
|
||||
"reason": "No unrepaired severe problem is visible.",
|
||||
},
|
||||
{
|
||||
"sample_id": "sample-1",
|
||||
"prefix_valid": True,
|
||||
"unrecovered_major_or_critical": False,
|
||||
"behavior_issues": [
|
||||
{
|
||||
"assistant_turn": 3,
|
||||
"kind": "INEFFICIENCY",
|
||||
"severity": "MINOR",
|
||||
"reason": "The inspection was somewhat broad.",
|
||||
}
|
||||
],
|
||||
"dimensions": {
|
||||
"planning": 15,
|
||||
"investigation": 15,
|
||||
"tool_use_and_observation": 14,
|
||||
"progress": 12,
|
||||
"clarity_and_efficiency": 10,
|
||||
},
|
||||
"evidence": [],
|
||||
"reason": "Useful despite minor inefficiency.",
|
||||
},
|
||||
]
|
||||
)
|
||||
result = audit_trajectory(_record(), client) # type: ignore[arg-type]
|
||||
assert len(client.calls) == 2
|
||||
assert result["prefix"]["retained_turn_count"] == 6
|
||||
assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1
|
||||
assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE"
|
||||
+36
-1
@@ -1,6 +1,11 @@
|
||||
"""Tests for prompt-only trajectory compaction."""
|
||||
|
||||
from swe_data_processing.evidence import compact_text, compact_trajectory
|
||||
from swe_data_processing.evidence import (
|
||||
BOUNDARY_BLOCK_TURNS,
|
||||
build_trajectory_blocks,
|
||||
compact_text,
|
||||
compact_trajectory,
|
||||
)
|
||||
|
||||
|
||||
def test_compact_text_preserves_short_values() -> None:
|
||||
@@ -36,3 +41,33 @@ def test_compact_trajectory_prioritizes_test_observations() -> None:
|
||||
compacted, metadata = compact_trajectory(trajectory, signals)
|
||||
assert compacted[2]["content"] == long_content
|
||||
assert 3 in metadata["important_turn_ids"]
|
||||
|
||||
|
||||
def test_boundary_blocks_preserve_absolute_turns_without_overlap() -> None:
|
||||
trajectory = [
|
||||
{"turn_id": turn, "role": "assistant", "content": f"turn {turn}"}
|
||||
for turn in range(1, BOUNDARY_BLOCK_TURNS + 3)
|
||||
]
|
||||
blocks, metadata = build_trajectory_blocks(trajectory, {})
|
||||
assert len(blocks) == 2
|
||||
assert blocks[0]["start_turn"] == 1
|
||||
assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS
|
||||
assert blocks[1]["start_turn"] == BOUNDARY_BLOCK_TURNS + 1
|
||||
assert blocks[1]["end_turn"] == BOUNDARY_BLOCK_TURNS + 2
|
||||
turn_ids = [
|
||||
message["turn_id"] for block in blocks for message in block["messages"]
|
||||
]
|
||||
assert turn_ids == list(range(1, BOUNDARY_BLOCK_TURNS + 3))
|
||||
assert metadata["block_count"] == 2
|
||||
|
||||
|
||||
def test_boundary_block_keeps_immediate_tool_result_with_assistant() -> None:
|
||||
trajectory = [
|
||||
{"turn_id": turn, "role": "user", "content": f"turn {turn}"}
|
||||
for turn in range(1, BOUNDARY_BLOCK_TURNS + 2)
|
||||
]
|
||||
trajectory[BOUNDARY_BLOCK_TURNS - 1]["role"] = "assistant"
|
||||
trajectory[BOUNDARY_BLOCK_TURNS]["role"] = "tool"
|
||||
blocks, _ = build_trajectory_blocks(trajectory, {})
|
||||
assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS + 1
|
||||
assert len(blocks) == 1
|
||||
|
||||
Reference in New Issue
Block a user