Initial Open-SWE-Traces cleanup pipeline
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
"""Tests for local trajectory-audit policy and scoring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from swe_data_processing.audit import compute_quality_score, validate_audit
|
||||
from swe_data_processing.policy import PolicyViolation
|
||||
from swe_data_processing.workflow import prepare_audit_payload
|
||||
|
||||
|
||||
def _audit_result() -> dict:
|
||||
"""Return one internally consistent audit result."""
|
||||
|
||||
return {
|
||||
"evaluation_mode": "PROCESS_SALVAGE",
|
||||
"recommended_use": "PROCESS_PREFIX_CANDIDATE",
|
||||
"truncation": {
|
||||
"first_bad_assistant_turn": 3,
|
||||
"truncate_before_turn": 3,
|
||||
"acceptable_start_turn": 3,
|
||||
"acceptable_end_turn": 3,
|
||||
"prefix_usable": True,
|
||||
},
|
||||
"behavior_issues": [
|
||||
{
|
||||
"assistant_turn": 3,
|
||||
"tool_result_turn": 4,
|
||||
"kind": "ERROR",
|
||||
"severity": "MAJOR",
|
||||
}
|
||||
],
|
||||
"issue_counts": {
|
||||
"errors": 1,
|
||||
"inefficiencies": 0,
|
||||
"critical": 0,
|
||||
"major": 1,
|
||||
"minor": 0,
|
||||
},
|
||||
"quality_dimensions": {
|
||||
"planning": 4,
|
||||
"tool_selection": 4,
|
||||
"observation_use": 3,
|
||||
"efficiency": 4,
|
||||
"verification_discipline": 3,
|
||||
"claim_calibration": 3,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _record() -> dict:
|
||||
return {
|
||||
"trajectory": [
|
||||
{"role": "user", "content": "issue"},
|
||||
{"role": "tool", "content": "context"},
|
||||
{"role": "assistant", "content": "bad call"},
|
||||
{"role": "tool", "content": "failed"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_valid_audit_and_score() -> None:
|
||||
"""A consistent audit receives a deterministic bounded score."""
|
||||
|
||||
result = _audit_result()
|
||||
validate_audit(_record(), result)
|
||||
score = compute_quality_score(result)
|
||||
assert score["educational_quality_score"] == 65.0
|
||||
assert score["quality_tier"] == "LOW"
|
||||
|
||||
|
||||
def test_first_bad_turn_must_be_assistant() -> None:
|
||||
"""The truncation boundary cannot point at a tool observation."""
|
||||
|
||||
result = _audit_result()
|
||||
result["truncation"]["first_bad_assistant_turn"] = 4
|
||||
result["truncation"]["truncate_before_turn"] = 4
|
||||
with pytest.raises(PolicyViolation, match="assistant turn"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_issue_counts_are_recomputed() -> None:
|
||||
"""GLM cannot under-report the number of issues it listed."""
|
||||
|
||||
result = _audit_result()
|
||||
result["issue_counts"]["errors"] = 0
|
||||
with pytest.raises(PolicyViolation, match="issue_counts"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_process_salvage_cannot_become_full_trajectory_candidate() -> None:
|
||||
"""Outcome routing cannot be overridden by the model recommendation."""
|
||||
|
||||
result = _audit_result()
|
||||
result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE"
|
||||
with pytest.raises(PolicyViolation, match="full-trajectory"):
|
||||
validate_audit(_record(), result)
|
||||
|
||||
|
||||
def test_audit_payload_removes_reference_solution_signals() -> None:
|
||||
"""The causal audit cannot see reference patch text or derived metadata."""
|
||||
|
||||
record = {
|
||||
"trajectory_id": "sample-1",
|
||||
"resolved": 0,
|
||||
"trajectory": [{"role": "user", "content": "Fix the issue"}],
|
||||
"metadata": {
|
||||
"model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"},
|
||||
"reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"},
|
||||
},
|
||||
}
|
||||
payload = prepare_audit_payload(record)
|
||||
assert "reference_patch" not in payload
|
||||
assert not any("reference" in key for key in payload["static_signals"])
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Tests for API request compatibility and structured-response validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
from swe_data_processing.client import GLMClient
|
||||
from swe_data_processing.config import Settings
|
||||
|
||||
|
||||
def test_client_parses_fenced_json_and_never_sends_key_in_body() -> None:
|
||||
captured_body = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured_body.update(json.loads(request.content))
|
||||
assert request.headers["authorization"] == "Bearer test-secret"
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"x-request-id": "request-123"},
|
||||
json={
|
||||
"choices": [
|
||||
{"message": {"role": "assistant", "content": "```json\n{\"status\":\"ok\"}\n```"}}
|
||||
],
|
||||
"usage": {"total_tokens": 10},
|
||||
},
|
||||
)
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["status"],
|
||||
"properties": {"status": {"const": "ok"}},
|
||||
}
|
||||
settings = Settings(api_key="test-secret", max_retries=1)
|
||||
with GLMClient(settings, transport=httpx.MockTransport(handler)) as client:
|
||||
response = client.invoke_json(system_prompt="Return JSON.", payload={"ping": True}, schema=schema)
|
||||
|
||||
assert response.data == {"status": "ok"}
|
||||
assert response.request_id == "request-123"
|
||||
assert response.usage == {"total_tokens": 10}
|
||||
assert "test-secret" not in json.dumps(captured_body)
|
||||
assert captured_body["model"] == "glm-5.2"
|
||||
|
||||
|
||||
def test_client_falls_back_when_vendor_extensions_are_rejected() -> None:
|
||||
calls = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
calls.append(body)
|
||||
if len(calls) == 1:
|
||||
return httpx.Response(400, json={"error": {"message": "unknown parameter: thinking"}})
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"choices": [{"message": {"content": "{\"status\":\"ok\"}"}}]},
|
||||
)
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"required": ["status"],
|
||||
"properties": {"status": {"const": "ok"}},
|
||||
}
|
||||
settings = Settings(api_key="test-secret", max_retries=2)
|
||||
with GLMClient(settings, transport=httpx.MockTransport(handler)) as client:
|
||||
response = client.invoke_json(system_prompt="Return JSON.", payload={}, schema=schema)
|
||||
|
||||
assert "thinking" in calls[0]
|
||||
assert "thinking" not in calls[1]
|
||||
assert response.compatibility_fallback_used is True
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Tests for strict environment configuration and endpoint normalization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from swe_data_processing.config import Settings
|
||||
|
||||
|
||||
def test_endpoint_normalization() -> None:
|
||||
settings = Settings(
|
||||
api_key="test-secret",
|
||||
api_base="https://llm-api.cowin.run/",
|
||||
api_path="v1/chat/completions",
|
||||
)
|
||||
assert settings.endpoint == "https://llm-api.cowin.run/v1/chat/completions"
|
||||
|
||||
|
||||
def test_missing_api_key_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("GLM_API_KEY", raising=False)
|
||||
with pytest.raises(ValueError, match="GLM_API_KEY"):
|
||||
Settings.from_env()
|
||||
|
||||
|
||||
def test_feature_only_settings_can_omit_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("GLM_API_KEY", raising=False)
|
||||
assert Settings.from_env(require_api_key=False).api_key == ""
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Tests for prompt-only trajectory compaction."""
|
||||
|
||||
from swe_data_processing.evidence import compact_text, compact_trajectory
|
||||
|
||||
|
||||
def test_compact_text_preserves_short_values() -> None:
|
||||
"""Short evidence must remain byte-for-byte identical."""
|
||||
|
||||
assert compact_text("short evidence", 100) == "short evidence"
|
||||
|
||||
|
||||
def test_compact_text_hashes_long_values() -> None:
|
||||
"""Long evidence must expose its size and digest for auditability."""
|
||||
|
||||
result = compact_text("a" * 1_000, 200)
|
||||
assert len(result) == 200
|
||||
assert "COMPACTED original_chars=1000 sha256=" in result
|
||||
|
||||
|
||||
def test_compact_trajectory_prioritizes_test_observations() -> None:
|
||||
"""Important test output receives a larger preview than ordinary output."""
|
||||
|
||||
long_content = "x" * 3_000
|
||||
trajectory = [
|
||||
{"turn_id": 1, "role": "user", "content": "issue"},
|
||||
{"turn_id": 2, "role": "assistant", "content": "", "tool_calls": []},
|
||||
{"turn_id": 3, "role": "tool", "content": long_content},
|
||||
{"turn_id": 4, "role": "assistant", "content": "done"},
|
||||
]
|
||||
signals = {
|
||||
"stateful_turns": [],
|
||||
"test_events": [{"command_turn": 2, "output_turn": 3}],
|
||||
"malformed_tool_turns": [],
|
||||
"unknown_tool_turns": [],
|
||||
}
|
||||
compacted, metadata = compact_trajectory(trajectory, signals)
|
||||
assert compacted[2]["content"] == long_content
|
||||
assert 3 in metadata["important_turn_ids"]
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Tests for deterministic static trajectory evidence extraction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from swe_data_processing.features import extract_patch_files, extract_static_signals
|
||||
|
||||
|
||||
def _tool(name: str) -> str:
|
||||
return json.dumps({"type": "function", "function": {"name": name, "parameters": {}}})
|
||||
|
||||
|
||||
def test_detects_masked_failing_test_after_edit() -> None:
|
||||
record = {
|
||||
"trajectory_id": "sample-1",
|
||||
"resolved": 0,
|
||||
"tools": [_tool("str_replace_editor"), _tool("execute_bash"), _tool("finish")],
|
||||
"trajectory": [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "Do not modify test files."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "edit-1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "str_replace_editor",
|
||||
"arguments": json.dumps(
|
||||
{"command": "str_replace", "path": "/workspace/src/main.py"}
|
||||
),
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "content": "The file was edited."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "test-1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "execute_bash",
|
||||
"arguments": json.dumps({"command": "pytest tests | tail -20"}),
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": "1 failed, 10 passed\n[Command finished with exit code 0]",
|
||||
},
|
||||
],
|
||||
"metadata": {
|
||||
"model_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"},
|
||||
"reference_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"},
|
||||
},
|
||||
}
|
||||
|
||||
signals = extract_static_signals(record)
|
||||
assert signals["last_stateful_turn"] == 3
|
||||
assert signals["instruction_constraints"] == ["Do not modify test files."]
|
||||
assert signals["test_events"][0]["masked_pipeline"] is True
|
||||
assert signals["test_events"][0]["explicit_failure"] is True
|
||||
assert signals["test_events"][0]["explicit_exit_codes"] == [0]
|
||||
|
||||
|
||||
def test_detects_malformed_and_unknown_tool_calls() -> None:
|
||||
record = {
|
||||
"trajectory_id": "sample-2",
|
||||
"tools": [_tool("execute_bash")],
|
||||
"trajectory": [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "issue"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "bad",
|
||||
"function": {"name": "made_up_tool", "arguments": "{bad json"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "content": "error"},
|
||||
],
|
||||
"metadata": {"model_patch": {}, "reference_patch": {}},
|
||||
}
|
||||
signals = extract_static_signals(record)
|
||||
assert signals["malformed_tool_turns"][0]["turn_id"] == 3
|
||||
assert signals["unknown_tool_turns"] == [{"turn_id": 3, "tool_name": "made_up_tool"}]
|
||||
|
||||
|
||||
def test_extract_patch_files_is_stable_and_unique() -> None:
|
||||
patch = "+++ b/src/a.py\n+++ b/src/b.py\n+++ b/src/a.py\n+++ /dev/null\n"
|
||||
assert extract_patch_files(patch) == ["src/a.py", "src/b.py"]
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Tests for immutable classification and repair policy enforcement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from swe_data_processing.policy import (
|
||||
PolicyViolation,
|
||||
enforce_classification_policy,
|
||||
enforce_repair_policy,
|
||||
)
|
||||
|
||||
|
||||
def _classification() -> dict:
|
||||
dimensions = {
|
||||
"trajectory_integrity": "PASS",
|
||||
"tool_integrity": "PASS",
|
||||
"patch_presence": "PASS",
|
||||
"patch_trajectory_consistency": "PASS",
|
||||
"instruction_compliance": "PASS",
|
||||
"verification_consistency": "PASS",
|
||||
"final_claim_alignment": "PASS",
|
||||
"patch_hygiene": "PASS",
|
||||
"issue_patch_alignment": "PASS",
|
||||
}
|
||||
return {
|
||||
"source_outcome_class": "POSITIVE_CANDIDATE",
|
||||
"qc_decision": "ACCEPT_SILVER_POSITIVE",
|
||||
"training_use": "SFT_FULL",
|
||||
"qc_passed": True,
|
||||
"hard_fail_codes": [],
|
||||
"verification": {"status": "PASS_RELIABLE"},
|
||||
"dimensions": dimensions,
|
||||
}
|
||||
|
||||
|
||||
def test_valid_silver_positive_passes() -> None:
|
||||
enforce_classification_policy({"resolved": 1}, _classification())
|
||||
|
||||
|
||||
def test_resolved_zero_cannot_be_promoted() -> None:
|
||||
result = _classification()
|
||||
result["source_outcome_class"] = "EXPLICIT_NEGATIVE"
|
||||
with pytest.raises(PolicyViolation, match="resolved=0"):
|
||||
enforce_classification_policy({"resolved": 0}, result)
|
||||
|
||||
|
||||
def test_unknown_cannot_be_full_sft() -> None:
|
||||
result = _classification()
|
||||
result["source_outcome_class"] = "UNVERIFIED"
|
||||
result["qc_decision"] = "HOLD_UNVERIFIED"
|
||||
result["qc_passed"] = False
|
||||
with pytest.raises(PolicyViolation, match="resolved=-1"):
|
||||
enforce_classification_policy({"resolved": -1}, result)
|
||||
|
||||
|
||||
def _repair_result() -> dict:
|
||||
"""Return a minimal non-mutating repair result for policy tests."""
|
||||
|
||||
return {
|
||||
"repair_decision": "NO_CHANGE",
|
||||
"maximum_training_use": "HOLD",
|
||||
"invariants": {
|
||||
"resolved_unchanged": True,
|
||||
"tool_outputs_unchanged": True,
|
||||
"model_patch_unchanged": True,
|
||||
"reference_patch_unchanged": True,
|
||||
"no_synthetic_execution_result": True,
|
||||
},
|
||||
"operations": [],
|
||||
"requires_second_review": True,
|
||||
}
|
||||
|
||||
|
||||
def test_nonmutating_repair_decision_rejects_operations() -> None:
|
||||
"""A hold decision cannot smuggle in an operation the applier will ignore."""
|
||||
|
||||
result = _repair_result()
|
||||
result["repair_decision"] = "REQUIRES_EXECUTION"
|
||||
result["operations"] = [
|
||||
{
|
||||
"op": "REWRITE_FINAL_SUMMARY",
|
||||
"target_turns": [3],
|
||||
"preconditions": ["Turn 3 is an assistant summary"],
|
||||
"replacement": "Honest summary",
|
||||
"reason": "Align the claim with evidence",
|
||||
}
|
||||
]
|
||||
with pytest.raises(PolicyViolation, match="must not contain"):
|
||||
enforce_repair_policy(result)
|
||||
|
||||
|
||||
def test_step_example_requires_operations() -> None:
|
||||
"""Step-only salvage must include a concrete correction and truncation."""
|
||||
|
||||
result = _repair_result()
|
||||
result["repair_decision"] = "CREATE_STEP_EXAMPLE"
|
||||
result["maximum_training_use"] = "SFT_STEP_ONLY"
|
||||
with pytest.raises(PolicyViolation, match="requires at least one"):
|
||||
enforce_repair_policy(result)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Tests for deterministic static repair application and evidence preservation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from swe_data_processing.repair import apply_static_repair
|
||||
|
||||
|
||||
def _plan() -> dict:
|
||||
return {
|
||||
"sample_id": "sample-1",
|
||||
"repair_decision": "APPLY_STATIC_REPAIR",
|
||||
"maximum_training_use": "ERROR_ANALYSIS",
|
||||
"invariants": {
|
||||
"resolved_unchanged": True,
|
||||
"tool_outputs_unchanged": True,
|
||||
"model_patch_unchanged": True,
|
||||
"reference_patch_unchanged": True,
|
||||
"no_synthetic_execution_result": True,
|
||||
},
|
||||
"operations": [
|
||||
{
|
||||
"op": "REWRITE_FINAL_SUMMARY",
|
||||
"target_turns": [5],
|
||||
"preconditions": ["Turn 5 is an assistant-only summary."],
|
||||
"replacement": "The build passed, but functional behavior remains unverified.",
|
||||
"reason": "Remove an unsupported success claim.",
|
||||
}
|
||||
],
|
||||
"requires_second_review": True,
|
||||
"summary": "Correct the final claim without changing execution evidence.",
|
||||
}
|
||||
|
||||
|
||||
def test_rewrite_summary_preserves_tool_output_and_patch() -> None:
|
||||
record = {
|
||||
"trajectory_id": "sample-1",
|
||||
"resolved": -1,
|
||||
"trajectory": [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "issue"},
|
||||
{"role": "assistant", "content": "build", "tool_calls": []},
|
||||
{"role": "tool", "content": "build completed with exit code 0"},
|
||||
{"role": "assistant", "content": "All tests pass.", "tool_calls": []},
|
||||
],
|
||||
"metadata": {
|
||||
"model_patch": {"patch": "+++ b/src/a.py\n"},
|
||||
"reference_patch": {"patch": "+++ b/src/a.py\n"},
|
||||
},
|
||||
}
|
||||
repaired, diff = apply_static_repair(record, _plan())
|
||||
assert repaired["trajectory"][3] == record["trajectory"][3]
|
||||
assert repaired["trajectory"][4]["content"].startswith("The build passed")
|
||||
assert repaired["metadata"] == record["metadata"]
|
||||
assert diff["replaced_original_turn_ids"] == [5]
|
||||
Reference in New Issue
Block a user