101 lines
3.6 KiB
Python
101 lines
3.6 KiB
Python
"""Tests for deterministic static trajectory evidence extraction."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from swe_data_processing.features import extract_patch_files, extract_static_signals
|
|
|
|
|
|
def _tool(name: str) -> str:
|
|
return json.dumps({"type": "function", "function": {"name": name, "parameters": {}}})
|
|
|
|
|
|
def test_detects_masked_failing_test_after_edit() -> None:
|
|
record = {
|
|
"trajectory_id": "sample-1",
|
|
"resolved": 0,
|
|
"tools": [_tool("str_replace_editor"), _tool("execute_bash"), _tool("finish")],
|
|
"trajectory": [
|
|
{"role": "system", "content": "system"},
|
|
{"role": "user", "content": "Do not modify test files."},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "edit-1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "str_replace_editor",
|
|
"arguments": json.dumps(
|
|
{"command": "str_replace", "path": "/workspace/src/main.py"}
|
|
),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "content": "The file was edited."},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "test-1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "execute_bash",
|
|
"arguments": json.dumps({"command": "pytest tests | tail -20"}),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"content": "1 failed, 10 passed\n[Command finished with exit code 0]",
|
|
},
|
|
],
|
|
"metadata": {
|
|
"model_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"},
|
|
"reference_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"},
|
|
},
|
|
}
|
|
|
|
signals = extract_static_signals(record)
|
|
assert signals["last_stateful_turn"] == 3
|
|
assert signals["instruction_constraints"] == ["Do not modify test files."]
|
|
assert signals["test_events"][0]["masked_pipeline"] is True
|
|
assert signals["test_events"][0]["explicit_failure"] is True
|
|
assert signals["test_events"][0]["explicit_exit_codes"] == [0]
|
|
|
|
|
|
def test_detects_malformed_and_unknown_tool_calls() -> None:
|
|
record = {
|
|
"trajectory_id": "sample-2",
|
|
"tools": [_tool("execute_bash")],
|
|
"trajectory": [
|
|
{"role": "system", "content": "system"},
|
|
{"role": "user", "content": "issue"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "bad",
|
|
"function": {"name": "made_up_tool", "arguments": "{bad json"},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "content": "error"},
|
|
],
|
|
"metadata": {"model_patch": {}, "reference_patch": {}},
|
|
}
|
|
signals = extract_static_signals(record)
|
|
assert signals["malformed_tool_turns"][0]["turn_id"] == 3
|
|
assert signals["unknown_tool_turns"] == [{"turn_id": 3, "tool_name": "made_up_tool"}]
|
|
|
|
|
|
def test_extract_patch_files_is_stable_and_unique() -> None:
|
|
patch = "+++ b/src/a.py\n+++ b/src/b.py\n+++ b/src/a.py\n+++ /dev/null\n"
|
|
assert extract_patch_files(patch) == ["src/a.py", "src/b.py"]
|