From c6c583dceeb3caa1e10f81030394be10b611afbf Mon Sep 17 00:00:00 2001 From: Tom Lu Date: Fri, 7 Aug 2026 00:47:31 +0800 Subject: [PATCH] Replace trajectory audit with isolated prefix scoring --- README.md | 45 +++- changelog.md | 55 ++++ pyproject.toml | 2 +- src/swe_data_processing/__init__.py | 2 +- src/swe_data_processing/audit.py | 202 ++++++++------ src/swe_data_processing/cli.py | 6 +- src/swe_data_processing/client.py | 2 +- src/swe_data_processing/evidence.py | 41 +++ src/swe_data_processing/prompts/boundary.md | 35 +++ .../prompts/prefix_quality.md | 30 +++ .../prompts/trajectory_audit.md | 121 --------- .../schemas/boundary_output.schema.json | 43 +++ .../schemas/prefix_quality_output.schema.json | 62 +++++ .../trajectory_audit_output.schema.json | 116 -------- src/swe_data_processing/workflow.py | 180 ++++++++++--- tests/test_audit.py | 248 +++++++++++------- tests/test_audit_workflow.py | 156 +++++++++++ tests/test_evidence.py | 37 ++- 18 files changed, 914 insertions(+), 469 deletions(-) create mode 100644 changelog.md create mode 100644 src/swe_data_processing/prompts/boundary.md create mode 100644 src/swe_data_processing/prompts/prefix_quality.md delete mode 100644 src/swe_data_processing/prompts/trajectory_audit.md create mode 100644 src/swe_data_processing/schemas/boundary_output.schema.json create mode 100644 src/swe_data_processing/schemas/prefix_quality_output.schema.json delete mode 100644 src/swe_data_processing/schemas/trajectory_audit_output.schema.json create mode 100644 tests/test_audit_workflow.py diff --git a/README.md b/README.md index 4e039ea..881e0cd 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ README. swe_data_processing/ ├── pyproject.toml ├── README.md +├── changelog.md # Versioned strategy changes ├── .env.example ├── src/swe_data_processing/ │ ├── cli.py # Command-line entry points @@ -203,18 +204,28 @@ The classifier returns one of: Training use is tracked separately as `SFT_FULL`, `SFT_STEP_ONLY`, `DPO_REJECTED`, `ERROR_ANALYSIS`, `HOLD`, or `DROP`. -### 4. Audit educational process quality +### 4. Locate a safe prefix and score it independently -This stage does not repair trajectories. It identifies defensible erroneous or -inefficient calls, scores six process-quality dimensions, and optionally finds -a causal first-bad assistant turn for prefix-only learning: +`audit` uses two isolated GLM calls for failed and unknown trajectories: + +1. The boundary call receives the trajectory as roughly 32-turn blocks without + splitting an assistant action from its immediate tool result. It cannot see + `resolved`, model patches, or reference patches. It selects the earliest + unrepaired major/critical assistant turn, keeps a safe full process trace, + or returns `HOLD` when neither decision is supported. +2. Python slices the exact messages before that assistant turn. The quality call + receives only this materialized prefix, so suffix behavior cannot affect its + score. + +Successful trajectories skip boundary selection and are scored as complete +trajectories. The command remains simple: ```bash swe-qc audit \ --input samples/sample_20_seed_20260805.jsonl \ --output qc_outputs/sample20.audits.jsonl \ --errors qc_outputs/sample20.audit.errors.jsonl \ - --workers 20 \ + --workers 5 \ --resume ``` @@ -223,15 +234,23 @@ remain serialized in the main thread, so each completed record is appended atomically even when API requests run in parallel. Output order follows request completion order; `sample_id` remains the stable join key. -The deterministic score combines weighted process dimensions with penalties -for minor, major, and critical behavior issues. Failed exploratory calls are -not penalized when the agent interprets them correctly and recovers. +The gateway previously returned frequent HTTP 429 responses at 20 workers, so +five workers is the practical default for long runs unless the service limit is +raised. -The causal audit payload excludes the reference patch and every derived signal, -including reference file names, patch length, and model/reference size ratios. -The immutable outcome is used only to select the mode: failed or unknown -trajectories can become process-prefix candidates, but never full-trajectory -SFT candidates. +The quality call lists concrete erroneous and inefficient assistant actions and +scores five 0-20 dimensions: planning, investigation, tool use and observation, +progress, and clarity/efficiency. Python recomputes issue counts, sums the five +dimensions, and assigns: + +- `HIGH`: 80-100 +- `MEDIUM`: 60-79 +- `LOW`: 40-59 +- `REJECT`: below 40, or an unrepaired major/critical problem remains + +Incomplete prefixes can still be useful. Normal failed experiments are retained +when the agent later diagnoses, repairs, and verifies them. A prefix ending in a +complete tool result is valid process-SFT structure. ### 5. Create static repair plans diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..8d66815 --- /dev/null +++ b/changelog.md @@ -0,0 +1,55 @@ +# Changelog + +This file records strategy changes that materially affect dataset decisions or +training-data semantics. Generated audit manifests are not treated as stable API +contracts. + +## 2.0.0 - 2026-08-07 + +### Strategy + +- Replaced the single-call trajectory audit with two isolated model calls: + boundary selection followed by prefix-only quality scoring. +- The boundary call receives an outcome-blind and patch-blind trajectory view. + Long trajectories are presented as ordered blocks targeting 32 turns, with + absolute turn IDs and immediate assistant/tool-result pairs kept together. +- Python materializes the exact prefix before the selected assistant turn. The + quality call is then built again from that prefix, so it cannot see the + removed suffix, external outcome, model patch, reference patch, or boundary + rationale. +- Simplified both prompts and added concrete positive, recovery, and late-claim + examples. +- Replaced the weighted six-dimension score and issue penalties with five + equally weighted 0-20 dimensions. Python computes the final 0-100 score and + tier. +- Distinguished a safe full process trajectory (`KEEP_FULL`) from an uncertain + trajectory (`HOLD`) so the boundary stage does not reject useful data merely + because no truncation is needed. +- Kept an explicit list of erroneous and inefficient assistant actions as a + quality surrogate; Python recomputes its counts locally. + +### Validation + +- A boundary must be an assistant turn inside the selected block. +- Boundary evidence must include an exact quote grounded in that assistant + turn. +- A prefix scorer cannot cite a turn outside the materialized prefix. +- `HIGH`, `MEDIUM`, `LOW`, and `REJECT` are computed locally rather than trusted + from model output. + +### Motivation + +- In the previous 500-record pilot, 360 of 423 prefix candidates included + suffix behavior in the score despite prompt instructions to score only the + retained prefix. +- Manual stress review found useful conservative cutoffs, but also late cutoffs + that retained failed patches and normal experiments incorrectly treated as + permanent errors. +- Prompt wording alone could not guarantee suffix isolation, so the new design + enforces it in the data passed to each call. + +## 1.0.0 - 2026-08-06 + +- Initial static classification, one-call trajectory audit, allowlisted repair + planning, deterministic repair application, and independent repair review. +- Added API retries and concurrent JSONL processing. diff --git a/pyproject.toml b/pyproject.toml index 957be2b..0272758 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "swe-data-processing" -version = "0.1.0" +version = "2.0.0" description = "Static quality control and repair planning for Open-SWE-Traces." readme = "README.md" requires-python = ">=3.10" diff --git a/src/swe_data_processing/__init__.py b/src/swe_data_processing/__init__.py index 46275a5..c100162 100644 --- a/src/swe_data_processing/__init__.py +++ b/src/swe_data_processing/__init__.py @@ -3,4 +3,4 @@ from .config import Settings __all__ = ["Settings"] -__version__ = "0.1.0" +__version__ = "2.0.0" diff --git a/src/swe_data_processing/audit.py b/src/swe_data_processing/audit.py index 086ae1c..51070c1 100644 --- a/src/swe_data_processing/audit.py +++ b/src/swe_data_processing/audit.py @@ -1,106 +1,158 @@ -"""Local validation and scoring for GLM trajectory-quality audits.""" +"""Validation and deterministic scoring for the two-call audit pipeline.""" from __future__ import annotations +import json +import re from typing import Any from .features import index_trajectory +from .io import get_sample_id from .policy import PolicyViolation -DIMENSION_WEIGHTS = { - "planning": 0.15, - "tool_selection": 0.20, - "observation_use": 0.20, - "efficiency": 0.15, - "verification_discipline": 0.20, - "claim_calibration": 0.10, -} - -SEVERITY_PENALTIES = {"MINOR": 1.0, "MAJOR": 5.0, "CRITICAL": 12.0} +QUALITY_DIMENSIONS = ( + "planning", + "investigation", + "tool_use_and_observation", + "progress", + "clarity_and_efficiency", +) -def validate_audit(record: dict[str, Any], result: dict[str, Any]) -> None: - """Reject internally inconsistent or non-causal audit outputs.""" +def _flatten_text(value: Any) -> str: + if isinstance(value, str): + return value + if isinstance(value, dict): + return " ".join(_flatten_text(item) for item in value.values()) + if isinstance(value, list): + return " ".join(_flatten_text(item) for item in value) + return json.dumps(value, ensure_ascii=False, default=str) + + +def _normalized_text(value: Any) -> str: + return re.sub(r"\s+", " ", _flatten_text(value)).strip() + + +def _evidence_is_grounded(message: dict[str, Any], quote: str) -> bool: + """Return whether a short evidence quote occurs in the referenced message.""" + + message_text = _normalized_text({key: value for key, value in message.items() if key != "turn_id"}) + return _normalized_text(quote) in message_text + + +def validate_boundary( + record: dict[str, Any], + blocks: list[dict[str, Any]], + result: dict[str, Any], +) -> None: + """Reject structurally inconsistent or ungrounded boundary decisions.""" trajectory = index_trajectory(record) + messages = {message["turn_id"]: message for message in trajectory} assistant_turns = { message["turn_id"] for message in trajectory if message.get("role") == "assistant" } - all_turns = {message["turn_id"] for message in trajectory} + block_ranges = { + block["block_id"]: range(block["start_turn"], block["end_turn"] + 1) + for block in blocks + } + if result["sample_id"] != get_sample_id(record): + raise PolicyViolation("Boundary sample_id does not match the source record") - first_bad_turn = result["truncation"]["first_bad_assistant_turn"] - truncate_before = result["truncation"]["truncate_before_turn"] - prefix_usable = result["truncation"]["prefix_usable"] - evaluation_mode = result["evaluation_mode"] - recommended_use = result["recommended_use"] - if evaluation_mode == "PROCESS_SALVAGE" and recommended_use == "FULL_TRAJECTORY_CANDIDATE": - raise PolicyViolation("A failed/unknown trajectory cannot be a full-trajectory candidate") - if evaluation_mode == "SUCCESS_QUALITY" and recommended_use == "PROCESS_PREFIX_CANDIDATE": - raise PolicyViolation("A successful trajectory is not evaluated as process salvage") - if prefix_usable != (recommended_use == "PROCESS_PREFIX_CANDIDATE"): - raise PolicyViolation("prefix_usable and recommended_use are inconsistent") - if first_bad_turn is not None and first_bad_turn not in assistant_turns: - raise PolicyViolation("first_bad_assistant_turn must reference an assistant turn") - if truncate_before is not None and truncate_before not in assistant_turns: + decision = result["decision"] + block_id = result["candidate_block_id"] + boundary = result["truncate_before_turn"] + if decision in {"KEEP_FULL", "HOLD"}: + if block_id is not None or boundary is not None: + raise PolicyViolation(f"A {decision} decision cannot contain a boundary") + if result["category"] != "NONE" or result["severity"] != "NONE": + raise PolicyViolation(f"A {decision} decision must use NONE category and severity") + if decision == "KEEP_FULL" and not result["prefix_safe_before_boundary"]: + raise PolicyViolation("KEEP_FULL requires a safe full trajectory") + return + + if block_id not in block_ranges: + raise PolicyViolation("candidate_block_id does not reference an input block") + if boundary not in assistant_turns: raise PolicyViolation("truncate_before_turn must reference an assistant turn") - if ( - evaluation_mode == "PROCESS_SALVAGE" - and prefix_usable - and (first_bad_turn is None or truncate_before is None) - ): - raise PolicyViolation("A usable process prefix requires an explicit bad assistant turn") - if first_bad_turn is not None and truncate_before != first_bad_turn: - raise PolicyViolation("Truncation must begin at the first bad assistant turn") + if boundary not in block_ranges[block_id]: + raise PolicyViolation("truncate_before_turn must be inside candidate_block_id") + if not result["prefix_safe_before_boundary"]: + raise PolicyViolation("A truncation boundary requires a safe retained prefix") + if result["severity"] not in {"MAJOR", "CRITICAL"}: + raise PolicyViolation("A truncation boundary requires MAJOR or CRITICAL severity") - start = result["truncation"]["acceptable_start_turn"] - end = result["truncation"]["acceptable_end_turn"] - if (start is None) != (end is None): - raise PolicyViolation("Acceptable turn range must be fully null or fully specified") - if start is not None and (start not in assistant_turns or end not in assistant_turns or start > end): - raise PolicyViolation("Acceptable turn range must reference ordered assistant turns") + evidence = result["evidence"] + if not any(item["turn_id"] == boundary for item in evidence): + raise PolicyViolation("Boundary evidence must quote the excluded assistant turn") + for item in evidence: + turn_id = item["turn_id"] + if turn_id not in messages: + raise PolicyViolation("Boundary evidence references a missing turn") + if not _evidence_is_grounded(messages[turn_id], item["quote"]): + raise PolicyViolation("Boundary evidence quote is not present in its source turn") + +def materialize_prefix( + record: dict[str, Any], boundary: int | None +) -> list[dict[str, Any]]: + """Return the exact retained messages without rewriting any source content.""" + + trajectory = record.get("trajectory") or [] + return list(trajectory if boundary is None else trajectory[: boundary - 1]) + + +def validate_prefix_quality( + sample_id: str, + prefix: list[dict[str, Any]], + result: dict[str, Any], +) -> None: + """Validate a quality judgment that was made from the prefix alone.""" + + if result["sample_id"] != sample_id: + raise PolicyViolation("Quality sample_id does not match the source record") + if result["prefix_valid"] == result["unrecovered_major_or_critical"]: + raise PolicyViolation("prefix_valid contradicts unrecovered_major_or_critical") + if result["unrecovered_major_or_critical"] and not result["evidence"]: + raise PolicyViolation("An unsafe prefix requires grounded evidence") + + messages = {message["turn_id"]: message for message in index_trajectory({"trajectory": prefix})} + assistant_turns = { + turn_id for turn_id, message in messages.items() if message.get("role") == "assistant" + } for issue in result["behavior_issues"]: if issue["assistant_turn"] not in assistant_turns: - raise PolicyViolation("Every behavior issue must reference an assistant turn") - result_turn = issue["tool_result_turn"] - if result_turn is not None and result_turn not in all_turns: - raise PolicyViolation("tool_result_turn references a missing turn") - - counts = result["issue_counts"] - issues = result["behavior_issues"] - expected = { - "errors": sum(issue["kind"] == "ERROR" for issue in issues), - "inefficiencies": sum(issue["kind"] == "INEFFICIENCY" for issue in issues), - "critical": sum(issue["severity"] == "CRITICAL" for issue in issues), - "major": sum(issue["severity"] == "MAJOR" for issue in issues), - "minor": sum(issue["severity"] == "MINOR" for issue in issues), - } - if counts != expected: - raise PolicyViolation(f"issue_counts do not match behavior_issues: expected {expected}") + raise PolicyViolation("Behavior issue references an assistant turn outside the prefix") + for item in result["evidence"]: + turn_id = item["turn_id"] + if turn_id not in messages: + raise PolicyViolation("Quality evidence references a turn outside the prefix") + if not _evidence_is_grounded(messages[turn_id], item["quote"]): + raise PolicyViolation("Quality evidence quote is not present in its source turn") -def compute_quality_score(result: dict[str, Any]) -> dict[str, Any]: - """Compute a deterministic educational-quality score from audit outputs.""" +def compute_prefix_quality(result: dict[str, Any]) -> dict[str, Any]: + """Compute the final score and tier from five model-provided dimensions.""" - dimensions = result["quality_dimensions"] - base_score = sum(dimensions[name] * 20.0 * weight for name, weight in DIMENSION_WEIGHTS.items()) - issue_penalty = sum(SEVERITY_PENALTIES[issue["severity"]] for issue in result["behavior_issues"]) - issue_penalty = min(35.0, issue_penalty) - score = max(0.0, min(100.0, base_score - issue_penalty)) - - counts = result["issue_counts"] - if counts["critical"]: - tier = "LOW" - elif score >= 85: + score = sum(int(result["dimensions"][name]) for name in QUALITY_DIMENSIONS) + if not result["prefix_valid"] or result["unrecovered_major_or_critical"] or score < 40: + tier = "REJECT" + elif score >= 80: tier = "HIGH" - elif score >= 70: + elif score >= 60: tier = "MEDIUM" else: tier = "LOW" + issues = result["behavior_issues"] return { - "base_dimension_score": round(base_score, 2), - "issue_penalty": round(issue_penalty, 2), - "educational_quality_score": round(score, 2), + "educational_quality_score": score, "quality_tier": tier, - "formula_version": "weighted-dimensions-minus-issue-severity-v1", + "issue_counts": { + "errors": sum(issue["kind"] == "ERROR" for issue in issues), + "inefficiencies": sum(issue["kind"] == "INEFFICIENCY" for issue in issues), + "minor": sum(issue["severity"] == "MINOR" for issue in issues), + "major": sum(issue["severity"] == "MAJOR" for issue in issues), + "critical": sum(issue["severity"] == "CRITICAL" for issue in issues), + }, + "formula_version": "five-equal-dimensions-v2", } diff --git a/src/swe_data_processing/cli.py b/src/swe_data_processing/cli.py index 36cf6bf..83c5775 100644 --- a/src/swe_data_processing/cli.py +++ b/src/swe_data_processing/cli.py @@ -154,7 +154,7 @@ def command_classify(args: argparse.Namespace) -> int: def command_audit(args: argparse.Namespace) -> int: - """Score educational process quality and locate causal truncation points.""" + """Locate a safe boundary, materialize its prefix, then score the prefix.""" settings = Settings.from_env() with GLMClient(settings) as client: @@ -165,7 +165,7 @@ def command_audit(args: argparse.Namespace) -> int: limit=args.limit, resume=args.resume, workers=args.workers, - stage_name="trajectory_audit", + stage_name="two_call_audit", processor=lambda record: audit_trajectory(record, client), ) @@ -324,7 +324,7 @@ def build_parser() -> argparse.ArgumentParser: classify.set_defaults(func=command_classify) audit = subparsers.add_parser( - "audit", help="Score process quality and locate causal truncation points" + "audit", help="Locate a safe boundary, then score only the retained prefix" ) _add_stream_arguments(audit) audit.set_defaults(func=command_audit) diff --git a/src/swe_data_processing/client.py b/src/swe_data_processing/client.py index eb40a26..876f380 100644 --- a/src/swe_data_processing/client.py +++ b/src/swe_data_processing/client.py @@ -79,7 +79,7 @@ class GLMClient: "Authorization": f"Bearer {settings.api_key}", "Content-Type": "application/json", "Accept": "application/json", - "User-Agent": "swe-data-processing/0.1.0", + "User-Agent": "swe-data-processing/2.0.0", }, ) diff --git a/src/swe_data_processing/evidence.py b/src/swe_data_processing/evidence.py index b202064..8155b82 100644 --- a/src/swe_data_processing/evidence.py +++ b/src/swe_data_processing/evidence.py @@ -12,6 +12,8 @@ import hashlib import json from typing import Any +BOUNDARY_BLOCK_TURNS = 32 + def _as_text(value: Any) -> str: """Convert arbitrary JSON-compatible content into deterministic text.""" @@ -151,3 +153,42 @@ def compact_trajectory( ), } return compacted, metadata + + +def build_trajectory_blocks( + trajectory: list[dict[str, Any]], + static_signals: dict[str, Any], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Split a compact trajectory into stable blocks for boundary review. + + Blocks are only a prompt representation. They never change source turns, + and every message keeps its absolute one-based ``turn_id``. A fixed block + size keeps the architecture predictable without adding tuning parameters to + the command line. + """ + + compacted, metadata = compact_trajectory(trajectory, static_signals) + blocks = [] + offset = 0 + while offset < len(compacted): + end = min(offset + BOUNDARY_BLOCK_TURNS, len(compacted)) + # Keep the immediate observation with the assistant action that caused + # it. This may make a block one or two turns larger than the target. + while end < len(compacted) and compacted[end].get("role") == "tool": + end += 1 + messages = compacted[offset:end] + block_number = len(blocks) + 1 + blocks.append( + { + "block_id": f"block-{block_number:03d}", + "start_turn": messages[0]["turn_id"], + "end_turn": messages[-1]["turn_id"], + "messages": messages, + } + ) + offset = end + return blocks, { + **metadata, + "block_turns": BOUNDARY_BLOCK_TURNS, + "block_count": len(blocks), + } diff --git a/src/swe_data_processing/prompts/boundary.md b/src/swe_data_processing/prompts/boundary.md new file mode 100644 index 0000000..e2bffd5 --- /dev/null +++ b/src/swe_data_processing/prompts/boundary.md @@ -0,0 +1,35 @@ +You are finding a safe truncation point in a coding-agent trajectory. You cannot run code. + +The trajectory is split into ordered blocks. Turn IDs are absolute. Find the earliest assistant turn B that begins +a MAJOR or CRITICAL problem which is never repaired. The saved training prefix will contain only turns before B. + +Choose TRUNCATE only when both are true: + +1. Turn B clearly starts a serious problem, such as an unrepaired harmful edit, a persistent wrong implementation, + a literal user-constraint violation, a broken tool call, or an unsupported final claim. +2. Everything before B is still safe process training data. It must not contain an earlier unrepaired harmful state. + +Return KEEP_FULL when the whole trajectory has no unrepaired MAJOR or CRITICAL problem. It may still contain normal +failed experiments or minor inefficiency. Return HOLD only when the evidence is incomplete or contradictory enough +that you cannot establish either a safe full trajectory or a safe prefix. + +Keep normal debugging. A failed command, a plausible experiment, or temporarily broken code is not a boundary when +the agent later diagnoses, repairs, and verifies it. Do not punish harmless inefficiency. + +Find the cause, not the last symptom. If a bad patch remains and the final answer falsely claims success, choose the +assistant turn that created the bad patch. Do not merely remove the final answer while retaining failed code. + +Read user constraints literally. Do not invent stronger restrictions. Reading tests, running tests, or creating a +requested reproduction script is not the same as modifying repository tests. + +Evidence must include a short exact quote from assistant turn B. Supporting quotes may come from later tool results. +The candidate block must be the block that contains B. + +Positive example: turn 40 writes code that calls a nonexistent method, turn 46 shows the resulting failure, and the +edit is never repaired. Choose turn 40. + +Recovery example: turn 40 tries a wrong edit, turn 44 diagnoses it, turn 48 reverts it, and turn 52 passes the relevant +tests. Do not choose turn 40. + +Late-claim example: tests fail after a patch and the final turn says "all tests pass." If the patch is visibly still +bad, choose the earlier patch turn. Choose the final claim only when the retained implementation is otherwise safe. diff --git a/src/swe_data_processing/prompts/prefix_quality.md b/src/swe_data_processing/prompts/prefix_quality.md new file mode 100644 index 0000000..7feec09 --- /dev/null +++ b/src/swe_data_processing/prompts/prefix_quality.md @@ -0,0 +1,30 @@ +You are scoring process-SFT data. You can see only the exact prefix that would be kept for training. You cannot run +code, and you know nothing about any removed suffix or external outcome. + +First decide whether this prefix itself contains an unrepaired MAJOR or CRITICAL harmful behavior. If it does, mark +the prefix invalid and quote the harmful turn. Otherwise mark it valid. + +Also list concrete erroneous or inefficient assistant actions. An ERROR is a wrong tool call, edit, interpretation, +or claim. An INEFFICIENCY is avoidable repetition, noise, or a clearly wasteful detour. A reasonable experiment that +fails and is then read and handled correctly is not automatically an error. Recovered mistakes may be listed, but +they do not by themselves make the prefix invalid. + +An incomplete prefix can still be useful. It may teach problem understanding, repository exploration, reproduction, +tool use, or correct reading of tool results. It does not need to solve the issue. Ending after a tool result is valid. + +Score these five dimensions from 0 to 20: + +- planning: understands the task and forms sensible next steps; +- investigation: finds and reads relevant code or evidence; +- tool_use_and_observation: uses tools sensibly and interprets results honestly; +- progress: makes useful progress toward reproduction, diagnosis, or implementation; +- clarity_and_efficiency: avoids severe loops, noise, and unsupported claims. + +Judge only visible prefix behavior. Do not reduce the score for imagined later failures. Apply only literal user +constraints, not stronger paraphrases. + +Positive example: the prefix identifies the relevant files, reproduces the bug, reads the failure correctly, and +ends before implementation. It can be MEDIUM or HIGH even though it is incomplete. + +Negative example: the prefix itself contains an unreverted wrong-file edit and then ends. Mark it invalid even if +the earlier investigation was useful. diff --git a/src/swe_data_processing/prompts/trajectory_audit.md b/src/swe_data_processing/prompts/trajectory_audit.md deleted file mode 100644 index c8ae76f..0000000 --- a/src/swe_data_processing/prompts/trajectory_audit.md +++ /dev/null @@ -1,121 +0,0 @@ -You are auditing the educational process quality of an Open-SWE-Traces coding-agent trajectory. You cannot run -code and must not repair or rewrite the trajectory. Return exactly one schema-valid JSON object. - -There are two related tasks: - -1. Identify clear erroneous or inefficient assistant/tool-call behavior using only evidence visible in the - trajectory at that point. -2. For a trajectory that is not externally marked successful, decide whether a prefix before the first clearly - bad assistant turn still teaches useful problem analysis, tool use, or observation interpretation. - -## Exact truncation-boundary definition - -For `PROCESS_SALVAGE`, scan assistant turns in chronological order and choose a boundary `B` only if all of the -following are true: - -1. The retained prefix is every message with `turn_id < B`; turn `B` and every later message are discarded. -2. Turn `B` is an assistant turn, and the action or reasoning at `B` is the earliest *major or critical, - unrecovered* defect that should not be taught as positive process data. -3. The prefix before `B` is still coherent and educational: it contains useful investigation, tool use, or correct - interpretation of observations, with at most a small number of minor or recovered defects. -4. Keeping turn `B` would materially lower the educational validity of the prefix. If keeping `B` is still - reasonable exploration, move the boundary later or return null. - -Use this decision procedure for tool interactions: - -- If an assistant chooses an invalid command, violates an explicit constraint, performs an unsafe state-changing - action, or targets a file known to be wrong, `B` is that assistant turn. -- If a reasonable tool call returns an error, nonzero exit status, timeout, or failed test, the tool result is not - a boundary. Keep it when it teaches useful diagnosis. If the next assistant ignores, contradicts, or falsely - explains that result, `B` is that next assistant turn. -- If the assistant makes a plausible hypothesis that is later disproved and then correctly adapts, do not truncate - at the hypothesis. Normal exploration and recovered mistakes are allowed. -- If an assistant starts a no-progress loop, `B` is the first assistant turn where repetition without meaningful - adaptation becomes clear, not the first failed attempt. -- If the implementation is useful but the final answer claims unsupported success, `B` is the assistant turn that - makes the unsupported claim; preserve the useful work before it. - -Before returning a non-null `B`, perform this two-sided check: - -- `B-1 check`: the last assistant action before `B` can still be shown to an SFT learner without teaching a major - known error. -- `B check`: quote visible evidence proving why assistant turn `B` itself must be excluded. - -Also perform a retained-prefix terminal-state check. The prefix immediately before `B` must not leave an earlier -major/critical episode open. Reject or move the boundary when the retained prefix contains an unreverted wrong-file -edit, constraint violation, destructive action, known patch pollution, repeated failing state-changing call, or -other harmful repository state whose recovery occurs at turn `B` or later. Never truncate immediately before a -rollback, cleanup, correction, or successful adaptation when doing so would preserve the mistake but discard its -recovery. Either move the boundary before the original harmful action or include the recovery and search later. - -An earlier-than-optimal boundary is acceptable when it is conservative: the retained prefix is coherent, has -meaningful educational signal, and contains no unrecovered major/critical behavior. Prefer safety over maximizing -length. Do not reject a safe candidate merely because useful later work would be omitted; that is yield loss, not -training-data corruption. - -If either side cannot be supported from the supplied trajectory, return a null boundary and `prefix_usable=false`. -Do not invent a boundary merely because the external outcome is unsuccessful. - -Causal rules: - -- Do not use the external outcome label or reference patch to claim an earlier action was wrong. The outcome label - only selects the evaluation mode. -- A failed tool call is not automatically bad. It may be a useful probe when its result is read correctly and the - next action makes progress. -- Mark an ERROR only for visible behavior such as ignoring a failure, contradicting an observation, violating an - explicit constraint, using an invalid argument, targeting the wrong file after contrary evidence, making an - unsafe state-changing action, or claiming unsupported success. -- Mark an INEFFICIENCY only when evidence is strong: essentially duplicate reads, repeated identical failures - without adaptation, long no-progress loops, unnecessary re-verification, or repetitive summaries. -- Do not punish normal exploration, one failed search, or a reasonable hypothesis that is corrected later. -- Distinguish the first observable minor defect from the first disqualifying turn. A valid process prefix may - contain a small number of recovered errors or inefficiencies. Truncate only before the first major/critical - assistant action that makes the remaining suffix unsuitable to teach. In other words, find the first turn after - which the process should no longer be shown as positive training data, not merely the first imperfect turn. -- The truncation turn must be an assistant turn. Tool output can be evidence, but truncation occurs before the - assistant action that mishandles prior evidence. -- A process prefix is usable only when the prefix before that assistant turn is coherent and contains meaningful - learning signal. It need not solve the full task and need not be perfectly efficient. -- If no causal first-bad turn can be proven, return null rather than using hidden outcome knowledge. -- A prompt `[COMPACTED ...]` marker is not trajectory corruption. If decisive text is unavailable, lower confidence - or return null. -- Every non-null truncation boundary and acceptable-range endpoint must be a `turn_id` whose role is `assistant`. -- `acceptable_start_turn` and `acceptable_end_turn` are a narrow uncertainty interval around `B`, not the retained - prefix range. Both endpoints must be actual assistant `turn_id` values near `B`. Use identical endpoints when the - boundary is clear. Never use the first trajectory turn as a default range start. -- In `SUCCESS_QUALITY` mode, audit the complete trajectory. Set every truncation boundary/range field to null and - set `prefix_usable` to false because process salvage is not applicable. -- In `PROCESS_SALVAGE` mode, behavior issues, issue counts, and quality dimensions must describe only the retained - prefix strictly before `truncate_before_turn`. The truncation evidence may separately describe the excluded bad - turn. This makes the score a score of the candidate training prefix, not of the discarded suffix. - -Quality dimensions are integers from 0 to 5: - -- planning: decomposition and hypothesis quality; -- tool_selection: appropriate tools, commands, and targets; -- observation_use: reads and responds to tool evidence correctly; -- efficiency: avoids redundant/no-progress work; -- verification_discipline: uses relevant checks and interprets their status honestly; -- claim_calibration: summaries match what was actually observed. - -Recommended-use rules: - -- `FULL_TRAJECTORY_CANDIDATE`: externally successful and no critical visible process defect; -- `PROCESS_PREFIX_CANDIDATE`: not successful, but a causal usable prefix and bad-turn boundary are identified; -- `HOLD`: evidence is insufficient or ambiguous; -- `REJECT`: no meaningful safe prefix or severe bad behavior starts too early. - -List only defensible behavior issues. Quotes must be short and copied from the supplied evidence. Do not propose -edits, corrected tool calls, or synthetic results. - -Final self-check before emitting JSON: - -- every boundary, range endpoint, and `behavior_issues[].assistant_turn` names an assistant turn; -- every `tool_result_turn` names an existing tool-result turn or is null; -- each evidence object contains exactly `turn_id` and `quote` (never use a `content` field), and every behavior issue - contains every schema-required field including `reason`; -- for process salvage, all listed behavior issues and dimension scores describe only retained messages before `B`; -- `first_bad_assistant_turn == truncate_before_turn == B` when non-null; -- a usable prefix has a non-null `B`; otherwise all four boundary/range values are null. -- the retained prefix has no open major/critical error episode or harmful state mutation awaiting recovery at or - after the boundary. diff --git a/src/swe_data_processing/schemas/boundary_output.schema.json b/src/swe_data_processing/schemas/boundary_output.schema.json new file mode 100644 index 0000000..3695c0e --- /dev/null +++ b/src/swe_data_processing/schemas/boundary_output.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "sample_id", "decision", "candidate_block_id", "truncate_before_turn", + "prefix_safe_before_boundary", "category", "severity", "state_effect", + "evidence", "reason" + ], + "properties": { + "sample_id": {"type": "string", "minLength": 1}, + "decision": {"enum": ["TRUNCATE", "KEEP_FULL", "HOLD"]}, + "candidate_block_id": {"type": ["string", "null"], "pattern": "^block-[0-9]{3}$"}, + "truncate_before_turn": {"type": ["integer", "null"], "minimum": 1}, + "prefix_safe_before_boundary": {"type": "boolean"}, + "category": { + "enum": [ + "NONE", "USER_CONSTRAINT_VIOLATION", "HARMFUL_STATE_CHANGE", + "PERSISTENT_WRONG_IMPLEMENTATION", "BROKEN_TOOL_STRUCTURE", + "UNGROUNDED_FINAL_CLAIM" + ] + }, + "severity": {"enum": ["NONE", "MAJOR", "CRITICAL"]}, + "state_effect": {"enum": ["NONE", "UNRECOVERED", "UNCLEAR"]}, + "evidence": { + "type": "array", + "maxItems": 4, + "items": {"$ref": "#/$defs/evidence"} + }, + "reason": {"type": "string", "minLength": 1, "maxLength": 1000} + }, + "$defs": { + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["turn_id", "quote"], + "properties": { + "turn_id": {"type": "integer", "minimum": 1}, + "quote": {"type": "string", "minLength": 1, "maxLength": 600} + } + } + } +} diff --git a/src/swe_data_processing/schemas/prefix_quality_output.schema.json b/src/swe_data_processing/schemas/prefix_quality_output.schema.json new file mode 100644 index 0000000..ba61955 --- /dev/null +++ b/src/swe_data_processing/schemas/prefix_quality_output.schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "sample_id", "prefix_valid", "unrecovered_major_or_critical", + "behavior_issues", "dimensions", "evidence", "reason" + ], + "properties": { + "sample_id": {"type": "string", "minLength": 1}, + "prefix_valid": {"type": "boolean"}, + "unrecovered_major_or_critical": {"type": "boolean"}, + "behavior_issues": { + "type": "array", + "maxItems": 20, + "items": {"$ref": "#/$defs/behavior_issue"} + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "planning", "investigation", "tool_use_and_observation", "progress", + "clarity_and_efficiency" + ], + "properties": { + "planning": {"type": "integer", "minimum": 0, "maximum": 20}, + "investigation": {"type": "integer", "minimum": 0, "maximum": 20}, + "tool_use_and_observation": {"type": "integer", "minimum": 0, "maximum": 20}, + "progress": {"type": "integer", "minimum": 0, "maximum": 20}, + "clarity_and_efficiency": {"type": "integer", "minimum": 0, "maximum": 20} + } + }, + "evidence": { + "type": "array", + "maxItems": 4, + "items": {"$ref": "#/$defs/evidence"} + }, + "reason": {"type": "string", "minLength": 1, "maxLength": 800} + }, + "$defs": { + "behavior_issue": { + "type": "object", + "additionalProperties": false, + "required": ["assistant_turn", "kind", "severity", "reason"], + "properties": { + "assistant_turn": {"type": "integer", "minimum": 1}, + "kind": {"enum": ["ERROR", "INEFFICIENCY"]}, + "severity": {"enum": ["MINOR", "MAJOR", "CRITICAL"]}, + "reason": {"type": "string", "minLength": 1, "maxLength": 500} + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": ["turn_id", "quote"], + "properties": { + "turn_id": {"type": "integer", "minimum": 1}, + "quote": {"type": "string", "minLength": 1, "maxLength": 600} + } + } + } +} diff --git a/src/swe_data_processing/schemas/trajectory_audit_output.schema.json b/src/swe_data_processing/schemas/trajectory_audit_output.schema.json deleted file mode 100644 index 20e42ec..0000000 --- a/src/swe_data_processing/schemas/trajectory_audit_output.schema.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "additionalProperties": false, - "required": [ - "sample_id", "evaluation_mode", "truncation", "behavior_issues", "issue_counts", - "quality_dimensions", "recommended_use", "confidence", "summary" - ], - "properties": { - "sample_id": {"type": "string", "minLength": 1}, - "evaluation_mode": {"enum": ["SUCCESS_QUALITY", "PROCESS_SALVAGE"]}, - "truncation": { - "type": "object", - "additionalProperties": false, - "required": [ - "first_bad_assistant_turn", "truncate_before_turn", "acceptable_start_turn", - "acceptable_end_turn", "prefix_usable", "category", "evidence", "reason" - ], - "properties": { - "first_bad_assistant_turn": {"type": ["integer", "null"], "minimum": 1}, - "truncate_before_turn": {"type": ["integer", "null"], "minimum": 1}, - "acceptable_start_turn": {"type": ["integer", "null"], "minimum": 1}, - "acceptable_end_turn": {"type": ["integer", "null"], "minimum": 1}, - "prefix_usable": {"type": "boolean"}, - "category": { - "enum": [ - "NONE", "IGNORED_FAILURE", "CONSTRAINT_VIOLATION", "UNGROUNDED_SUCCESS", - "WRONG_TOOL_OR_TARGET", "UNSAFE_STATE_CHANGE", "REPEATED_NO_PROGRESS", - "MISREAD_OBSERVATION", "INVALID_TOOL_CALL" - ] - }, - "evidence": { - "type": "array", "maxItems": 4, - "items": {"$ref": "#/$defs/evidence"} - }, - "reason": {"type": "string", "minLength": 1, "maxLength": 1400} - } - }, - "behavior_issues": { - "type": "array", "maxItems": 40, - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "assistant_turn", "tool_result_turn", "tool_name", "kind", "category", - "severity", "state_effect", "recovered_later", "quote", "reason" - ], - "properties": { - "assistant_turn": {"type": "integer", "minimum": 1}, - "tool_result_turn": {"type": ["integer", "null"], "minimum": 1}, - "tool_name": {"type": ["string", "null"], "maxLength": 80}, - "kind": {"enum": ["ERROR", "INEFFICIENCY"]}, - "category": { - "enum": [ - "IGNORED_FAILURE", "CONSTRAINT_VIOLATION", "UNGROUNDED_CLAIM", - "WRONG_TOOL_OR_TARGET", "MISREAD_OBSERVATION", "INVALID_ARGUMENT", "INVALID_TOOL_CALL", - "UNSAFE_ACTION", - "REDUNDANT_READ", "REPEATED_FAILURE", "NO_PROGRESS_LOOP", - "OVER_VERIFICATION", "REPETITIVE_SUMMARY", "MASKED_EXIT_STATUS", - "ENVIRONMENT_MISDIAGNOSIS", "DISCARDED_USEFUL_EVIDENCE" - ] - }, - "severity": {"enum": ["MINOR", "MAJOR", "CRITICAL"]}, - "state_effect": {"enum": ["NONE", "POSSIBLE", "CONFIRMED"]}, - "recovered_later": {"type": "boolean"}, - "quote": {"type": "string", "minLength": 1, "maxLength": 800}, - "reason": {"type": "string", "minLength": 1, "maxLength": 500} - } - } - }, - "issue_counts": { - "type": "object", - "additionalProperties": false, - "required": ["errors", "inefficiencies", "critical", "major", "minor"], - "properties": { - "errors": {"type": "integer", "minimum": 0}, - "inefficiencies": {"type": "integer", "minimum": 0}, - "critical": {"type": "integer", "minimum": 0}, - "major": {"type": "integer", "minimum": 0}, - "minor": {"type": "integer", "minimum": 0} - } - }, - "quality_dimensions": { - "type": "object", - "additionalProperties": false, - "required": [ - "planning", "tool_selection", "observation_use", "efficiency", - "verification_discipline", "claim_calibration" - ], - "properties": { - "planning": {"type": "integer", "minimum": 0, "maximum": 5}, - "tool_selection": {"type": "integer", "minimum": 0, "maximum": 5}, - "observation_use": {"type": "integer", "minimum": 0, "maximum": 5}, - "efficiency": {"type": "integer", "minimum": 0, "maximum": 5}, - "verification_discipline": {"type": "integer", "minimum": 0, "maximum": 5}, - "claim_calibration": {"type": "integer", "minimum": 0, "maximum": 5} - } - }, - "recommended_use": { - "enum": ["FULL_TRAJECTORY_CANDIDATE", "PROCESS_PREFIX_CANDIDATE", "HOLD", "REJECT"] - }, - "confidence": {"type": "number", "minimum": 0, "maximum": 1}, - "summary": {"type": "string", "minLength": 1, "maxLength": 1000} - }, - "$defs": { - "evidence": { - "type": "object", - "additionalProperties": false, - "required": ["turn_id", "quote"], - "properties": { - "turn_id": {"type": "integer", "minimum": 1}, - "quote": {"type": "string", "minLength": 1, "maxLength": 800} - } - } - } -} diff --git a/src/swe_data_processing/workflow.py b/src/swe_data_processing/workflow.py index 12991be..ace9209 100644 --- a/src/swe_data_processing/workflow.py +++ b/src/swe_data_processing/workflow.py @@ -7,9 +7,19 @@ import json from datetime import datetime, timezone from typing import Any -from .audit import compute_quality_score, validate_audit +from .audit import ( + compute_prefix_quality, + materialize_prefix, + validate_boundary, + validate_prefix_quality, +) from .client import GLMClient, GLMResponse -from .evidence import compact_patch_object, compact_text, compact_trajectory +from .evidence import ( + build_trajectory_blocks, + compact_patch_object, + compact_text, + compact_trajectory, +) from .features import extract_instruction_constraints, extract_static_signals, index_trajectory from .io import get_sample_id from .policy import enforce_classification_policy, enforce_repair_policy @@ -102,52 +112,150 @@ def classify_record(record: dict[str, Any], client: GLMClient) -> dict[str, Any] return result -def prepare_audit_payload(record: dict[str, Any]) -> dict[str, Any]: - """Build an audit payload with all reference-solution evidence removed.""" +def _trajectory_only_record( + record: dict[str, Any], trajectory: list[dict[str, Any]] +) -> dict[str, Any]: + """Return the fields allowed to influence trajectory-only evidence.""" - payload = prepare_classification_payload(record) - # The reference solution is deliberately hidden from this task. A causal - # bad-turn label must be supported by information the original agent had. - # Reference-derived summary features must also be removed; otherwise file - # names and size ratios can leak the hidden solution even without its text. - payload.pop("reference_patch", None) - static_signals = dict(payload["static_signals"]) - for key in ( - "reference_patch_files", - "reference_patch_chars", - "patch_size_ratio_to_reference", - ): - static_signals.pop(key, None) - payload["static_signals"] = static_signals - resolved = int(record.get("resolved", -1)) - payload["evaluation_mode"] = "SUCCESS_QUALITY" if resolved == 1 else "PROCESS_SALVAGE" - payload["outcome_label_usage"] = ( - "Routing only. Do not use resolved to identify a bad action or justify a truncation turn." - ) - return payload + return {"trajectory": trajectory, "tools": record.get("tools") or []} + + +def prepare_boundary_payload(record: dict[str, Any]) -> dict[str, Any]: + """Build an outcome-blind, patch-blind block view for boundary selection.""" + + raw_trajectory = record.get("trajectory") or [] + indexed = index_trajectory(record) + signals = extract_static_signals(_trajectory_only_record(record, raw_trajectory)) + blocks, compaction = build_trajectory_blocks(indexed, signals) + return { + "sample_id": get_sample_id(record), + "user_request": compact_text(_first_user_issue(indexed), 16_000), + "allowed_tool_names": signals["allowed_tool_names"], + "trajectory_blocks": blocks, + "prompt_compaction": compaction, + } + + +def prepare_prefix_quality_payload( + record: dict[str, Any], prefix: list[dict[str, Any]] +) -> dict[str, Any]: + """Build a quality payload from the materialized prefix and nothing later.""" + + prefix_record = _trajectory_only_record(record, prefix) + indexed = index_trajectory(prefix_record) + signals = extract_static_signals(prefix_record) + trajectory, compaction = compact_trajectory(indexed, signals) + assistant_turns = sum(message.get("role") == "assistant" for message in prefix) + return { + "sample_id": get_sample_id(record), + "user_request": compact_text(_first_user_issue(indexed), 16_000), + "allowed_tool_names": signals["allowed_tool_names"], + "trajectory": trajectory, + "prefix_metadata": { + "turn_count": len(prefix), + "assistant_turn_count": assistant_turns, + "ends_with_role": prefix[-1].get("role") if prefix else None, + "trajectory_sha256": _stable_hash({"trajectory": prefix}), + }, + "prompt_compaction": compaction, + } def audit_trajectory(record: dict[str, Any], client: GLMClient) -> dict[str, Any]: - """Audit causal process quality and possible prefix salvage without repair.""" + """Locate a boundary, materialize its prefix, then score only that prefix.""" - payload = prepare_audit_payload(record) - response = client.invoke_json( - system_prompt=load_prompt("trajectory_audit.md"), - payload=payload, - schema=load_schema("trajectory_audit_output.schema.json"), + sample_id = get_sample_id(record) + resolved = int(record.get("resolved", -1)) + source_trajectory = record.get("trajectory") or [] + boundary_result: dict[str, Any] | None = None + boundary_provenance: dict[str, Any] | None = None + + if resolved == 1: + prefix = materialize_prefix(record, None) + evaluation_mode = "FULL_TRAJECTORY" + else: + boundary_payload = prepare_boundary_payload(record) + boundary_response = client.invoke_json( + system_prompt=load_prompt("boundary.md"), + payload=boundary_payload, + schema=load_schema("boundary_output.schema.json"), + ) + blocks = boundary_payload["trajectory_blocks"] + validate_boundary(record, blocks, boundary_response.data) + boundary_result = dict(boundary_response.data) + boundary_provenance = _provenance( + "boundary", boundary_response, boundary_payload, client + ) + evaluation_mode = "PROCESS_PREFIX" + if boundary_result["decision"] == "HOLD": + return { + "sample_id": sample_id, + "pipeline_version": "2.0", + "evaluation_mode": evaluation_mode, + "boundary": boundary_result, + "prefix": None, + "quality": None, + "recommended_use": "HOLD", + "source_record": _audit_source_record(record), + "provenance": {"boundary": boundary_provenance, "quality": None}, + } + prefix = materialize_prefix(record, boundary_result["truncate_before_turn"]) + + quality_payload = prepare_prefix_quality_payload(record, prefix) + quality_response = client.invoke_json( + system_prompt=load_prompt("prefix_quality.md"), + payload=quality_payload, + schema=load_schema("prefix_quality_output.schema.json"), ) - validate_audit(record, response.data) - result = dict(response.data) - result["source_record"] = { + validate_prefix_quality(sample_id, prefix, quality_response.data) + quality = dict(quality_response.data) + quality["local_score"] = compute_prefix_quality(quality) + tier = quality["local_score"]["quality_tier"] + if tier == "REJECT": + recommended_use = "REJECT" + elif resolved == 1: + recommended_use = "FULL_TRAJECTORY_CANDIDATE" + else: + recommended_use = "PROCESS_PREFIX_CANDIDATE" + + boundary_turn = ( + boundary_result["truncate_before_turn"] if boundary_result is not None else None + ) + return { + "sample_id": sample_id, + "pipeline_version": "2.0", + "evaluation_mode": evaluation_mode, + "boundary": boundary_result, + "prefix": { + "truncate_before_turn": boundary_turn, + "source_turn_count": len(source_trajectory), + "retained_turn_count": len(prefix), + "retained_assistant_turn_count": sum( + message.get("role") == "assistant" for message in prefix + ), + "ends_with_role": prefix[-1].get("role") if prefix else None, + "trajectory_sha256": quality_payload["prefix_metadata"]["trajectory_sha256"], + }, + "quality": quality, + "recommended_use": recommended_use, + "source_record": _audit_source_record(record), + "provenance": { + "boundary": boundary_provenance, + "quality": _provenance("prefix_quality", quality_response, quality_payload, client), + }, + } + + +def _audit_source_record(record: dict[str, Any]) -> dict[str, Any]: + """Return compact immutable routing metadata for the final manifest.""" + + return { "resolved": record.get("resolved"), "instance_id": record.get("instance_id"), "repo": record.get("repo"), "language": record.get("language"), "sample_provenance": record.get("_sample"), } - result["local_quality_score"] = compute_quality_score(result) - result["provenance"] = _provenance("trajectory_audit", response, payload, client) - return result def plan_static_repair( diff --git a/tests/test_audit.py b/tests/test_audit.py index acc030b..33d8570 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -1,114 +1,160 @@ -"""Tests for local trajectory-audit policy and scoring.""" +"""Tests for the two-call boundary and prefix-quality policy.""" from __future__ import annotations +import json + import pytest -from swe_data_processing.audit import compute_quality_score, validate_audit +from swe_data_processing.audit import ( + compute_prefix_quality, + materialize_prefix, + validate_boundary, + validate_prefix_quality, +) from swe_data_processing.policy import PolicyViolation -from swe_data_processing.workflow import prepare_audit_payload - - -def _audit_result() -> dict: - """Return one internally consistent audit result.""" - - return { - "evaluation_mode": "PROCESS_SALVAGE", - "recommended_use": "PROCESS_PREFIX_CANDIDATE", - "truncation": { - "first_bad_assistant_turn": 3, - "truncate_before_turn": 3, - "acceptable_start_turn": 3, - "acceptable_end_turn": 3, - "prefix_usable": True, - }, - "behavior_issues": [ - { - "assistant_turn": 3, - "tool_result_turn": 4, - "kind": "ERROR", - "severity": "MAJOR", - } - ], - "issue_counts": { - "errors": 1, - "inefficiencies": 0, - "critical": 0, - "major": 1, - "minor": 0, - }, - "quality_dimensions": { - "planning": 4, - "tool_selection": 4, - "observation_use": 3, - "efficiency": 4, - "verification_discipline": 3, - "claim_calibration": 3, - }, - } +from swe_data_processing.workflow import ( + prepare_boundary_payload, + prepare_prefix_quality_payload, +) def _record() -> dict: return { - "trajectory": [ - {"role": "user", "content": "issue"}, - {"role": "tool", "content": "context"}, - {"role": "assistant", "content": "bad call"}, - {"role": "tool", "content": "failed"}, - ] - } - - -def test_valid_audit_and_score() -> None: - """A consistent audit receives a deterministic bounded score.""" - - result = _audit_result() - validate_audit(_record(), result) - score = compute_quality_score(result) - assert score["educational_quality_score"] == 65.0 - assert score["quality_tier"] == "LOW" - - -def test_first_bad_turn_must_be_assistant() -> None: - """The truncation boundary cannot point at a tool observation.""" - - result = _audit_result() - result["truncation"]["first_bad_assistant_turn"] = 4 - result["truncation"]["truncate_before_turn"] = 4 - with pytest.raises(PolicyViolation, match="assistant turn"): - validate_audit(_record(), result) - - -def test_issue_counts_are_recomputed() -> None: - """GLM cannot under-report the number of issues it listed.""" - - result = _audit_result() - result["issue_counts"]["errors"] = 0 - with pytest.raises(PolicyViolation, match="issue_counts"): - validate_audit(_record(), result) - - -def test_process_salvage_cannot_become_full_trajectory_candidate() -> None: - """Outcome routing cannot be overridden by the model recommendation.""" - - result = _audit_result() - result["recommended_use"] = "FULL_TRAJECTORY_CANDIDATE" - with pytest.raises(PolicyViolation, match="full-trajectory"): - validate_audit(_record(), result) - - -def test_audit_payload_removes_reference_solution_signals() -> None: - """The causal audit cannot see reference patch text or derived metadata.""" - - record = { "trajectory_id": "sample-1", "resolved": 0, - "trajectory": [{"role": "user", "content": "Fix the issue"}], - "metadata": { - "model_patch": {"patch": "--- a/model.py\n+++ b/model.py\n"}, - "reference_patch": {"patch": "--- a/secret.py\n+++ b/secret.py\n"}, - }, + "tools": [], + "trajectory": [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "Fix the bug."}, + {"role": "assistant", "content": "I will inspect the code."}, + {"role": "tool", "content": "relevant.py"}, + {"role": "assistant", "content": "I will apply the harmful patch."}, + {"role": "tool", "content": "tests failed"}, + ], } - payload = prepare_audit_payload(record) - assert "reference_patch" not in payload - assert not any("reference" in key for key in payload["static_signals"]) + + +def _boundary() -> dict: + return { + "sample_id": "sample-1", + "decision": "TRUNCATE", + "candidate_block_id": "block-001", + "truncate_before_turn": 5, + "prefix_safe_before_boundary": True, + "category": "PERSISTENT_WRONG_IMPLEMENTATION", + "severity": "MAJOR", + "state_effect": "UNRECOVERED", + "evidence": [{"turn_id": 5, "quote": "apply the harmful patch"}], + "reason": "The patch fails and is not repaired.", + } + + +def _blocks() -> list[dict]: + return [{"block_id": "block-001", "start_turn": 1, "end_turn": 6}] + + +def _quality() -> dict: + return { + "sample_id": "sample-1", + "prefix_valid": True, + "unrecovered_major_or_critical": False, + "behavior_issues": [], + "dimensions": { + "planning": 14, + "investigation": 15, + "tool_use_and_observation": 14, + "progress": 12, + "clarity_and_efficiency": 10, + }, + "evidence": [], + "reason": "Useful investigation.", + } + + +def test_valid_boundary_materializes_exact_prefix() -> None: + validate_boundary(_record(), _blocks(), _boundary()) + prefix = materialize_prefix(_record(), 5) + assert prefix == _record()["trajectory"][:4] + + +def test_boundary_must_be_an_assistant_turn() -> None: + result = _boundary() + result["truncate_before_turn"] = 4 + with pytest.raises(PolicyViolation, match="assistant turn"): + validate_boundary(_record(), _blocks(), result) + + +def test_boundary_evidence_must_quote_boundary_turn() -> None: + result = _boundary() + result["evidence"] = [{"turn_id": 6, "quote": "tests failed"}] + with pytest.raises(PolicyViolation, match="excluded assistant turn"): + validate_boundary(_record(), _blocks(), result) + + +def test_boundary_quote_must_be_grounded() -> None: + result = _boundary() + result["evidence"][0]["quote"] = "invented evidence" + with pytest.raises(PolicyViolation, match="not present"): + validate_boundary(_record(), _blocks(), result) + + +def test_keep_full_requires_safe_full_trajectory() -> None: + result = _boundary() + result.update( + decision="KEEP_FULL", + candidate_block_id=None, + truncate_before_turn=None, + prefix_safe_before_boundary=True, + category="NONE", + severity="NONE", + state_effect="NONE", + evidence=[], + ) + validate_boundary(_record(), _blocks(), result) + result["prefix_safe_before_boundary"] = False + with pytest.raises(PolicyViolation, match="safe full trajectory"): + validate_boundary(_record(), _blocks(), result) + + +def test_prefix_quality_cannot_reference_suffix() -> None: + prefix = materialize_prefix(_record(), 5) + result = _quality() + result["prefix_valid"] = False + result["unrecovered_major_or_critical"] = True + result["evidence"] = [{"turn_id": 5, "quote": "harmful patch"}] + with pytest.raises(PolicyViolation, match="outside the prefix"): + validate_prefix_quality("sample-1", prefix, result) + + +def test_quality_score_and_tier_are_computed_locally() -> None: + score = compute_prefix_quality(_quality()) + assert score["educational_quality_score"] == 65 + assert score["quality_tier"] == "MEDIUM" + + +def test_boundary_payload_hides_outcome_and_patch_metadata() -> None: + record = _record() + record["resolved"] = -1 + record["metadata"] = { + "model_patch": {"patch": "MODEL_PATCH_SECRET"}, + "reference_patch": {"patch": "REFERENCE_PATCH_SECRET"}, + } + serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False) + assert "resolved" not in serialized + assert "MODEL_PATCH_SECRET" not in serialized + assert "REFERENCE_PATCH_SECRET" not in serialized + + +def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None: + first = _record() + second = _record() + second["resolved"] = -1 + second["trajectory"][4]["content"] = "DIFFERENT SUFFIX" + second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT" + second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}} + first_prefix = materialize_prefix(first, 5) + second_prefix = materialize_prefix(second, 5) + assert prepare_prefix_quality_payload( + first, first_prefix + ) == prepare_prefix_quality_payload(second, second_prefix) diff --git a/tests/test_audit_workflow.py b/tests/test_audit_workflow.py new file mode 100644 index 0000000..1bd6c27 --- /dev/null +++ b/tests/test_audit_workflow.py @@ -0,0 +1,156 @@ +"""End-to-end tests for call isolation in the audit workflow.""" + +from __future__ import annotations + +import json + +from swe_data_processing.client import GLMResponse +from swe_data_processing.config import Settings +from swe_data_processing.workflow import audit_trajectory + + +class FakeClient: + """Return deterministic responses while recording every API payload.""" + + def __init__(self, responses: list[dict]) -> None: + self.settings = Settings(api_key="test-secret") + self.responses = iter(responses) + self.calls: list[dict] = [] + + def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse: + self.calls.append( + {"system_prompt": system_prompt, "payload": payload, "schema": schema} + ) + return GLMResponse( + data=next(self.responses), + request_id=f"request-{len(self.calls)}", + usage={"total_tokens": 10}, + compatibility_fallback_used=False, + ) + + +def _record() -> dict: + return { + "trajectory_id": "sample-1", + "resolved": 0, + "trajectory": [ + {"role": "system", "content": "system"}, + {"role": "user", "content": "Fix the bug."}, + {"role": "assistant", "content": "Inspect relevant.py"}, + {"role": "tool", "content": "relevant code"}, + {"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"}, + {"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"}, + ], + } + + +def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None: + client = FakeClient( + [ + { + "sample_id": "sample-1", + "decision": "TRUNCATE", + "candidate_block_id": "block-001", + "truncate_before_turn": 5, + "prefix_safe_before_boundary": True, + "category": "PERSISTENT_WRONG_IMPLEMENTATION", + "severity": "MAJOR", + "state_effect": "UNRECOVERED", + "evidence": [{"turn_id": 5, "quote": "BAD_SUFFIX_SENTINEL patch"}], + "reason": "The patch is not repaired.", + }, + { + "sample_id": "sample-1", + "prefix_valid": True, + "unrecovered_major_or_critical": False, + "behavior_issues": [], + "dimensions": { + "planning": 15, + "investigation": 15, + "tool_use_and_observation": 14, + "progress": 12, + "clarity_and_efficiency": 10, + }, + "evidence": [], + "reason": "The prefix is useful.", + }, + ] + ) + result = audit_trajectory(_record(), client) # type: ignore[arg-type] + + assert len(client.calls) == 2 + quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False) + assert "BAD_SUFFIX_SENTINEL" not in quality_payload + assert "truncate_before_turn" not in quality_payload + assert result["prefix"]["retained_turn_count"] == 4 + assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM" + assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE" + + +def test_hold_boundary_skips_quality_call() -> None: + client = FakeClient( + [ + { + "sample_id": "sample-1", + "decision": "HOLD", + "candidate_block_id": None, + "truncate_before_turn": None, + "prefix_safe_before_boundary": False, + "category": "NONE", + "severity": "NONE", + "state_effect": "UNCLEAR", + "evidence": [], + "reason": "No clear unrepaired defect.", + } + ] + ) + result = audit_trajectory(_record(), client) # type: ignore[arg-type] + assert len(client.calls) == 1 + assert result["recommended_use"] == "HOLD" + assert result["quality"] is None + + +def test_keep_full_process_trajectory_is_scored() -> None: + client = FakeClient( + [ + { + "sample_id": "sample-1", + "decision": "KEEP_FULL", + "candidate_block_id": None, + "truncate_before_turn": None, + "prefix_safe_before_boundary": True, + "category": "NONE", + "severity": "NONE", + "state_effect": "NONE", + "evidence": [], + "reason": "No unrepaired severe problem is visible.", + }, + { + "sample_id": "sample-1", + "prefix_valid": True, + "unrecovered_major_or_critical": False, + "behavior_issues": [ + { + "assistant_turn": 3, + "kind": "INEFFICIENCY", + "severity": "MINOR", + "reason": "The inspection was somewhat broad.", + } + ], + "dimensions": { + "planning": 15, + "investigation": 15, + "tool_use_and_observation": 14, + "progress": 12, + "clarity_and_efficiency": 10, + }, + "evidence": [], + "reason": "Useful despite minor inefficiency.", + }, + ] + ) + result = audit_trajectory(_record(), client) # type: ignore[arg-type] + assert len(client.calls) == 2 + assert result["prefix"]["retained_turn_count"] == 6 + assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1 + assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE" diff --git a/tests/test_evidence.py b/tests/test_evidence.py index dc336b7..3e86eba 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -1,6 +1,11 @@ """Tests for prompt-only trajectory compaction.""" -from swe_data_processing.evidence import compact_text, compact_trajectory +from swe_data_processing.evidence import ( + BOUNDARY_BLOCK_TURNS, + build_trajectory_blocks, + compact_text, + compact_trajectory, +) def test_compact_text_preserves_short_values() -> None: @@ -36,3 +41,33 @@ def test_compact_trajectory_prioritizes_test_observations() -> None: compacted, metadata = compact_trajectory(trajectory, signals) assert compacted[2]["content"] == long_content assert 3 in metadata["important_turn_ids"] + + +def test_boundary_blocks_preserve_absolute_turns_without_overlap() -> None: + trajectory = [ + {"turn_id": turn, "role": "assistant", "content": f"turn {turn}"} + for turn in range(1, BOUNDARY_BLOCK_TURNS + 3) + ] + blocks, metadata = build_trajectory_blocks(trajectory, {}) + assert len(blocks) == 2 + assert blocks[0]["start_turn"] == 1 + assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS + assert blocks[1]["start_turn"] == BOUNDARY_BLOCK_TURNS + 1 + assert blocks[1]["end_turn"] == BOUNDARY_BLOCK_TURNS + 2 + turn_ids = [ + message["turn_id"] for block in blocks for message in block["messages"] + ] + assert turn_ids == list(range(1, BOUNDARY_BLOCK_TURNS + 3)) + assert metadata["block_count"] == 2 + + +def test_boundary_block_keeps_immediate_tool_result_with_assistant() -> None: + trajectory = [ + {"turn_id": turn, "role": "user", "content": f"turn {turn}"} + for turn in range(1, BOUNDARY_BLOCK_TURNS + 2) + ] + trajectory[BOUNDARY_BLOCK_TURNS - 1]["role"] = "assistant" + trajectory[BOUNDARY_BLOCK_TURNS]["role"] = "tool" + blocks, _ = build_trajectory_blocks(trajectory, {}) + assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS + 1 + assert len(blocks) == 1