From 4a062134d3494694d56c2355497da1ca270a4b70 Mon Sep 17 00:00:00 2001 From: jiachun <2609024259@qq.com> Date: Tue, 18 Aug 2026 17:36:36 +0800 Subject: [PATCH] Replace LLM cleanup with deterministic profiling --- .env.example | 15 - README.md | 437 +++++------------- changelog.md | 29 +- pyproject.toml | 15 +- scripts/merge_audits_in_source_order.py | 61 --- scripts/sample_open_swe_filtered.py | 94 ---- scripts/split_jsonl_remaining.py | 74 --- src/swe_data_processing/__init__.py | 8 +- src/swe_data_processing/audit.py | 195 -------- src/swe_data_processing/cli.py | 317 ++++--------- src/swe_data_processing/client.py | 206 --------- src/swe_data_processing/config.py | 93 ---- src/swe_data_processing/evidence.py | 194 -------- src/swe_data_processing/features.py | 283 ------------ src/swe_data_processing/heuristics.py | 121 +++++ src/swe_data_processing/io.py | 32 +- src/swe_data_processing/metrics.py | 362 +++++++++++++++ src/swe_data_processing/policy.py | 89 ---- src/swe_data_processing/prompts/boundary.md | 55 --- .../prompts/classification.md | 37 -- .../prompts/prefix_quality.md | 50 -- src/swe_data_processing/prompts/repair.md | 47 -- src/swe_data_processing/prompts/review.md | 16 - src/swe_data_processing/repair.py | 183 -------- src/swe_data_processing/resources.py | 21 - src/swe_data_processing/sampling.py | 90 ++++ .../schemas/boundary_output.schema.json | 44 -- .../schemas/classification_output.schema.json | 128 ----- .../schemas/prefix_quality_output.schema.json | 53 --- .../schemas/repair_output.schema.json | 63 --- .../schemas/review_output.schema.json | 43 -- src/swe_data_processing/summary.py | 178 +++++++ src/swe_data_processing/tokenization.py | 36 ++ src/swe_data_processing/workflow.py | 343 -------------- tests/test_audit.py | 216 --------- tests/test_audit_workflow.py | 165 ------- tests/test_cli.py | 50 +- tests/test_client.py | 103 ----- tests/test_config.py | 29 -- tests/test_evidence.py | 73 --- tests/test_features.py | 100 ---- tests/test_heuristics.py | 84 ++++ tests/test_metrics.py | 123 +++++ tests/test_policy.py | 100 ---- tests/test_repair.py | 54 --- tests/test_sampling.py | 47 ++ tests/test_summary.py | 61 +++ 47 files changed, 1371 insertions(+), 3846 deletions(-) delete mode 100644 .env.example delete mode 100644 scripts/merge_audits_in_source_order.py delete mode 100644 scripts/sample_open_swe_filtered.py delete mode 100644 scripts/split_jsonl_remaining.py delete mode 100644 src/swe_data_processing/audit.py delete mode 100644 src/swe_data_processing/client.py delete mode 100644 src/swe_data_processing/config.py delete mode 100644 src/swe_data_processing/evidence.py delete mode 100644 src/swe_data_processing/features.py create mode 100644 src/swe_data_processing/heuristics.py create mode 100644 src/swe_data_processing/metrics.py delete mode 100644 src/swe_data_processing/policy.py delete mode 100644 src/swe_data_processing/prompts/boundary.md delete mode 100644 src/swe_data_processing/prompts/classification.md delete mode 100644 src/swe_data_processing/prompts/prefix_quality.md delete mode 100644 src/swe_data_processing/prompts/repair.md delete mode 100644 src/swe_data_processing/prompts/review.md delete mode 100644 src/swe_data_processing/repair.py delete mode 100644 src/swe_data_processing/resources.py create mode 100644 src/swe_data_processing/sampling.py delete mode 100644 src/swe_data_processing/schemas/boundary_output.schema.json delete mode 100644 src/swe_data_processing/schemas/classification_output.schema.json delete mode 100644 src/swe_data_processing/schemas/prefix_quality_output.schema.json delete mode 100644 src/swe_data_processing/schemas/repair_output.schema.json delete mode 100644 src/swe_data_processing/schemas/review_output.schema.json create mode 100644 src/swe_data_processing/summary.py create mode 100644 src/swe_data_processing/tokenization.py delete mode 100644 src/swe_data_processing/workflow.py delete mode 100644 tests/test_audit.py delete mode 100644 tests/test_audit_workflow.py delete mode 100644 tests/test_client.py delete mode 100644 tests/test_config.py delete mode 100644 tests/test_evidence.py delete mode 100644 tests/test_features.py create mode 100644 tests/test_heuristics.py create mode 100644 tests/test_metrics.py delete mode 100644 tests/test_policy.py delete mode 100644 tests/test_repair.py create mode 100644 tests/test_sampling.py create mode 100644 tests/test_summary.py diff --git a/.env.example b/.env.example deleted file mode 100644 index 4891ed8..0000000 --- a/.env.example +++ /dev/null @@ -1,15 +0,0 @@ -# The API key must be provided at runtime and must never be committed. -GLM_API_KEY=replace-with-a-runtime-secret - -# The gateway exposes an OpenAI-compatible chat-completions route. -GLM_API_BASE=https://llm-api.cowin.run -GLM_API_PATH=/v1/chat/completions -GLM_MODEL=glm-5.2 - -# Conservative defaults for structured quality-control output. -GLM_TIMEOUT_SECONDS=300 -GLM_MAX_RETRIES=5 -GLM_MAX_TOKENS=8192 -GLM_TEMPERATURE=0.0 -GLM_REASONING_EFFORT=high -GLM_THINKING_ENABLED=true diff --git a/README.md b/README.md index 5a1539c..7780129 100644 --- a/README.md +++ b/README.md @@ -1,364 +1,139 @@ -# SWE Data Processing +# Open-SWE-Traces Deterministic Profiler -`swe-data-processing` is a conservative, auditable Python pipeline for cleaning -[`nvidia/Open-SWE-Traces`](https://huggingface.co/datasets/nvidia/Open-SWE-Traces) -before supervised fine-tuning of smaller coding agents. +This project profiles `nvidia/Open-SWE-Traces` for coding-agent SFT without an +LLM judge, repository sandbox, or trajectory rewriting. It never claims that a +patch is correct. It records facts that can be reproduced from the dataset and +uses a small set of explicit heuristics to remove traces that are definitely +broken or unsuitable for a chosen training context. -The project is designed for a restricted environment in which repository -containers and agent sandboxes are unavailable. It combines deterministic local -checks with GLM-5.2 API judgments. It does **not** claim that static inspection can -prove code correctness. Instead, it separates internally consistent silver -positives, useful negatives, safely repairable formatting issues, unverified -records, and corrupted records. +The local dataset snapshot contains 207,489 trajectories over 22,320 issues: -## Dataset snapshot +| `resolved` | Meaning | Count | +|---:|---|---:| +| `1` | Externally marked successful | 65,244 | +| `0` | Externally marked failed | 95,487 | +| `-1` | Outcome unknown | 46,758 | -The current local snapshot contains 207,489 trajectories over 22,320 unique -issues. The immutable dataset outcome field is named `resolved`: +`resolved` is never changed. Only `resolved=1` is eligible for successful SFT; +the profiler can still describe failed and unknown traces for analysis. Final +decisions label `resolved=0` as `EXCLUDE_FAILED_OUTCOME` and `resolved=-1` as +`HOLD_UNVERIFIED_OUTCOME` rather than recommending them for training. -| `resolved` | Meaning | Count | Percentage | -|---:|---|---:|---:| -| `1` | Successful candidate | 65,244 | 31.44% | -| `0` | Explicit failure | 95,487 | 46.02% | -| `-1` | Unknown outcome | 46,758 | 22.54% | +## Design -Records must be split by `instance_id`, not by trajectory, to prevent the same -issue from leaking across train and evaluation sets. +The pipeline has three commands: -## Safety model +1. `profile` streams the original JSONL or Parquet dataset and writes one + deterministic metrics row per trajectory. +2. `summarize` computes dataset quantiles and writes transparent decisions. +3. `sample` selects full trajectories from every rule and length bucket for + human validation. -Static cleanup may improve representation quality, but it may not create new -execution facts. The implementation enforces these invariants locally after -every model response: +No command edits, truncates, repairs, or invents trajectory turns. Source data +and generated files are joined by `trajectory_id`/`sample_id`. -1. `resolved` is immutable. -2. A `resolved=0` record can never become a full successful SFT example. -3. A `resolved=-1` record cannot become `SFT_FULL` without execution. -4. Existing tool observations cannot be rewritten or fabricated. -5. Model and reference patches are immutable during static repair. -6. Truncated records have a maximum use of `SFT_STEP_ONLY`. -7. A silver positive requires reliable post-edit verification, no hard failures, - and a passing value for every QC dimension. -8. Every repair requires a separate review API call. +## Metrics -The API model proposes decisions and repair plans. Deterministic Python code -validates schemas, enforces outcome policy, applies only allowlisted edits, and -records provenance. +Each profile row contains: -Long trajectories are sent to the API through a turn-preserving evidence view. -Test observations, state-changing turns, malformed calls, final turns, patches, -and stable turn IDs receive priority. Shortened values include their original -character count and SHA-256 hash. This compaction affects only the API prompt; -classification never rewrites source JSONL or Parquet records. +- turn and role counts; +- canonical serialized characters and UTF-8 bytes; +- exact token count when a `tokenizer.json` is supplied; +- tool-call count and count by tool name; +- failed tool-call count and rate; +- observable error type and tool distributions; +- normalized error positions across five equal trajectory regions; +- early-error count and fraction; +- longest consecutive failed-tool-call run; +- malformed calls, unknown tools, missing results, and orphan results. -## API endpoint +The canonical token stream is compact, sorted JSON containing only `tools` and +`trajectory`. This is reproducible but is not presented as a universal chat +template. If training uses another serializer or tokenizer, rerun `profile` +with that exact tokenizer instead of converting characters to fake token counts. -The configured gateway is OpenAI chat-completions compatible: +## Heuristics -```text -POST https://llm-api.cowin.run/v1/chat/completions -``` +The following conditions are hard rejections because they represent broken +training structure or an unambiguous repeated-failure pattern: -`/v1/text-completion` and `/v1/text-completions` resolve to the gateway's web -console rather than an inference API, so they are not used. +- malformed/unknown tool calls or broken assistant/tool pairing; +- at least five consecutive failed tool calls; +- at least five failed calls and a failure rate of at least 50%. -The default model is `glm-5.2`. The API key is read only from `GLM_API_KEY`. Do -not write a key into source code, command history, output manifests, or this -README. +Three weaker patterns are review flags, not automatic rejection: -## Project layout +- error count at or above the dataset-wide p99 threshold; +- at least eight failed calls, a failure rate of at least 20%, and errors in at + least four of five trajectory regions; +- at least three errors with at least 60% of all errors in the first 20% of tool + calls. -```text -swe_data_processing/ -├── pyproject.toml -├── README.md -├── changelog.md # Versioned strategy changes -├── .env.example -├── src/swe_data_processing/ -│ ├── cli.py # Command-line entry points -│ ├── client.py # GLM API client and JSON validation -│ ├── config.py # Environment-only runtime settings -│ ├── evidence.py # Prompt-only evidence compaction -│ ├── features.py # Deterministic static evidence extraction -│ ├── io.py # Streaming Parquet and JSONL readers -│ ├── policy.py # Immutable local policy guards -│ ├── repair.py # Allowlisted deterministic repair application -│ ├── resources.py # Packaged prompt/schema loading -│ ├── workflow.py # Classify, plan repair, and review stages -│ ├── prompts/ # Version-controlled GLM system prompts -│ └── schemas/ # JSON Schemas for every API stage -├── tests/ # Offline unit tests; no network calls -├── scripts/ # Existing sampling and profiling utilities -├── raw/Open-SWE-Traces/ # Downloaded dataset; ignored by Git -├── samples/ # Human-review samples -├── reports/ # Dataset reports and QC rubrics -└── qc_outputs/ # Generated manifests; ignored by Git -``` +Length is separated from quality. Tokenized records are assigned to: + +- `LE_81920`: fast 80K training subset; +- `81921_TO_131072`: 128K training subset; +- `131073_TO_262144`: deferred long-context subset; +- `GT_262144`: excluded from the default small-model training run. + +The summary reports how much of the total token mass is contributed by the +longest 0.1%, 1%, and 5% of samples. This makes long-tail removal a measurable +dataset decision rather than a guess. ## Installation -The existing remote virtual environment can install the package in editable -mode: - ```bash -cd /mnt/beegfs/yi/swe_data_processing -./.venv/bin/pip install -e '.[dev]' +cd /home/kxqandccx/kxq/tomlu/OpenSWETraces_cleanup +./.venv/bin/pip install -e '.[dev,tokens]' ``` -For a fresh environment: +`tokenizers` is optional. Without it, profiling still produces turn, character, +byte, and tool-error metrics, but no token bucket decision. + +## Full workflow + +Use an explicit tokenizer already present on the machine: ```bash -python3 -m venv .venv -./.venv/bin/pip install --upgrade pip -./.venv/bin/pip install -e '.[dev]' -``` +TOKENIZER_JSON=/path/to/tokenizer.json +mkdir -p qc_outputs/deterministic_v3 -## Configuration - -Export credentials in the shell that launches the pipeline: - -```bash -export GLM_API_KEY='your-runtime-secret' -export GLM_API_BASE='https://llm-api.cowin.run' -export GLM_API_PATH='/v1/chat/completions' -export GLM_MODEL='glm-5.2' -``` - -Optional settings and their defaults: - -```bash -export GLM_TIMEOUT_SECONDS=300 -export GLM_MAX_RETRIES=5 -export GLM_MAX_TOKENS=8192 -export GLM_TEMPERATURE=0.0 -export GLM_REASONING_EFFORT=high -export GLM_THINKING_ENABLED=true -``` - -`GLM_MAX_RETRIES=5` means one initial request plus at most five retries. The -client retries timeouts, connection failures, HTTP 429/5xx responses, malformed -JSON, and schema-invalid model output with bounded exponential backoff. HTTP -401/403 authentication failures are never retried. - -If the gateway rejects GLM-specific `thinking` or `reasoning_effort` fields, the -client automatically retries with the portable OpenAI-compatible request subset. - -## Commands - -### 1. Verify API authentication and structured output - -```bash -swe-qc smoke-test -``` - -The command prints the endpoint, model, request ID, usage, and a tiny validated -JSON response. It never prints the API key. - -### 2. Extract deterministic evidence without API calls - -From the 20-record review sample: - -```bash -mkdir -p qc_outputs -swe-qc features \ - --input samples/sample_20_seed_20260805.jsonl \ - --output qc_outputs/sample20.features.jsonl \ - --errors qc_outputs/sample20.features.errors.jsonl -``` - -From all Parquet shards: - -```bash -swe-qc features \ +swe-qc profile \ --input raw/Open-SWE-Traces \ - --output qc_outputs/all.features.jsonl \ - --errors qc_outputs/all.features.errors.jsonl \ + --output qc_outputs/deterministic_v3/metrics.jsonl \ + --errors qc_outputs/deterministic_v3/profile.errors.jsonl \ + --tokenizer-json "$TOKENIZER_JSON" \ + --workers 8 \ --resume + +swe-qc summarize \ + --metrics qc_outputs/deterministic_v3/metrics.jsonl \ + --summary qc_outputs/deterministic_v3/summary.json \ + --decisions qc_outputs/deterministic_v3/decisions.jsonl + +swe-qc sample \ + --input raw/Open-SWE-Traces \ + --decisions qc_outputs/deterministic_v3/decisions.jsonl \ + --output qc_outputs/deterministic_v3/review_sample.jsonl \ + --per-group 20 \ + --seed 20260818 ``` -Features include malformed tool arguments, unknown tools, role alternation, -state-changing turns, post-edit test evidence, masked shell pipelines, patch file -sets, patch size ratios, and explicit user constraints. +`profile --resume` skips sample IDs already present in the metrics file. Writes +are append-only and flushed per record. `summarize` and `sample` atomically +replace their outputs so a partial file is never mistaken for a complete run. +Any dataset subdirectory containing Parquet shards is also a valid `--input`, +so the four trajectory families can be profiled in separate CPU processes and +their metrics concatenated before `summarize`. +Profile rows and decisions record the absolute source Parquet shard. `sample` +uses that provenance to read only shards containing selected IDs instead of +scanning the complete dataset. -### 3. Classify trajectories through GLM-5.2 +## Error detection boundary -Run a small pilot first: - -```bash -swe-qc classify \ - --input samples/sample_20_seed_20260805.jsonl \ - --output qc_outputs/sample20.classifications.jsonl \ - --errors qc_outputs/sample20.classification.errors.jsonl \ - --limit 20 \ - --resume -``` - -The classifier returns one of: - -- `ACCEPT_SILVER_POSITIVE` -- `ACCEPT_NEGATIVE` -- `STATIC_REPAIR` -- `HOLD_UNVERIFIED` -- `REJECT` - -Training use is tracked separately as `SFT_FULL`, `SFT_STEP_ONLY`, -`DPO_REJECTED`, `ERROR_ANALYSIS`, `HOLD`, or `DROP`. - -### 4. Locate a safe prefix and score it independently - -`audit` uses two isolated GLM calls for failed and unknown trajectories: - -1. The boundary call receives the trajectory as roughly 32-turn blocks without - splitting an assistant action from its immediate tool result. It cannot see - `resolved` or reference patches, but it does see the trajectory's own final - model patch to check task coverage and diff pollution. It selects the earliest - unrepaired major/critical assistant turn, keeps a safe full process trace, - or returns `HOLD` when neither decision is supported. -2. Python slices the exact messages before that assistant turn. The quality call - receives only this materialized prefix, so suffix behavior cannot affect its - score. - -For a complete trajectory, the quality call also sees its final model patch. A -truncated prefix never receives that suffix-derived patch. State-changing turns -are recorded as audit telemetry but are not automatic boundaries: writing code -is not itself an error, and recovered experiments are useful process data. The -effective boundary is the earliest unrecovered major or critical behavior found -by the boundary stage. The isolated quality stage rejects a retained prefix that -still contains an unrecovered severe problem. - -Successful trajectories skip boundary selection and are scored as complete -trajectories. The command remains simple: - -```bash -swe-qc audit \ - --input samples/sample_20_seed_20260805.jsonl \ - --output qc_outputs/sample20.audits.jsonl \ - --errors qc_outputs/sample20.audit.errors.jsonl \ - --workers 5 \ - --resume -``` - -`--workers` bounds the number of records processed concurrently. JSONL writes -remain serialized in the main thread, so each completed record is appended -atomically even when API requests run in parallel. Output order follows request -completion order; `sample_id` remains the stable join key. - -The gateway previously returned frequent HTTP 429 responses at 20 workers, so -five workers is the practical default for long runs unless the service limit is -raised. - -The quality call lists concrete erroneous and inefficient assistant actions and -scores five 0-20 dimensions: planning, investigation, tool use and observation, -progress, and clarity/efficiency. Python recomputes issue counts, sums the five -dimensions, and assigns: - -- `HIGH`: 80-100 -- `MEDIUM`: 60-79 -- `LOW`: 40-59 -- `REJECT`: below 40, or an unrepaired major/critical problem remains - -Incomplete prefixes can still be useful. Normal failed experiments are retained -when the agent later diagnoses, repairs, and verifies them. A prefix ending in a -complete tool result is valid process-SFT structure. - -### 5. Create static repair plans - -Only classifications that explicitly return `STATIC_REPAIR` are processed by -default: - -```bash -swe-qc repair-plan \ - --input samples/sample_20_seed_20260805.jsonl \ - --classifications qc_outputs/sample20.classifications.jsonl \ - --output qc_outputs/sample20.repair-plans.jsonl \ - --errors qc_outputs/sample20.repair-plan.errors.jsonl \ - --resume -``` - -The model may propose only operations from the allowlist. Code changes, test -result synthesis, patch replacement, and outcome upgrades are prohibited. - -### 6. Apply plans deterministically - -```bash -swe-qc apply-repair \ - --input samples/sample_20_seed_20260805.jsonl \ - --plans qc_outputs/sample20.repair-plans.jsonl \ - --output qc_outputs/sample20.repaired.jsonl \ - --errors qc_outputs/sample20.apply.errors.jsonl \ - --resume -``` - -The applier checks target turns, read-only pair deletion, immutable patch fields, -and retained tool-output hashes. It emits a structured diff with input and output -SHA-256 hashes. - -### 7. Independently review repairs - -```bash -swe-qc review \ - --input samples/sample_20_seed_20260805.jsonl \ - --classifications qc_outputs/sample20.classifications.jsonl \ - --plans qc_outputs/sample20.repair-plans.jsonl \ - --repaired qc_outputs/sample20.repaired.jsonl \ - --output qc_outputs/sample20.reviews.jsonl \ - --errors qc_outputs/sample20.review.errors.jsonl \ - --resume -``` - -Approval means that a static repair preserved evidence and structure. It does not -mean that the code patch was executed or proved correct. - -## Output and resume behavior - -Pipeline manifests are append-only JSONL. `--resume` reads completed sample IDs -from the output and skips them. Each API response includes non-sensitive -provenance: - -- model and endpoint; -- gateway request ID; -- token usage when available; -- input SHA-256; -- compatibility-fallback flag; -- UTC creation time. - -Errors are written as compact records containing sample ID, stage, exception -type, and a bounded message. Raw prompts and credentials are not copied into -error logs. - -## Recommended rollout - -1. Run offline tests. -2. Run `smoke-test` once. -3. Run classification on the 20 manually reviewed records. -4. Compare GLM decisions with the human labels. -5. Build a 2,000-record stratified calibration set. -6. Require at least 95% precision for `ACCEPT_SILVER_POSITIVE` before scaling. -7. Keep `resolved=0` as negative data rather than attempting to turn it into - positive trajectories. -8. Keep `resolved=-1` in an unverified manifest unless it contains explicit, - reliable failure evidence. - -## Development - -Run all offline tests: - -```bash -./.venv/bin/pytest -``` - -Run lint checks: - -```bash -./.venv/bin/ruff check src tests -``` - -Tests use an `httpx.MockTransport` and never contact the GLM endpoint. - -## Credential handling - -- `.env` files are ignored by Git. -- The provided API key is not stored anywhere in this project. -- Use a secret manager or a protected runtime environment variable for batch - jobs. -- Rotate the key if it has been copied into a public log, issue, or repository. +The detector intentionally favors precision over recall. It recognizes explicit +non-zero exit codes, test/build failure summaries, timeouts, permission errors, +missing commands/files, and tool exceptions. It does not treat the mere word +"error" as a failure, because tools often print source code or logs containing +that word. Human samples should be used to refine patterns only when the raw +tool observation provides an unambiguous signal. diff --git a/changelog.md b/changelog.md index 04adc06..c6dee0b 100644 --- a/changelog.md +++ b/changelog.md @@ -4,7 +4,34 @@ This file records strategy changes that materially affect dataset decisions or training-data semantics. Generated audit manifests are not treated as stable API contracts. -## Unreleased +## 3.0.0 - 2026-08-18 + +### Strategy replacement + +- Removed the GLM client, prompts, schemas, model-based classification, boundary + selection, quality scoring, repair planning, and repair review. +- Replaced model judgments with deterministic length, token, tool-failure, + failure-position, structure, and streak metrics. +- Added explicit 80K, 128K, 256K, and over-256K context buckets. +- Added dataset-level quantiles, token-tail contribution statistics, p99 tool + error outlier detection, and stratified human-review sampling. +- Reserved automatic rejection for broken tool structure, five consecutive + failures, or a failure rate of at least 50%. Distributed failures, relative + error outliers, and early failure clusters require review. +- Token counts now require an explicit local `tokenizer.json`; the code never + substitutes a character-based approximation. +- Reduced the public CLI to `profile`, `summarize`, and `sample`. + +### Motivation + +- Multi-call LLM auditing was operationally unstable and could hallucinate safe + boundaries or miss incorrect final patches. +- Model-generated trajectory repair had no deterministic invariant capable of + proving that the rewritten trace remained correct. +- The new pipeline is reproducible, inspectable, CPU-only, and does not mutate + source trajectories. + +## 2.1.0 - 2026-08-09 ### Strategy correction diff --git a/pyproject.toml b/pyproject.toml index 0272758..3b99741 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "swe-data-processing" -version = "2.0.0" -description = "Static quality control and repair planning for Open-SWE-Traces." +version = "3.0.0" +description = "Deterministic profiling and filtering for Open-SWE-Traces." readme = "README.md" requires-python = ">=3.10" license = {text = "MIT"} @@ -13,12 +13,13 @@ authors = [ {name = "TIGER Lab"} ] dependencies = [ - "httpx>=0.27,<1", - "jsonschema>=4.23,<5", "pyarrow>=16,<26" ] [project.optional-dependencies] +tokens = [ + "tokenizers>=0.21,<1" +] dev = [ "pytest>=8,<10", "ruff>=0.9,<1" @@ -34,12 +35,6 @@ include-package-data = true [tool.setuptools.packages.find] where = ["src"] -[tool.setuptools.package-data] -swe_data_processing = [ - "prompts/*.md", - "schemas/*.json" -] - [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q" diff --git a/scripts/merge_audits_in_source_order.py b/scripts/merge_audits_in_source_order.py deleted file mode 100644 index 57d04f9..0000000 --- a/scripts/merge_audits_in_source_order.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -"""Merge audit JSONL manifests into one unique source-ordered file.""" - -from __future__ import annotations - -import argparse -import json -import os -from pathlib import Path - - -def main() -> None: - """Merge the last result for each sample and write it in source order.""" - - parser = argparse.ArgumentParser() - parser.add_argument("source", type=Path, help="Source sample JSONL defining output order") - parser.add_argument("output", type=Path) - parser.add_argument("manifests", type=Path, nargs="+") - parser.add_argument("--require-count", type=int) - args = parser.parse_args() - - audits: dict[str, dict] = {} - for path in args.manifests: - if not path.exists(): - continue - with path.open(encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - value = json.loads(line) - sample_id = value.get("sample_id") - if not isinstance(sample_id, str) or not sample_id: - raise ValueError(f"Audit in {path} has no sample_id") - audits[sample_id] = value - - ordered: list[dict] = [] - with args.source.open(encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - source_record = json.loads(line) - sample_id = source_record.get("trajectory_id") or source_record.get("sample_id") - if sample_id in audits: - ordered.append(audits.pop(sample_id)) - - if audits: - raise ValueError(f"Found {len(audits)} audit IDs absent from the source sample") - if args.require_count is not None and len(ordered) != args.require_count: - raise ValueError(f"Expected {args.require_count} merged records, found {len(ordered)}") - - args.output.parent.mkdir(parents=True, exist_ok=True) - temporary = args.output.with_suffix(args.output.suffix + ".tmp") - with temporary.open("w", encoding="utf-8") as handle: - for value in ordered: - handle.write(json.dumps(value, ensure_ascii=False, default=str) + "\n") - os.replace(temporary, args.output) - print(json.dumps({"output": str(args.output), "records": len(ordered)}, indent=2)) - - -if __name__ == "__main__": - main() diff --git a/scripts/sample_open_swe_filtered.py b/scripts/sample_open_swe_filtered.py deleted file mode 100644 index 872a1b0..0000000 --- a/scripts/sample_open_swe_filtered.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -"""Create a deterministic uniform sample from selected Open-SWE outcome classes.""" - -from __future__ import annotations - -import argparse -import json -import random -from pathlib import Path - -import pyarrow.parquet as pq - - -def main() -> None: - """Sample complete rows uniformly from records matching the requested outcomes.""" - - parser = argparse.ArgumentParser() - parser.add_argument("dataset_dir", type=Path) - parser.add_argument("output", type=Path) - parser.add_argument("--count", type=int, default=50) - parser.add_argument("--seed", type=int, default=20260806) - parser.add_argument( - "--resolved", - type=int, - nargs="+", - default=[0, -1], - help="Outcome values eligible for sampling; defaults to failed and unknown trajectories.", - ) - args = parser.parse_args() - - files = sorted(args.dataset_dir.glob("data/**/*.parquet")) - if not files: - raise SystemExit(f"No parquet files found under {args.dataset_dir}") - - allowed = set(args.resolved) - eligible: list[tuple[Path, int]] = [] - for path in files: - # Reading only the outcome column keeps the full-dataset eligibility pass inexpensive. - outcomes = pq.read_table(path, columns=["resolved"])["resolved"].to_pylist() - eligible.extend((path, index) for index, value in enumerate(outcomes) if value in allowed) - - if args.count > len(eligible): - raise SystemExit(f"Requested {args.count} rows from only {len(eligible)} eligible rows") - - rng = random.Random(args.seed) - chosen = rng.sample(eligible, args.count) - selected: list[dict] = [] - for path, file_row_index in chosen: - parquet_file = pq.ParquetFile(path) - row_group_start = 0 - for row_group_index in range(parquet_file.num_row_groups): - row_group_rows = parquet_file.metadata.row_group(row_group_index).num_rows - if file_row_index < row_group_start + row_group_rows: - table = parquet_file.read_row_group(row_group_index) - row = table.slice(file_row_index - row_group_start, 1).to_pylist()[0] - row["_sample"] = { - "sampling_method": "uniform without replacement over selected resolved values", - "resolved_values": sorted(allowed), - "seed": args.seed, - "source_file": str(path.relative_to(args.dataset_dir)), - "file_row_index": file_row_index, - "row_group_index": row_group_index, - } - selected.append(row) - break - row_group_start += row_group_rows - else: - raise RuntimeError(f"Could not locate row {file_row_index} in {path}") - - args.output.parent.mkdir(parents=True, exist_ok=True) - with args.output.open("w", encoding="utf-8") as handle: - for row in selected: - handle.write(json.dumps(row, ensure_ascii=False, default=str) + "\n") - - outcome_counts = {str(value): 0 for value in sorted(allowed)} - for row in selected: - outcome_counts[str(row["resolved"])] += 1 - print( - json.dumps( - { - "output": str(args.output), - "seed": args.seed, - "eligible_rows": len(eligible), - "sample_rows": len(selected), - "sample_outcome_counts": outcome_counts, - }, - ensure_ascii=False, - indent=2, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/split_jsonl_remaining.py b/scripts/split_jsonl_remaining.py deleted file mode 100644 index 8b5f9d3..0000000 --- a/scripts/split_jsonl_remaining.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -"""Split unfinished JSONL records into balanced temporary retry shards.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - - -def record_id(value: dict) -> str: - """Return the stable trajectory identifier used by audit manifests.""" - - sample_id = value.get("trajectory_id") or value.get("sample_id") - if not isinstance(sample_id, str) or not sample_id: - raise ValueError("JSONL record has no trajectory_id/sample_id") - return sample_id - - -def main() -> None: - """Write unfinished input records round-robin across the requested shards.""" - - parser = argparse.ArgumentParser() - parser.add_argument("input", type=Path) - parser.add_argument("completed", type=Path) - parser.add_argument("output_dir", type=Path) - parser.add_argument("--shards", type=int, default=3) - args = parser.parse_args() - if args.shards < 1: - raise SystemExit("--shards must be positive") - - completed_ids: set[str] = set() - with args.completed.open(encoding="utf-8") as handle: - for line in handle: - if line.strip(): - completed_ids.add(record_id(json.loads(line))) - - args.output_dir.mkdir(parents=True, exist_ok=True) - handles = [ - (args.output_dir / f"part_{index}.input.jsonl").open("w", encoding="utf-8") - for index in range(args.shards) - ] - counts = [0] * args.shards - try: - unfinished = 0 - with args.input.open(encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - value = json.loads(line) - if record_id(value) in completed_ids: - continue - shard = unfinished % args.shards - handles[shard].write(line) - counts[shard] += 1 - unfinished += 1 - finally: - for handle in handles: - handle.close() - - print( - json.dumps( - { - "completed": len(completed_ids), - "unfinished": sum(counts), - "shard_counts": counts, - }, - indent=2, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/src/swe_data_processing/__init__.py b/src/swe_data_processing/__init__.py index c100162..bfc010e 100644 --- a/src/swe_data_processing/__init__.py +++ b/src/swe_data_processing/__init__.py @@ -1,6 +1,6 @@ -"""Static quality-control utilities for the Open-SWE-Traces dataset.""" +"""Deterministic profiling utilities for the Open-SWE-Traces dataset.""" -from .config import Settings +from .metrics import extract_metrics -__all__ = ["Settings"] -__version__ = "2.0.0" +__all__ = ["extract_metrics"] +__version__ = "3.0.0" diff --git a/src/swe_data_processing/audit.py b/src/swe_data_processing/audit.py deleted file mode 100644 index b315fa3..0000000 --- a/src/swe_data_processing/audit.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Validation and deterministic scoring for the two-call audit pipeline.""" - -from __future__ import annotations - -from typing import Any - -from .features import extract_static_signals, index_trajectory -from .io import get_sample_id -from .policy import PolicyViolation - -QUALITY_DIMENSIONS = ( - "planning", - "investigation", - "tool_use_and_observation", - "progress", - "clarity_and_efficiency", -) - - -def validate_boundary( - record: dict[str, Any], - result: dict[str, Any], -) -> None: - """Reject structurally inconsistent or ungrounded boundary decisions.""" - - trajectory = index_trajectory(record) - messages = {message["turn_id"]: message for message in trajectory} - assistant_turns = { - message["turn_id"] for message in trajectory if message.get("role") == "assistant" - } - if result["sample_id"] != get_sample_id(record): - raise PolicyViolation("Boundary sample_id does not match the source record") - - decision = result["decision"] - boundary = result["truncate_before_turn"] - if decision in {"KEEP_FULL", "HOLD"}: - if boundary is not None: - raise PolicyViolation(f"A {decision} decision cannot contain a boundary") - if result["category"] != "NONE" or result["severity"] != "NONE": - raise PolicyViolation(f"A {decision} decision must use NONE category and severity") - if decision == "KEEP_FULL" and result["checks"] != { - "task_coverage": "COMPLETE", - "final_patch_scope": "CLEAN", - "constraints": "RESPECTED", - "claims_vs_observations": "CONSISTENT", - }: - raise PolicyViolation("KEEP_FULL requires all four checks to pass") - return - - if boundary not in assistant_turns: - raise PolicyViolation("truncate_before_turn must reference an assistant turn") - if result["severity"] not in {"MAJOR", "CRITICAL"}: - raise PolicyViolation("A truncation boundary requires MAJOR or CRITICAL severity") - - evidence_turns = result["evidence_turns"] - if boundary not in evidence_turns: - raise PolicyViolation("Boundary evidence must include the excluded assistant turn") - for turn_id in evidence_turns: - if turn_id not in messages: - raise PolicyViolation("Boundary evidence references a missing turn") - - -def materialize_prefix( - record: dict[str, Any], boundary: int | None -) -> list[dict[str, Any]]: - """Return the exact retained messages without rewriting any source content.""" - - trajectory = record.get("trajectory") or [] - return list(trajectory if boundary is None else trajectory[: boundary - 1]) - - -def effective_boundary_policy( - record: dict[str, Any], model_result: dict[str, Any] -) -> dict[str, Any]: - """Use the semantic boundary while retaining the old cap as telemetry. - - A state-changing turn is not evidence of an error by itself. Normal coding, - failed experiments, recovery, and verification are useful process data, so - the first stateful turn must not become an unconditional cutoff. Prefix - safety is enforced independently by the isolated quality stage. - """ - - model_boundary = model_result["truncate_before_turn"] - first_stateful = min( - extract_static_signals(record)["stateful_turns"], default=None - ) - if model_result["decision"] == "TRUNCATE": - return { - "decision": "TRUNCATE", - "truncate_before_turn": model_boundary, - "source": "MODEL_BOUNDARY", - "model_truncate_before_turn": model_boundary, - "first_stateful_turn": first_stateful, - } - return { - "decision": model_result["decision"], - "truncate_before_turn": None, - "source": "MODEL_DECISION", - "model_truncate_before_turn": model_boundary, - "first_stateful_turn": first_stateful, - } - - -def validate_prefix_quality( - sample_id: str, - prefix: list[dict[str, Any]], - result: dict[str, Any], -) -> None: - """Validate a quality judgment that was made from the prefix alone.""" - - if result["sample_id"] != sample_id: - raise PolicyViolation("Quality sample_id does not match the source record") - - messages = {message["turn_id"]: message for message in index_trajectory({"trajectory": prefix})} - for issue in result["behavior_issues"]: - if issue["turn_id"] not in messages: - raise PolicyViolation("Behavior issue references a turn outside the prefix") - unrecovered_severe = [ - issue - for issue in result["behavior_issues"] - if issue["severity"] in {"MAJOR", "CRITICAL"} and not issue["recovered"] - ] - if unrecovered_severe and not any( - issue["turn_id"] in result["evidence_turns"] for issue in unrecovered_severe - ): - raise PolicyViolation("Unsafe prefix evidence must include an unrecovered severe issue") - for turn_id in result["evidence_turns"]: - if turn_id not in messages: - raise PolicyViolation("Quality evidence references a turn outside the prefix") - - -def derive_prefix_safety(result: dict[str, Any]) -> dict[str, Any]: - """Derive safety from model-listed issues instead of redundant booleans.""" - - unrecovered_severe = [ - issue - for issue in result["behavior_issues"] - if issue["severity"] in {"MAJOR", "CRITICAL"} and not issue["recovered"] - ] - return { - "prefix_valid": not unrecovered_severe, - "unrecovered_major_or_critical": bool(unrecovered_severe), - } - - -def materialize_evidence( - trajectory: list[dict[str, Any]], turn_ids: list[int] -) -> list[dict[str, Any]]: - """Attach deterministic source excerpts to model-selected turn IDs.""" - - messages = { - message["turn_id"]: message - for message in index_trajectory({"trajectory": trajectory}) - } - evidence = [] - for turn_id in turn_ids: - message = messages[turn_id] - value = message.get("content") - if not value and message.get("tool_calls"): - value = message["tool_calls"] - excerpt = str(value)[:600] - evidence.append( - {"turn_id": turn_id, "role": message.get("role"), "excerpt": excerpt} - ) - return evidence - - -def compute_prefix_quality(result: dict[str, Any]) -> dict[str, Any]: - """Compute the final score and tier from five model-provided dimensions.""" - - score = sum(int(result["dimensions"][name]) for name in QUALITY_DIMENSIONS) - safety = derive_prefix_safety(result) - if not safety["prefix_valid"] or score < 40: - tier = "REJECT" - elif score >= 80: - tier = "HIGH" - elif score >= 60: - tier = "MEDIUM" - else: - tier = "LOW" - issues = result["behavior_issues"] - return { - "educational_quality_score": score, - "quality_tier": tier, - "issue_counts": { - "errors": sum(issue["kind"] == "ERROR" for issue in issues), - "inefficiencies": sum(issue["kind"] == "INEFFICIENCY" for issue in issues), - "minor": sum(issue["severity"] == "MINOR" for issue in issues), - "major": sum(issue["severity"] == "MAJOR" for issue in issues), - "critical": sum(issue["severity"] == "CRITICAL" for issue in issues), - "recovered": sum(issue["recovered"] for issue in issues), - "unrecovered": sum(not issue["recovered"] for issue in issues), - }, - "formula_version": "five-equal-dimensions-v2", - } diff --git a/src/swe_data_processing/cli.py b/src/swe_data_processing/cli.py index 83c5775..2d15b95 100644 --- a/src/swe_data_processing/cli.py +++ b/src/swe_data_processing/cli.py @@ -1,4 +1,4 @@ -"""Command-line interface for static Open-SWE-Traces quality control.""" +"""Command-line interface for deterministic Open-SWE-Traces profiling.""" from __future__ import annotations @@ -10,20 +10,27 @@ from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait from pathlib import Path from typing import Any -from .client import GLMClient -from .config import Settings -from .features import extract_static_signals -from .io import append_jsonl, get_sample_id, iter_records, load_completed_ids, load_jsonl_index -from .repair import apply_static_repair -from .workflow import audit_trajectory, classify_record, plan_static_repair, review_repaired_record +from .heuristics import classify_metrics +from .io import ( + append_jsonl, + get_sample_id, + iter_jsonl, + iter_records, + load_completed_ids, + write_json, + write_jsonl, +) +from .metrics import extract_metrics +from .sampling import iter_review_records, select_review_ids +from .summary import build_summary +from .tokenization import TokenCounter -def _write_error(path: Path | None, sample_id: str, stage: str, error: Exception) -> None: - """Write a compact non-sensitive error record for later retry.""" +def _write_error(path: Path | None, sample_id: str, error: Exception) -> None: + """Write a compact error without interrupting a long profile run.""" value = { "sample_id": sample_id, - "stage": stage, "error_type": type(error).__name__, "error": str(error)[:2000], } @@ -41,10 +48,9 @@ def _run_streaming_stage( limit: int | None, resume: bool, workers: int, - stage_name: str, - processor: Callable[[dict[str, Any]], dict[str, Any] | None], + processor: Callable[[dict[str, Any]], dict[str, Any]], ) -> int: - """Run one append-only stage with resume support and progress reporting.""" + """Profile records concurrently while serializing writes in one thread.""" if workers < 1: raise ValueError("workers must be at least 1") @@ -55,8 +61,7 @@ def _run_streaming_stage( def selected_records() -> Iterator[dict[str, Any]]: selected = 0 for record in iter_records(input_path): - sample_id = get_sample_id(record) - if sample_id in completed: + if get_sample_id(record) in completed: continue if limit is not None and selected >= limit: break @@ -68,58 +73,55 @@ def _run_streaming_stage( ) -> tuple[str, dict[str, Any] | None, Exception | None]: sample_id = get_sample_id(record) try: - result = processor(record) - except Exception as exc: # noqa: BLE001 - each sample must fail independently. + return sample_id, processor(record), None + except Exception as exc: # noqa: BLE001 - isolate individual records. return sample_id, None, exc - return sample_id, result, None def record_outcome( - outcome: tuple[str, dict[str, Any] | None, Exception | None], + outcome: tuple[str, dict[str, Any] | None, Exception | None] ) -> None: nonlocal processed, failures sample_id, result, error = outcome if error is not None: failures += 1 - _write_error(errors_path, sample_id, stage_name, error) + _write_error(errors_path, sample_id, error) elif result is not None: append_jsonl(output_path, result) processed += 1 - if processed % 10 == 0: - print(f"{stage_name}: processed={processed} failures={failures}", file=sys.stderr) + if processed % 1000 == 0: + print(f"profile: processed={processed} failures={failures}", file=sys.stderr) records = iter(selected_records()) if workers == 1: for record in records: record_outcome(process_one(record)) else: - with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=stage_name) as executor: + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="profile") as executor: pending: set[ Future[tuple[str, dict[str, Any] | None, Exception | None]] ] = set() for _ in range(workers): try: - record = next(records) + pending.add(executor.submit(process_one, next(records))) except StopIteration: break - pending.add(executor.submit(process_one, record)) - while pending: finished, pending = wait(pending, return_when=FIRST_COMPLETED) for future in finished: record_outcome(future.result()) try: - record = next(records) + pending.add(executor.submit(process_one, next(records))) except StopIteration: - continue - pending.add(executor.submit(process_one, record)) + pass - print(f"{stage_name}: completed={processed} failures={failures}", file=sys.stderr) + print(f"profile: completed={processed} failures={failures}", file=sys.stderr) return 1 if failures else 0 -def command_features(args: argparse.Namespace) -> int: - """Extract deterministic features without requiring an API key.""" +def command_profile(args: argparse.Namespace) -> int: + """Extract per-trajectory metrics without modifying source records.""" + counter = TokenCounter(args.tokenizer_json) if args.tokenizer_json else None return _run_streaming_stage( input_path=args.input, output_path=args.output, @@ -127,236 +129,77 @@ def command_features(args: argparse.Namespace) -> int: limit=args.limit, resume=args.resume, workers=args.workers, - stage_name="features", - processor=lambda record: { - "sample_id": get_sample_id(record), - "resolved": record.get("resolved"), - "static_signals": extract_static_signals(record), - }, + processor=lambda record: extract_metrics(record, counter), ) -def command_classify(args: argparse.Namespace) -> int: - """Classify records through GLM-5.2 and local policy enforcement.""" +def command_summarize(args: argparse.Namespace) -> int: + """Derive global thresholds, summary statistics, and decisions.""" - settings = Settings.from_env() - with GLMClient(settings) as client: - return _run_streaming_stage( - input_path=args.input, - output_path=args.output, - errors_path=args.errors, - limit=args.limit, - resume=args.resume, - workers=args.workers, - stage_name="classification", - processor=lambda record: classify_record(record, client), - ) - - -def command_audit(args: argparse.Namespace) -> int: - """Locate a safe boundary, materialize its prefix, then score the prefix.""" - - settings = Settings.from_env() - with GLMClient(settings) as client: - return _run_streaming_stage( - input_path=args.input, - output_path=args.output, - errors_path=args.errors, - limit=args.limit, - resume=args.resume, - workers=args.workers, - stage_name="two_call_audit", - processor=lambda record: audit_trajectory(record, client), - ) - - -def command_repair_plan(args: argparse.Namespace) -> int: - """Ask GLM-5.2 for allowlisted static repair plans.""" - - classifications = load_jsonl_index(args.classifications) - settings = Settings.from_env() - with GLMClient(settings) as client: - def processor(record: dict[str, Any]) -> dict[str, Any] | None: - sample_id = get_sample_id(record) - classification = classifications[sample_id] - if classification["qc_decision"] != "STATIC_REPAIR" and not args.include_nonrepair: - return None - return plan_static_repair(record, classification, client) - - return _run_streaming_stage( - input_path=args.input, - output_path=args.output, - errors_path=args.errors, - limit=args.limit, - resume=args.resume, - workers=args.workers, - stage_name="repair_plan", - processor=processor, - ) - - -def command_apply_repair(args: argparse.Namespace) -> int: - """Apply validated static repair plans deterministically without API calls.""" - - plans = load_jsonl_index(args.plans) - completed = load_completed_ids(args.output) if args.resume else set() - processed = 0 - failures = 0 - for record in iter_records(args.input): - sample_id = get_sample_id(record) - if sample_id in completed or sample_id not in plans: - continue - if args.limit is not None and processed >= args.limit: - break - try: - repaired, diff = apply_static_repair(record, plans[sample_id]) - repaired["sample_id"] = sample_id - repaired["static_repair_diff"] = diff - append_jsonl(args.output, repaired) - except Exception as exc: # noqa: BLE001 - failures += 1 - _write_error(args.errors, sample_id, "apply_repair", exc) - processed += 1 - print(f"apply_repair: completed={processed} failures={failures}", file=sys.stderr) - return 1 if failures else 0 - - -def command_review(args: argparse.Namespace) -> int: - """Run independent GLM review of applied static repairs.""" - - classifications = load_jsonl_index(args.classifications) - plans = load_jsonl_index(args.plans) - repaired_records = load_jsonl_index(args.repaired) - settings = Settings.from_env() - with GLMClient(settings) as client: - def processor(record: dict[str, Any]) -> dict[str, Any] | None: - sample_id = get_sample_id(record) - if sample_id not in repaired_records: - return None - repaired = repaired_records[sample_id] - return review_repaired_record( - original_record=record, - classification=classifications[sample_id], - repair_plan=plans[sample_id], - repaired_record=repaired, - structured_diff=repaired.get("static_repair_diff") or {}, - client=client, - ) - - return _run_streaming_stage( - input_path=args.input, - output_path=args.output, - errors_path=args.errors, - limit=args.limit, - resume=args.resume, - workers=args.workers, - stage_name="repair_review", - processor=processor, - ) - - -def command_smoke_test(_: argparse.Namespace) -> int: - """Perform one minimal authenticated structured-output request.""" - - settings = Settings.from_env() - schema = { - "type": "object", - "additionalProperties": False, - "required": ["status", "model"], - "properties": { - "status": {"const": "ok"}, - "model": {"type": "string", "minLength": 1}, - }, - } - with GLMClient(settings) as client: - response = client.invoke_json( - system_prompt=( - "Return a JSON object with status='ok' and model set to the model name you were asked to use." - ), - payload={"requested_model": settings.model}, - schema=schema, - ) - print( - json.dumps( - { - "endpoint": settings.endpoint, - "model": settings.model, - "response": response.data, - "request_id": response.request_id, - "usage": response.usage, - "compatibility_fallback_used": response.compatibility_fallback_used, - }, - ensure_ascii=False, - indent=2, - ) + summary, threshold = build_summary(args.metrics) + write_json(args.summary, summary) + write_jsonl( + args.decisions, + ( + classify_metrics(metrics, threshold) + for metrics in iter_jsonl(args.metrics) + ), ) return 0 -def _add_stream_arguments(parser: argparse.ArgumentParser) -> None: - """Add arguments shared by streaming pipeline stages.""" +def command_sample(args: argparse.Namespace) -> int: + """Materialize a deterministic stratified review sample.""" - parser.add_argument("--input", type=Path, required=True, help="JSONL file or dataset directory") - parser.add_argument("--output", type=Path, required=True, help="Append-only output JSONL") - parser.add_argument("--errors", type=Path, help="Optional error JSONL") - parser.add_argument("--limit", type=int, help="Maximum number of new records") - parser.add_argument("--resume", action="store_true", help="Skip IDs already present in output") - parser.add_argument( - "--workers", - type=int, - default=1, - help="Concurrent record workers; API stages can use this for parallel requests", + selected, decisions = select_review_ids(args.decisions, args.per_group, args.seed) + write_jsonl( + args.output, + iter_review_records(args.input, selected, decisions), ) + print(f"sample: selected={len(selected)}", file=sys.stderr) + return 0 def build_parser() -> argparse.ArgumentParser: - """Construct the complete command-line parser.""" + """Build the intentionally small three-command interface.""" parser = argparse.ArgumentParser(prog="swe-qc", description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) - features = subparsers.add_parser("features", help="Extract deterministic static QC features") - _add_stream_arguments(features) - features.set_defaults(func=command_features) + profile = subparsers.add_parser("profile", help="Extract deterministic metrics") + profile.add_argument("--input", type=Path, required=True) + profile.add_argument("--output", type=Path, required=True) + profile.add_argument("--errors", type=Path) + profile.add_argument("--tokenizer-json", type=Path) + profile.add_argument("--limit", type=int) + profile.add_argument("--resume", action="store_true") + profile.add_argument("--workers", type=int, default=1) + profile.set_defaults(func=command_profile) - classify = subparsers.add_parser("classify", help="Classify trajectories through GLM-5.2") - _add_stream_arguments(classify) - classify.set_defaults(func=command_classify) - - audit = subparsers.add_parser( - "audit", help="Locate a safe boundary, then score only the retained prefix" + summarize = subparsers.add_parser( + "summarize", help="Build summary and deterministic decisions" ) - _add_stream_arguments(audit) - audit.set_defaults(func=command_audit) + summarize.add_argument("--metrics", type=Path, required=True) + summarize.add_argument("--summary", type=Path, required=True) + summarize.add_argument("--decisions", type=Path, required=True) + summarize.set_defaults(func=command_summarize) - repair_plan = subparsers.add_parser("repair-plan", help="Create allowlisted static repair plans") - _add_stream_arguments(repair_plan) - repair_plan.add_argument("--classifications", type=Path, required=True) - repair_plan.add_argument("--include-nonrepair", action="store_true") - repair_plan.set_defaults(func=command_repair_plan) - - apply_repair = subparsers.add_parser("apply-repair", help="Apply static plans deterministically") - _add_stream_arguments(apply_repair) - apply_repair.add_argument("--plans", type=Path, required=True) - apply_repair.set_defaults(func=command_apply_repair) - - review = subparsers.add_parser("review", help="Independently review repaired records") - _add_stream_arguments(review) - review.add_argument("--classifications", type=Path, required=True) - review.add_argument("--plans", type=Path, required=True) - review.add_argument("--repaired", type=Path, required=True) - review.set_defaults(func=command_review) - - smoke_test = subparsers.add_parser("smoke-test", help="Test API authentication and JSON output") - smoke_test.set_defaults(func=command_smoke_test) + sample = subparsers.add_parser( + "sample", help="Sample full trajectories by rule and length bucket" + ) + sample.add_argument("--input", type=Path, required=True) + sample.add_argument("--decisions", type=Path, required=True) + sample.add_argument("--output", type=Path, required=True) + sample.add_argument("--per-group", type=int, default=20) + sample.add_argument("--seed", type=int, default=20260818) + sample.set_defaults(func=command_sample) return parser def main(argv: list[str] | None = None) -> int: - """CLI entry point used by the ``swe-qc`` console script.""" + """Run the selected command.""" - parser = build_parser() - args = parser.parse_args(argv) + args = build_parser().parse_args(argv) return int(args.func(args)) diff --git a/src/swe_data_processing/client.py b/src/swe_data_processing/client.py deleted file mode 100644 index 876f380..0000000 --- a/src/swe_data_processing/client.py +++ /dev/null @@ -1,206 +0,0 @@ -"""HTTP client for GLM-5.2 structured JSON generation.""" - -from __future__ import annotations - -import json -import random -import re -import time -from dataclasses import dataclass -from typing import Any - -import httpx -from jsonschema import ValidationError, validate - -from .config import Settings - - -class GLMClientError(RuntimeError): - """Raised when a GLM request cannot produce a valid structured response.""" - - -class GLMAuthenticationError(GLMClientError): - """Raised for non-retryable authentication and authorization failures.""" - - -@dataclass(frozen=True) -class GLMResponse: - """A validated model response with non-sensitive request metadata.""" - - data: dict[str, Any] - request_id: str | None - usage: dict[str, Any] | None - compatibility_fallback_used: bool - - -def _strip_json_fence(text: str) -> str: - """Remove a single Markdown JSON fence without changing JSON contents.""" - - stripped = text.strip() - match = re.fullmatch(r"```(?:json)?\s*(.*?)\s*```", stripped, flags=re.DOTALL | re.IGNORECASE) - return match.group(1).strip() if match else stripped - - -def _extract_message_text(response_data: dict[str, Any]) -> str: - """Extract text from common OpenAI-compatible response layouts.""" - - try: - content = response_data["choices"][0]["message"]["content"] - except (KeyError, IndexError, TypeError) as exc: - # The fallback keys make the client tolerant of lightweight gateway - # adapters while keeping the primary path OpenAI-compatible. - for key in ("output_text", "text", "output"): - value = response_data.get(key) - if isinstance(value, str): - return value - raise GLMClientError("The API response did not contain assistant message content") from exc - - if isinstance(content, str): - return content - if isinstance(content, list): - text_parts = [] - for block in content: - if isinstance(block, dict) and isinstance(block.get("text"), str): - text_parts.append(block["text"]) - if text_parts: - return "".join(text_parts) - raise GLMClientError("The assistant message content was not textual") - - -class GLMClient: - """Call GLM-5.2 with retry, compatibility fallback, and schema validation.""" - - def __init__(self, settings: Settings, *, transport: httpx.BaseTransport | None = None) -> None: - self.settings = settings - self._client = httpx.Client( - timeout=httpx.Timeout(settings.timeout_seconds), - transport=transport, - headers={ - "Authorization": f"Bearer {settings.api_key}", - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": "swe-data-processing/2.0.0", - }, - ) - - def close(self) -> None: - """Close the underlying HTTP connection pool.""" - - self._client.close() - - def __enter__(self) -> GLMClient: - return self - - def __exit__(self, *_: object) -> None: - self.close() - - def invoke_json( - self, - *, - system_prompt: str, - payload: dict[str, Any], - schema: dict[str, Any], - ) -> GLMResponse: - """Request one JSON object and validate it against ``schema``. - - Invalid JSON and schema violations are retried because model output can - occasionally be malformed. ``max_retries`` counts retries after the - initial request. Authentication errors are never retried. - """ - - schema_text = json.dumps(schema, ensure_ascii=False, separators=(",", ":")) - full_system_prompt = ( - f"{system_prompt.rstrip()}\n\n" - "Return exactly one JSON object matching this JSON Schema:\n" - f"{schema_text}" - ) - compatibility_fallback = False - last_error: Exception | None = None - - total_attempts = self.settings.max_retries + 1 - for attempt in range(total_attempts): - request_payload = self._build_request( - full_system_prompt, - payload, - include_extensions=not compatibility_fallback, - ) - try: - response = self._client.post(self.settings.endpoint, json=request_payload) - request_id = response.headers.get("x-request-id") - - if response.status_code in {401, 403}: - raise GLMAuthenticationError( - f"Authentication failed with HTTP {response.status_code}; check GLM_API_KEY" - ) - if response.status_code == 400 and not compatibility_fallback: - # Some OpenAI-compatible gateways reject vendor-specific - # thinking parameters. Retry once with the portable subset. - compatibility_fallback = True - last_error = GLMClientError("The gateway rejected optional GLM parameters") - continue - if response.status_code == 429 or response.status_code >= 500: - raise httpx.HTTPStatusError( - f"Retryable API status {response.status_code}", - request=response.request, - response=response, - ) - response.raise_for_status() - - response_data = response.json() - raw_text = _extract_message_text(response_data) - parsed = json.loads(_strip_json_fence(raw_text)) - if not isinstance(parsed, dict): - raise GLMClientError("The model returned JSON that was not an object") - validate(instance=parsed, schema=schema) - usage = response_data.get("usage") - return GLMResponse( - data=parsed, - request_id=request_id, - usage=usage if isinstance(usage, dict) else None, - compatibility_fallback_used=compatibility_fallback, - ) - except GLMAuthenticationError: - raise - except (httpx.HTTPError, json.JSONDecodeError, ValidationError, GLMClientError) as exc: - last_error = exc - if attempt + 1 < total_attempts: - # Bounded exponential backoff avoids synchronized retries - # without making a single failed sample block indefinitely. - delay = min(20.0, 2.0**attempt) + random.uniform(0.0, 0.5) - time.sleep(delay) - - raise GLMClientError( - f"GLM failed after {total_attempts} attempts " - f"({self.settings.max_retries} retries): " - f"{type(last_error).__name__}: {last_error}" - ) - - def _build_request( - self, - system_prompt: str, - payload: dict[str, Any], - *, - include_extensions: bool, - ) -> dict[str, Any]: - """Build a portable chat-completions request body.""" - - request: dict[str, Any] = { - "model": self.settings.model, - "messages": [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": json.dumps(payload, ensure_ascii=False, separators=(",", ":")), - }, - ], - "temperature": self.settings.temperature, - "max_tokens": self.settings.max_tokens, - "response_format": {"type": "json_object"}, - "stream": False, - } - if include_extensions: - request["thinking"] = { - "type": "enabled" if self.settings.thinking_enabled else "disabled" - } - request["reasoning_effort"] = self.settings.reasoning_effort - return request diff --git a/src/swe_data_processing/config.py b/src/swe_data_processing/config.py deleted file mode 100644 index 00f4c36..0000000 --- a/src/swe_data_processing/config.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Runtime configuration for the GLM API and local processing pipeline.""" - -from __future__ import annotations - -import os -from dataclasses import dataclass - - -def _get_bool(name: str, default: bool) -> bool: - """Read a strict boolean environment variable with a safe default.""" - - raw_value = os.getenv(name) - if raw_value is None: - return default - normalized = raw_value.strip().lower() - if normalized in {"1", "true", "yes", "on"}: - return True - if normalized in {"0", "false", "no", "off"}: - return False - raise ValueError(f"{name} must be a boolean value, got {raw_value!r}") - - -def _get_int(name: str, default: int, minimum: int = 1) -> int: - """Read and validate an integer environment variable.""" - - value = int(os.getenv(name, str(default))) - if value < minimum: - raise ValueError(f"{name} must be at least {minimum}, got {value}") - return value - - -def _get_float(name: str, default: float, minimum: float = 0.0) -> float: - """Read and validate a floating-point environment variable.""" - - value = float(os.getenv(name, str(default))) - if value < minimum: - raise ValueError(f"{name} must be at least {minimum}, got {value}") - return value - - -@dataclass(frozen=True) -class Settings: - """Immutable settings used by API clients and command-line workflows.""" - - api_key: str - api_base: str = "https://llm-api.cowin.run" - api_path: str = "/v1/chat/completions" - model: str = "glm-5.2" - timeout_seconds: float = 300.0 - max_retries: int = 5 - max_tokens: int = 8192 - temperature: float = 0.0 - reasoning_effort: str = "high" - thinking_enabled: bool = True - - @property - def endpoint(self) -> str: - """Return the normalized absolute chat-completions URL.""" - - base = self.api_base.rstrip("/") - path = self.api_path if self.api_path.startswith("/") else f"/{self.api_path}" - return f"{base}{path}" - - @classmethod - def from_env(cls, *, require_api_key: bool = True) -> Settings: - """Construct settings from environment variables. - - The API key is intentionally loaded only from ``GLM_API_KEY``. The - project never reads a committed configuration file containing a key. - """ - - api_key = os.getenv("GLM_API_KEY", "").strip() - if require_api_key and not api_key: - raise ValueError("GLM_API_KEY is required but was not set") - - reasoning_effort = os.getenv("GLM_REASONING_EFFORT", "high").strip().lower() - if reasoning_effort not in {"low", "medium", "high", "max"}: - raise ValueError( - "GLM_REASONING_EFFORT must be one of: low, medium, high, max" - ) - - return cls( - api_key=api_key, - api_base=os.getenv("GLM_API_BASE", "https://llm-api.cowin.run"), - api_path=os.getenv("GLM_API_PATH", "/v1/chat/completions"), - model=os.getenv("GLM_MODEL", "glm-5.2"), - timeout_seconds=_get_float("GLM_TIMEOUT_SECONDS", 300.0, 1.0), - max_retries=_get_int("GLM_MAX_RETRIES", 5, 0), - max_tokens=_get_int("GLM_MAX_TOKENS", 8192, 1), - temperature=_get_float("GLM_TEMPERATURE", 0.0, 0.0), - reasoning_effort=reasoning_effort, - thinking_enabled=_get_bool("GLM_THINKING_ENABLED", True), - ) diff --git a/src/swe_data_processing/evidence.py b/src/swe_data_processing/evidence.py deleted file mode 100644 index 8155b82..0000000 --- a/src/swe_data_processing/evidence.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Prompt-only evidence compaction for long coding-agent trajectories. - -The functions in this module never modify the source dataset. They create a -bounded API representation while preserving turn identifiers, message roles, -commands, high-value verification evidence, and cryptographic hashes of every -shortened value. -""" - -from __future__ import annotations - -import hashlib -import json -from typing import Any - -BOUNDARY_BLOCK_TURNS = 32 - - -def _as_text(value: Any) -> str: - """Convert arbitrary JSON-compatible content into deterministic text.""" - - if isinstance(value, str): - return value - if value is None: - return "" - return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) - - -def compact_text(value: Any, max_chars: int) -> str: - """Return a head/tail preview with an auditable hash when text is long.""" - - text = _as_text(value) - if len(text) <= max_chars: - return text - digest = hashlib.sha256(text.encode("utf-8")).hexdigest() - marker = f"\n...[COMPACTED original_chars={len(text)} sha256={digest}]...\n" - available = max(2, max_chars - len(marker)) - head_chars = available // 3 - tail_chars = available - head_chars - return f"{text[:head_chars]}{marker}{text[-tail_chars:]}" - - -def compact_patch_object(value: Any, max_patch_chars: int = 30_000) -> dict[str, Any]: - """Compact only long string fields in a patch metadata object.""" - - if not isinstance(value, dict): - return {} - result: dict[str, Any] = {} - for key, item in value.items(): - if isinstance(item, str): - limit = max_patch_chars if key == "patch" else 4_000 - result[key] = compact_text(item, limit) - else: - result[key] = item - return result - - -def _compact_tool_call(call: Any, argument_limit: int) -> dict[str, Any]: - """Preserve tool identity and a bounded, hashed representation of arguments.""" - - if not isinstance(call, dict): - return {"malformed_call_preview": compact_text(call, argument_limit)} - function = call.get("function") - if not isinstance(function, dict): - return { - "id": call.get("id"), - "type": call.get("type"), - "malformed_function_preview": compact_text(function, argument_limit), - } - return { - "id": call.get("id"), - "type": call.get("type"), - "function": { - "name": function.get("name"), - "arguments": compact_text(function.get("arguments", "{}"), argument_limit), - }, - } - - -def compact_trajectory( - trajectory: list[dict[str, Any]], - static_signals: dict[str, Any], -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Build a bounded trajectory view that prioritizes decisive evidence. - - Test commands and observations, state-changing turns, malformed tool calls, - and the final eight turns receive larger previews. Other turns remain in the - prompt with their IDs and roles but use small previews, so ordering and - structural problems are still visible to the classifier. - """ - - important_turns = set(static_signals.get("stateful_turns") or []) - for event in static_signals.get("test_events") or []: - for key in ("command_turn", "output_turn"): - turn_id = event.get(key) - if isinstance(turn_id, int): - important_turns.add(turn_id) - for key in ("malformed_tool_turns", "unknown_tool_turns"): - for item in static_signals.get(key) or []: - turn_id = item.get("turn_id") if isinstance(item, dict) else None - if isinstance(turn_id, int): - important_turns.add(turn_id) - first_final_turn = max(1, len(trajectory) - 7) - important_turns.update(range(first_final_turn, len(trajectory) + 1)) - - compacted: list[dict[str, Any]] = [] - compacted_turns: list[int] = [] - for message in trajectory: - turn_id = int(message["turn_id"]) - role = message.get("role") - important = turn_id in important_turns - if role == "tool": - content_limit = 6_000 if important else 600 - elif role == "assistant": - content_limit = 4_000 if important else 800 - else: - content_limit = 2_000 if important else 800 - argument_limit = 6_000 if important else 1_500 - - original_content = _as_text(message.get("content")) - content = compact_text(original_content, content_limit) - if content != original_content: - compacted_turns.append(turn_id) - item: dict[str, Any] = { - "turn_id": turn_id, - "role": role, - "content": content, - } - if role == "assistant" and message.get("reasoning_content") is not None: - reasoning_limit = 3_000 if important else 700 - original_reasoning = _as_text(message.get("reasoning_content")) - item["reasoning_content"] = compact_text(original_reasoning, reasoning_limit) - if item["reasoning_content"] != original_reasoning: - compacted_turns.append(turn_id) - for key in ("name", "tool_call_id"): - if key in message: - item[key] = message.get(key) - if message.get("tool_calls") is not None: - item["tool_calls"] = [ - _compact_tool_call(call, argument_limit) - for call in (message.get("tool_calls") or []) - ] - compacted.append(item) - - metadata = { - "method": "turn-preserving-head-tail-v1", - "source_turn_count": len(trajectory), - "important_turn_ids": sorted(important_turns), - "content_compacted_turn_ids": compacted_turns, - "note": ( - "Compaction affects only the API prompt. Original dataset messages, " - "tool outputs, labels, and patches remain unchanged. Every shortened " - "value contains its original character count and SHA-256 hash." - ), - } - return compacted, metadata - - -def build_trajectory_blocks( - trajectory: list[dict[str, Any]], - static_signals: dict[str, Any], -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """Split a compact trajectory into stable blocks for boundary review. - - Blocks are only a prompt representation. They never change source turns, - and every message keeps its absolute one-based ``turn_id``. A fixed block - size keeps the architecture predictable without adding tuning parameters to - the command line. - """ - - compacted, metadata = compact_trajectory(trajectory, static_signals) - blocks = [] - offset = 0 - while offset < len(compacted): - end = min(offset + BOUNDARY_BLOCK_TURNS, len(compacted)) - # Keep the immediate observation with the assistant action that caused - # it. This may make a block one or two turns larger than the target. - while end < len(compacted) and compacted[end].get("role") == "tool": - end += 1 - messages = compacted[offset:end] - block_number = len(blocks) + 1 - blocks.append( - { - "block_id": f"block-{block_number:03d}", - "start_turn": messages[0]["turn_id"], - "end_turn": messages[-1]["turn_id"], - "messages": messages, - } - ) - offset = end - return blocks, { - **metadata, - "block_turns": BOUNDARY_BLOCK_TURNS, - "block_count": len(blocks), - } diff --git a/src/swe_data_processing/features.py b/src/swe_data_processing/features.py deleted file mode 100644 index c714273..0000000 --- a/src/swe_data_processing/features.py +++ /dev/null @@ -1,283 +0,0 @@ -"""Deterministic feature extraction from agent trajectories and patches.""" - -from __future__ import annotations - -import json -import re -from collections import Counter -from typing import Any - -TEST_COMMAND_RE = re.compile( - r"(?:^|[;&\s])(pytest|py\.test|cargo\s+(?:test|check)|go\s+(?:test|build)|" - r"npm\s+(?:run\s+)?test|yarn\s+test|pnpm\s+test|mvn\s+(?:test|verify)|" - r"gradle\w*\s+test|make\s+(?:test|check)|ctest|phpunit|rspec)(?:\s|$)", - re.IGNORECASE, -) -FAILURE_RE = re.compile( - r"traceback|compilation failed|build fail(?:ed|ure)|test result:\s*failed|" - r"(?:^|\s)[1-9][0-9]*\s+(?:failed|failures?)\b|^\s*fail(?:\s|$)|" - r"panic:|timed? out|timeout|" - r"command not found|permission denied|exit (?:code|status):?\s*[1-9]", - re.IGNORECASE | re.MULTILINE, -) -SUCCESS_RE = re.compile( - r"test result:\s*ok|\bbuild success\b|\btests? passed\b|" - r"\bpassed\b|exit (?:code|status):?\s*0", - re.IGNORECASE, -) -EXIT_CODE_RE = re.compile(r"exit (?:code|status):?\s*(-?\d+)", re.IGNORECASE) -STATEFUL_SHELL_RE = re.compile( - r"(?:^|[;&|]\s*)(?:sed\s+-i|perl\s+-pi|tee\s+|cp\s+|mv\s+|rm\s+|" - r"git\s+(?:apply|checkout|restore|reset|add)|patch\s+|mkdir\s+|touch\s+)|" - r"(?:^|\s)(?:>|>>)(?:\s|\S)", - re.IGNORECASE, -) -TEST_CONSTRAINT_RE = re.compile( - r"(?:do not|don't|must not|should not)\s+(?:modify|change|edit).*?test", - re.IGNORECASE | re.DOTALL, -) -VENDOR_CONSTRAINT_RE = re.compile( - r"(?:only|minimal changes? to)\s+(?:non-test|source|production)\s+files?", - re.IGNORECASE, -) - - -def _message_content(message: dict[str, Any]) -> str: - """Convert a message content field to stable text for pattern matching.""" - - content = message.get("content") - if isinstance(content, str): - return content - if content is None: - return "" - return json.dumps(content, ensure_ascii=False, default=str) - - -def _parse_tool_definitions(raw_tools: list[Any]) -> tuple[set[str], list[int]]: - """Return allowed tool names and indexes of malformed definitions.""" - - names: set[str] = set() - malformed: list[int] = [] - for index, raw_tool in enumerate(raw_tools): - try: - tool = json.loads(raw_tool) if isinstance(raw_tool, str) else raw_tool - if not isinstance(tool, dict): - raise TypeError("tool definition is not an object") - function = tool.get("function") or tool - name = function.get("name") if isinstance(function, dict) else None - if isinstance(name, str) and name: - names.add(name) - else: - raise ValueError("tool definition has no name") - except (json.JSONDecodeError, TypeError, ValueError): - malformed.append(index) - return names, malformed - - -def _tool_call_details(call: dict[str, Any]) -> tuple[str, dict[str, Any] | None, str | None]: - """Parse a tool call into name, arguments, and an optional parse error.""" - - function = call.get("function") or {} - if not isinstance(function, dict): - return "", None, "function is not an object" - name = function.get("name") - raw_arguments = function.get("arguments", "{}") - if not isinstance(name, str): - name = "" - try: - arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments - if not isinstance(arguments, dict): - return name, None, "arguments do not decode to an object" - return name, arguments, None - except json.JSONDecodeError as exc: - return name, None, f"{type(exc).__name__}: {exc}" - - -def _command_from_arguments(arguments: dict[str, Any] | None) -> str: - """Extract the command-like value used by common dataset tools.""" - - if not arguments: - return "" - for key in ("command", "cmd"): - value = arguments.get(key) - if isinstance(value, str): - return value - return "" - - -def _classify_test_level(command: str) -> str: - """Estimate test scope conservatively from a shell command.""" - - lowered = command.lower() - if "cargo check" in lowered or "go build" in lowered: - return "BUILD" - full_suite_tokens = ( - "go test ./...", - "npm test", - "cargo test", - "pytest tests", - "mvn test", - ) - if any(token in lowered for token in full_suite_tokens): - return "FULL" - if re.search(r"pytest\s+[^|;&]+::|go test\s+\S+/\.\.\.|cargo test\s+[-\w]*\s+\S+", lowered): - return "TARGETED" - return "MODULE" - - -def _extract_exit_codes(output: str) -> list[int]: - """Extract all explicit exit codes from a tool observation.""" - - return [int(value) for value in EXIT_CODE_RE.findall(output)] - - -def extract_patch_files(patch: str | None) -> list[str]: - """Extract normalized destination file paths from a unified diff.""" - - files: list[str] = [] - for match in re.finditer(r"^\+\+\+\s+(?:b/)?(.+)$", patch or "", flags=re.MULTILINE): - path = match.group(1).strip() - if path != "/dev/null" and path not in files: - files.append(path) - return files - - -def index_trajectory(record: dict[str, Any]) -> list[dict[str, Any]]: - """Return a trajectory copy with deterministic one-based turn identifiers.""" - - indexed = [] - for turn_id, message in enumerate(record.get("trajectory") or [], 1): - if isinstance(message, dict): - item = dict(message) - else: - item = {"role": "invalid", "content": str(message)} - item["turn_id"] = turn_id - indexed.append(item) - return indexed - - -def extract_instruction_constraints(trajectory: list[dict[str, Any]]) -> list[str]: - """Extract explicit file-modification constraints from user turns.""" - - user_text = "\n".join( - _message_content(message) - for message in trajectory - if message.get("role") == "user" - ) - constraints = [] - if TEST_CONSTRAINT_RE.search(user_text): - constraints.append("Do not modify test files.") - if VENDOR_CONSTRAINT_RE.search(user_text): - constraints.append("Limit changes to production/source files.") - return constraints - - -def extract_static_signals(record: dict[str, Any]) -> dict[str, Any]: - """Build deterministic QC signals without executing repository code.""" - - trajectory = index_trajectory(record) - allowed_tools, malformed_tool_definitions = _parse_tool_definitions(record.get("tools") or []) - malformed_calls: list[dict[str, Any]] = [] - unknown_calls: list[dict[str, Any]] = [] - tool_calls: list[dict[str, Any]] = [] - test_events: list[dict[str, Any]] = [] - stateful_turns: list[int] = [] - editor_paths: list[str] = [] - role_counts = Counter(message.get("role", "invalid") for message in trajectory) - - for index, message in enumerate(trajectory): - if message.get("role") != "assistant": - continue - for raw_call in message.get("tool_calls") or []: - if not isinstance(raw_call, dict): - malformed_calls.append({"turn_id": index + 1, "error": "tool call is not an object"}) - continue - name, arguments, parse_error = _tool_call_details(raw_call) - command = _command_from_arguments(arguments) - call_item = { - "turn_id": index + 1, - "tool_name": name, - "tool_call_id": raw_call.get("id"), - "command": command, - } - tool_calls.append(call_item) - if parse_error: - malformed_calls.append({"turn_id": index + 1, "tool_name": name, "error": parse_error}) - if allowed_tools and name not in allowed_tools: - unknown_calls.append({"turn_id": index + 1, "tool_name": name}) - - editor_command = arguments.get("command") if arguments and name == "str_replace_editor" else None - editor_path = arguments.get("path") if arguments else None - if name == "str_replace_editor" and editor_command in {"create", "str_replace", "insert"}: - stateful_turns.append(index + 1) - if isinstance(editor_path, str) and editor_path not in editor_paths: - editor_paths.append(editor_path) - if command and STATEFUL_SHELL_RE.search(command): - stateful_turns.append(index + 1) - - if command and TEST_COMMAND_RE.search(command): - output = "" - output_turn = None - if index + 1 < len(trajectory) and trajectory[index + 1].get("role") == "tool": - output = _message_content(trajectory[index + 1]) - output_turn = index + 2 - explicit_codes = _extract_exit_codes(output) - masked_pipeline = "|" in command and "pipefail" not in command - explicit_failure = bool(FAILURE_RE.search(output)) or any( - code != 0 for code in explicit_codes - ) - explicit_success = bool(SUCCESS_RE.search(output)) and not explicit_failure - test_events.append( - { - "command_turn": index + 1, - "output_turn": output_turn, - "command": command, - "level": _classify_test_level(command), - "masked_pipeline": masked_pipeline, - "explicit_exit_codes": explicit_codes, - "explicit_failure": explicit_failure, - "explicit_success": explicit_success, - "output_preview": output[-1500:], - } - ) - - # The dataset alternates assistant calls and tool observations. The check - # remains conservative because some valid messages can contain no tool call. - alternation_issues = [] - for index in range(2, len(trajectory)): - expected = "assistant" if index % 2 == 0 else "tool" - actual = trajectory[index].get("role") - if actual != expected: - alternation_issues.append({"turn_id": index + 1, "expected": expected, "actual": actual}) - - metadata = record.get("metadata") or {} - model_patch = metadata.get("model_patch") or {} - reference_patch = metadata.get("reference_patch") or {} - model_patch_text = model_patch.get("patch") or "" - reference_patch_text = reference_patch.get("patch") or "" - model_files = extract_patch_files(model_patch_text) - reference_files = extract_patch_files(reference_patch_text) - - return { - "allowed_tool_names": sorted(allowed_tools), - "malformed_tool_definition_indexes": malformed_tool_definitions, - "malformed_tool_turns": malformed_calls, - "unknown_tool_turns": unknown_calls, - "alternation_issues": alternation_issues, - "role_counts": dict(role_counts), - "tool_call_count": len(tool_calls), - "stateful_turns": sorted(set(stateful_turns)), - "last_stateful_turn": max(stateful_turns, default=None), - "editor_paths": editor_paths, - "test_events": test_events, - "model_patch_files": model_files, - "reference_patch_files": reference_files, - "model_patch_chars": len(model_patch_text), - "reference_patch_chars": len(reference_patch_text), - "patch_size_ratio_to_reference": ( - round(len(model_patch_text) / len(reference_patch_text), 4) - if reference_patch_text - else None - ), - "instruction_constraints": extract_instruction_constraints(trajectory), - } diff --git a/src/swe_data_processing/heuristics.py b/src/swe_data_processing/heuristics.py new file mode 100644 index 0000000..90ba587 --- /dev/null +++ b/src/swe_data_processing/heuristics.py @@ -0,0 +1,121 @@ +"""Transparent heuristics built only from deterministic trajectory metrics.""" + +from __future__ import annotations + +from typing import Any + +LENGTH_LIMITS = (81_920, 131_072, 262_144) + + +def length_bucket(token_count: int | None) -> str: + """Assign a training-oriented context bucket without changing the trace.""" + + if token_count is None: + return "TOKENIZER_REQUIRED" + if token_count <= LENGTH_LIMITS[0]: + return "LE_81920" + if token_count <= LENGTH_LIMITS[1]: + return "81921_TO_131072" + if token_count <= LENGTH_LIMITS[2]: + return "131073_TO_262144" + return "GT_262144" + + +def classify_metrics( + metrics: dict[str, Any], high_error_count_threshold: int +) -> dict[str, Any]: + """Classify one profile using high-precision, inspectable rules. + + Hard rejection is reserved for broken tool structure or persistent tool + failure patterns. Relative outliers and early clusters are review signals, + because high error count alone can be explained by a long trajectory. + """ + + tools = metrics["tools"] + structure = metrics["structure"] + positions = tools["error_positions"] + failed = int(tools["failed_tool_call_count"]) + rate = float(tools["failed_tool_call_rate"]) + hard_reasons: list[str] = [] + review_flags: list[str] = [] + + broken_structure = sum( + int(structure[key]) + for key in ( + "invalid_turn_count", + "malformed_tool_definition_count", + "malformed_tool_call_count", + "unknown_tool_call_count", + "missing_tool_result_count", + "orphan_tool_result_count", + ) + ) + if broken_structure: + hard_reasons.append("BROKEN_TOOL_STRUCTURE") + if int(tools["longest_consecutive_failure_run"]) >= 5: + hard_reasons.append("FIVE_CONSECUTIVE_TOOL_FAILURES") + if failed >= 5 and rate >= 0.5: + hard_reasons.append("HIGH_TOOL_FAILURE_RATE") + if failed >= high_error_count_threshold: + review_flags.append("EXTREME_ERROR_COUNT") + if failed >= 8 and rate >= 0.2 and int(positions["occupied_bins_5"]) >= 4: + review_flags.append("PERSISTENT_DISTRIBUTED_FAILURES") + if ( + int(positions["early_count"]) >= 3 + and float(positions["early_fraction"]) >= 0.6 + ): + review_flags.append("EARLY_FAILURE_CLUSTER") + + bucket = length_bucket(metrics["length"]["token_count"]) + resolved = metrics.get("resolved") + if resolved == 1: + outcome_use = "SUCCESS_SFT_CANDIDATE" + elif resolved == 0: + outcome_use = "EXCLUDE_FROM_SUCCESS_SFT" + else: + outcome_use = "UNVERIFIED" + + if hard_reasons: + action = "DROP_DEFINITE_TOOL_PROBLEM" + elif resolved == 0: + action = "EXCLUDE_FAILED_OUTCOME" + elif resolved != 1: + action = "HOLD_UNVERIFIED_OUTCOME" + elif bucket == "GT_262144": + action = "DROP_OVER_262144" + elif bucket == "131073_TO_262144": + action = "DEFER_LONG_CONTEXT" + elif review_flags: + action = "REVIEW_HEURISTIC_HIT" + elif bucket == "LE_81920": + action = "TRAIN_81920_BUCKET" + elif bucket == "81921_TO_131072": + action = "TRAIN_131072_BUCKET" + else: + action = "TOKENIZER_REQUIRED" + + return { + "sample_id": metrics["sample_id"], + "instance_id": metrics.get("instance_id"), + "source_group": metrics.get("source_group"), + "source_parquet": metrics.get("source_parquet"), + "resolved": resolved, + "length_bucket": bucket, + "outcome_use": outcome_use, + "hard_reject_reasons": hard_reasons, + "review_flags": review_flags, + "recommended_action": action, + "metrics": { + "turn_count": metrics["length"]["turn_count"], + "token_count": metrics["length"]["token_count"], + "tool_call_count": tools["tool_call_count"], + "failed_tool_call_count": failed, + "failed_tool_call_rate": rate, + "longest_consecutive_failure_run": tools[ + "longest_consecutive_failure_run" + ], + "error_positions": positions, + "error_type_counts": tools["error_type_counts"], + "error_tool_counts": tools["error_tool_counts"], + }, + } diff --git a/src/swe_data_processing/io.py b/src/swe_data_processing/io.py index 5f8c04c..8f4bd50 100644 --- a/src/swe_data_processing/io.py +++ b/src/swe_data_processing/io.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os from collections.abc import Iterator from pathlib import Path from typing import Any @@ -38,15 +39,17 @@ def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: def iter_parquet_dataset(dataset_dir: Path, *, batch_size: int = 16) -> Iterator[dict[str, Any]]: """Stream records from every Open-SWE-Traces Parquet shard in stable order.""" - files = sorted(dataset_dir.glob("data/**/*.parquet")) + files = sorted(dataset_dir.rglob("*.parquet")) if not files: - raise FileNotFoundError(f"No Parquet shards found under {dataset_dir / 'data'}") + raise FileNotFoundError(f"No Parquet shards found under {dataset_dir}") for path in files: parquet_file = pq.ParquetFile(path) for batch in parquet_file.iter_batches(batch_size=batch_size): for record in batch.to_pylist(): if isinstance(record, dict): - yield record + value = dict(record) + value["_source_parquet"] = str(path.resolve()) + yield value def iter_records(path: Path) -> Iterator[dict[str, Any]]: @@ -69,6 +72,29 @@ def append_jsonl(path: Path, value: dict[str, Any]) -> None: handle.flush() +def write_json(path: Path, value: dict[str, Any]) -> None: + """Atomically replace a JSON report after successful serialization.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n", + encoding="utf-8", + ) + os.replace(temporary, path) + + +def write_jsonl(path: Path, values: Iterator[dict[str, Any]]) -> None: + """Atomically replace a JSONL file from an iterator of objects.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as handle: + for value in values: + handle.write(json.dumps(value, ensure_ascii=False, default=str) + "\n") + os.replace(temporary, path) + + def load_completed_ids(path: Path) -> set[str]: """Load sample IDs already present in an append-only output manifest.""" diff --git a/src/swe_data_processing/metrics.py b/src/swe_data_processing/metrics.py new file mode 100644 index 0000000..33994d6 --- /dev/null +++ b/src/swe_data_processing/metrics.py @@ -0,0 +1,362 @@ +"""Programmatic metrics for Open-SWE-Traces trajectories.""" + +from __future__ import annotations + +import json +import math +import re +from collections import Counter +from pathlib import Path +from typing import Any, Protocol + +from .io import get_sample_id + +EXIT_CODE_RE = re.compile( + r"(?:exit(?:ed)?(?:\s+with)?(?:\s+code|\s+status)?|return\s+code)\s*[:=]?\s*(-?\d+)", + re.IGNORECASE, +) +TIMEOUT_RE = re.compile(r"\b(?:timed?\s*out|timeout)\b", re.IGNORECASE) +PERMISSION_RE = re.compile(r"\bpermission denied\b|\boperation not permitted\b", re.IGNORECASE) +NOT_FOUND_RE = re.compile( + r"\bcommand not found\b|\bno such file or directory\b|\bfile not found\b", + re.IGNORECASE, +) +TEST_FAILURE_RE = re.compile( + r"(?:^|\s)(?:[1-9]\d*)\s+(?:failed|failures?)\b|" + r"\bfailed tests?\b|\btest result:\s*failed\b|^FAIL(?:ED)?\b", + re.IGNORECASE | re.MULTILINE, +) +BUILD_FAILURE_RE = re.compile( + r"\b(?:build|compilation) fail(?:ed|ure)\b|\bfatal error:\b", + re.IGNORECASE, +) +EXCEPTION_RE = re.compile( + r"^\s*(?:traceback \(most recent call last\):|(?:error|exception|panic):)", + re.IGNORECASE | re.MULTILINE, +) +TOOL_REPORTED_ERROR_RE = re.compile( + r"^\s*(?:error:|toolusageerror:)|\bno replacement was performed\b", + re.IGNORECASE, +) +TEST_COMMAND_RE = re.compile( + r"(?:^|[;&|\s])(?:pytest|py\.test|cargo\s+(?:test|check)|go\s+(?:test|build)|" + r"npm\s+(?:run\s+)?test|yarn\s+test|pnpm\s+test|mvn\s+(?:test|verify)|" + r"gradle\w*\s+test|make\s+(?:test|check)|ctest|phpunit|rspec)(?:\s|$)", + re.IGNORECASE, +) + + +class TextTokenCounter(Protocol): + """Minimal interface accepted by :func:`extract_metrics`.""" + + path: Any + sha256: str + + def count(self, text: str) -> int: ... + + +def _content_text(message: dict[str, Any]) -> str: + """Return message content as stable text for high-precision matching.""" + + content = message.get("content") + if isinstance(content, str): + return content + if content is None: + return "" + return json.dumps(content, ensure_ascii=False, sort_keys=True, default=str) + + +def canonical_training_text(record: dict[str, Any]) -> str: + """Serialize only tools and trajectory into one reproducible token stream. + + The format is intentionally simple JSON rather than an assumed chat + template. Training code with a different serializer should rerun profiling + with that serializer instead of treating these counts as universal. + """ + + payload = { + "tools": record.get("tools") or [], + "trajectory": record.get("trajectory") or [], + } + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + +def _parse_tool_names(raw_tools: list[Any]) -> tuple[set[str], int]: + """Return declared tool names and the number of malformed definitions.""" + + names: set[str] = set() + malformed = 0 + for raw_tool in raw_tools: + try: + tool = json.loads(raw_tool) if isinstance(raw_tool, str) else raw_tool + if not isinstance(tool, dict): + raise TypeError + function = tool.get("function") or tool + name = function.get("name") if isinstance(function, dict) else None + if not isinstance(name, str) or not name: + raise ValueError + names.add(name) + except (json.JSONDecodeError, TypeError, ValueError): + malformed += 1 + return names, malformed + + +def _parse_call(raw_call: Any) -> tuple[str, dict[str, Any] | None, str | None]: + """Parse one tool call without guessing malformed arguments.""" + + if not isinstance(raw_call, dict): + return "", None, "tool_call_not_object" + function = raw_call.get("function") + if not isinstance(function, dict): + return "", None, "function_not_object" + name = function.get("name") + if not isinstance(name, str) or not name: + return "", None, "missing_tool_name" + raw_arguments = function.get("arguments", "{}") + try: + arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments + except json.JSONDecodeError: + return name, None, "invalid_arguments_json" + if not isinstance(arguments, dict): + return name, None, "arguments_not_object" + return name, arguments, None + + +def _command(arguments: dict[str, Any] | None) -> str: + """Extract command text from common shell-style tool arguments.""" + + if not arguments: + return "" + for key in ("command", "cmd", "script"): + value = arguments.get(key) + if isinstance(value, str): + return value + return "" + + +def _is_shell_tool(tool_name: str) -> bool: + """Return whether a tool executes shell-like commands.""" + + lowered = tool_name.lower() + return any(token in lowered for token in ("bash", "shell", "terminal")) + + +def _is_terminal_tool(tool_name: str) -> bool: + """Return whether a no-result call is a normal terminal action.""" + + return tool_name.lower() in {"finish", "submit", "final", "done"} + + +def _runtime_error_types(output: str, command: str, tool_name: str) -> list[str]: + """Detect observable tool failures with conservative text rules.""" + + if not _is_shell_tool(tool_name): + return ["tool_reported_error"] if TOOL_REPORTED_ERROR_RE.search(output) else [] + + codes = [int(value) for value in EXIT_CODE_RE.findall(output)] + nonzero = any(code != 0 for code in codes) + explicitly_successful = bool(codes) and not nonzero + types: list[str] = [] + + if nonzero: + types.append("nonzero_exit") + if TIMEOUT_RE.search(output) and not explicitly_successful: + types.append("timeout") + if PERMISSION_RE.search(output) and not explicitly_successful: + types.append("permission_denied") + if NOT_FOUND_RE.search(output) and not explicitly_successful: + types.append("not_found") + if TEST_COMMAND_RE.search(command) and TEST_FAILURE_RE.search(output): + types.append("test_failure") + if BUILD_FAILURE_RE.search(output) and not explicitly_successful: + types.append("build_failure") + if EXCEPTION_RE.search(output) and not explicitly_successful: + types.append("tool_exception") + return list(dict.fromkeys(types)) + + +def _longest_true_run(values: list[bool]) -> int: + """Return the longest consecutive run of failed tool calls.""" + + longest = 0 + current = 0 + for value in values: + current = current + 1 if value else 0 + longest = max(longest, current) + return longest + + +def _position_summary(error_ordinals: list[int], tool_call_count: int) -> dict[str, Any]: + """Describe where errors occur in a trajectory using five equal bins.""" + + if not error_ordinals or tool_call_count == 0: + return { + "mean": None, + "span": None, + "bins_5": [0, 0, 0, 0, 0], + "occupied_bins_5": 0, + "early_count": 0, + "early_fraction": 0.0, + } + + denominator = max(tool_call_count - 1, 1) + positions = [(ordinal - 1) / denominator for ordinal in error_ordinals] + bins = [0, 0, 0, 0, 0] + for position in positions: + bins[min(4, int(position * 5))] += 1 + early_limit = max(1, math.ceil(tool_call_count * 0.2)) + early_count = sum(ordinal <= early_limit for ordinal in error_ordinals) + return { + "mean": round(sum(positions) / len(positions), 6), + "span": round(max(positions) - min(positions), 6), + "bins_5": bins, + "occupied_bins_5": sum(value > 0 for value in bins), + "early_count": early_count, + "early_fraction": round(early_count / len(error_ordinals), 6), + } + + +def extract_metrics( + record: dict[str, Any], token_counter: TextTokenCounter | None = None +) -> dict[str, Any]: + """Extract deterministic length, structure, and tool-failure metrics.""" + + trajectory = record.get("trajectory") or [] + if not isinstance(trajectory, list): + trajectory = [] + canonical_text = canonical_training_text(record) + allowed_tools, malformed_definitions = _parse_tool_names(record.get("tools") or []) + role_counts: Counter[str] = Counter() + tool_counts: Counter[str] = Counter() + error_tools: Counter[str] = Counter() + error_types: Counter[str] = Counter() + error_events: list[dict[str, Any]] = [] + call_failed: list[bool] = [] + error_ordinals: list[int] = [] + invalid_turns = 0 + malformed_calls = 0 + unknown_calls = 0 + missing_results = 0 + orphan_results = 0 + + for turn_index, raw_message in enumerate(trajectory): + if not isinstance(raw_message, dict): + invalid_turns += 1 + role_counts["invalid"] += 1 + continue + role = raw_message.get("role") + role_counts[str(role or "missing")] += 1 + if role == "tool": + previous = trajectory[turn_index - 1] if turn_index else None + previous_calls = previous.get("tool_calls") if isinstance(previous, dict) else None + if not previous_calls: + orphan_results += 1 + continue + if role != "assistant": + continue + + raw_calls = raw_message.get("tool_calls") or [] + if not isinstance(raw_calls, list): + raw_calls = [raw_calls] + next_message = trajectory[turn_index + 1] if turn_index + 1 < len(trajectory) else None + has_result = isinstance(next_message, dict) and next_message.get("role") == "tool" + output = _content_text(next_message) if has_result else "" + + for call_index, raw_call in enumerate(raw_calls): + ordinal = len(call_failed) + 1 + name, arguments, parse_error = _parse_call(raw_call) + display_name = name or "" + tool_counts[display_name] += 1 + structural_types: list[str] = [] + if parse_error: + malformed_calls += 1 + structural_types.append(parse_error) + if allowed_tools and name and name not in allowed_tools: + unknown_calls += 1 + structural_types.append("unknown_tool") + if not has_result and not _is_terminal_tool(name): + missing_results += 1 + structural_types.append("missing_tool_result") + + runtime_types = ( + _runtime_error_types(output, _command(arguments), name) if has_result else [] + ) + types = list(dict.fromkeys(structural_types + runtime_types)) + failed = bool(types) + call_failed.append(failed) + if not failed: + continue + error_ordinals.append(ordinal) + error_tools[display_name] += 1 + error_types.update(types) + error_events.append( + { + "turn_id": turn_index + 1, + "tool_call_index": call_index, + "tool_call_ordinal": ordinal, + "tool_name": display_name, + "error_types": types, + } + ) + + tool_call_count = len(call_failed) + failed_count = sum(call_failed) + tokenizer = None + token_count = None + if token_counter is not None: + token_count = token_counter.count(canonical_text) + tokenizer = { + "path": str(token_counter.path), + "sha256": token_counter.sha256, + "serialization": "canonical-tools-and-trajectory-json-v1", + } + + return { + "sample_id": get_sample_id(record), + "instance_id": record.get("instance_id"), + "repo": record.get("repo"), + "source_dataset": record.get("hf_dataset_name"), + "source_group": ( + Path(record["_source_parquet"]).parent.name + if record.get("_source_parquet") + else None + ), + "source_parquet": record.get("_source_parquet"), + "resolved": record.get("resolved"), + "length": { + "turn_count": len(trajectory), + "role_counts": dict(sorted(role_counts.items())), + "canonical_chars": len(canonical_text), + "canonical_utf8_bytes": len(canonical_text.encode("utf-8")), + "token_count": token_count, + "tokenizer": tokenizer, + }, + "tools": { + "declared_tool_count": len(allowed_tools), + "tool_call_count": tool_call_count, + "tool_call_counts": dict(tool_counts.most_common()), + "failed_tool_call_count": failed_count, + "failed_tool_call_rate": round(failed_count / tool_call_count, 6) + if tool_call_count + else 0.0, + "longest_consecutive_failure_run": _longest_true_run(call_failed), + "error_tool_counts": dict(error_tools.most_common()), + "error_type_counts": dict(error_types.most_common()), + "error_positions": _position_summary(error_ordinals, tool_call_count), + "error_events": error_events, + }, + "structure": { + "invalid_turn_count": invalid_turns, + "malformed_tool_definition_count": malformed_definitions, + "malformed_tool_call_count": malformed_calls, + "unknown_tool_call_count": unknown_calls, + "missing_tool_result_count": missing_results, + "orphan_tool_result_count": orphan_results, + }, + } diff --git a/src/swe_data_processing/policy.py b/src/swe_data_processing/policy.py deleted file mode 100644 index 82f3d0f..0000000 --- a/src/swe_data_processing/policy.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Deterministic policy guards applied after every GLM classification.""" - -from __future__ import annotations - -from typing import Any - - -class PolicyViolation(ValueError): - """Raised when model output violates a non-negotiable QC invariant.""" - - -def enforce_classification_policy(record: dict[str, Any], result: dict[str, Any]) -> None: - """Reject classifications that illegally upgrade evidence or outcomes.""" - - resolved = int(record.get("resolved", -1)) - decision = result["qc_decision"] - training_use = result["training_use"] - qc_passed = result["qc_passed"] - - expected_source_class = { - 1: "POSITIVE_CANDIDATE", - 0: "EXPLICIT_NEGATIVE", - -1: "UNVERIFIED", - }.get(resolved) - if expected_source_class is None: - raise PolicyViolation(f"Unsupported resolved value: {resolved}") - if result["source_outcome_class"] != expected_source_class: - raise PolicyViolation("source_outcome_class does not match the immutable resolved label") - - if resolved == 0 and (decision == "ACCEPT_SILVER_POSITIVE" or training_use == "SFT_FULL"): - raise PolicyViolation("resolved=0 cannot be promoted to a successful full SFT sample") - if resolved == -1 and training_use == "SFT_FULL": - raise PolicyViolation("resolved=-1 cannot be promoted to SFT_FULL without execution") - if qc_passed != (decision == "ACCEPT_SILVER_POSITIVE"): - raise PolicyViolation("qc_passed must be true only for ACCEPT_SILVER_POSITIVE") - - if decision == "ACCEPT_SILVER_POSITIVE": - if resolved != 1: - raise PolicyViolation("Only resolved=1 can be accepted as a silver positive") - if result["hard_fail_codes"]: - raise PolicyViolation("A silver positive cannot contain a hard-fail code") - if result["verification"]["status"] != "PASS_RELIABLE": - raise PolicyViolation("A silver positive requires reliable passing verification") - if any(value != "PASS" for value in result["dimensions"].values()): - raise PolicyViolation("Every QC dimension must pass for a silver positive") - - -def enforce_repair_policy(result: dict[str, Any]) -> None: - """Ensure a repair plan explicitly preserves all immutable evidence.""" - - invariants = result["invariants"] - if not all(invariants.values()): - raise PolicyViolation("Every static-repair invariant must be true") - if not result["requires_second_review"]: - raise PolicyViolation("Every repair plan requires independent second review") - - creates_step_example = result["repair_decision"] == "CREATE_STEP_EXAMPLE" - if creates_step_example and result["maximum_training_use"] != "SFT_STEP_ONLY": - raise PolicyViolation("A truncated or corrected step example cannot exceed SFT_STEP_ONLY") - - decision = result["repair_decision"] - operations = result["operations"] - if decision in {"NO_CHANGE", "REQUIRES_EXECUTION", "DROP"} and operations: - raise PolicyViolation(f"{decision} must not contain mutation operations") - if decision in {"APPLY_STATIC_REPAIR", "CREATE_STEP_EXAMPLE"} and not operations: - raise PolicyViolation(f"{decision} requires at least one operation") - if result["maximum_training_use"] == "DPO_REJECTED" and decision == "APPLY_STATIC_REPAIR": - raise PolicyViolation("Authentic DPO rejected trajectories must not be rewritten") - - operation_names = [operation["op"] for operation in operations] - if creates_step_example: - if "EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN" not in operation_names: - raise PolicyViolation("A step example requires a corrected assistant turn") - if "TRUNCATE_BEFORE_TURN" not in operation_names: - raise PolicyViolation("A step example requires truncation after the corrected turn") - - for operation in operations: - op = operation["op"] - replacement = operation["replacement"] - if op in {"REMOVE_DUPLICATE_PROSE", "REWRITE_FINAL_SUMMARY", "REDACT_SECRET"}: - if not isinstance(replacement, str): - raise PolicyViolation(f"{op} requires a string replacement") - if op in { - "CANONICALIZE_JSON", - "NORMALIZE_TOOL_ALIAS", - "EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN", - }: - if not isinstance(replacement, dict) or replacement.get("role") != "assistant": - raise PolicyViolation(f"{op} requires a complete assistant-message object") diff --git a/src/swe_data_processing/prompts/boundary.md b/src/swe_data_processing/prompts/boundary.md deleted file mode 100644 index 7611d39..0000000 --- a/src/swe_data_processing/prompts/boundary.md +++ /dev/null @@ -1,55 +0,0 @@ -You are finding a safe truncation point in a coding-agent trajectory. You cannot run code. - -The trajectory is split into ordered blocks. Turn IDs are absolute. Find the earliest assistant turn B that begins -a MAJOR or CRITICAL problem which is never repaired. The saved training prefix will contain only turns before B. - -Before deciding, check the whole trajectory against the actual user request: - -- Did the final implementation satisfy every requested behavior and exact interface, not just a small subtask? -- Does the last visible diff/status leave tests, generated files, dependency files, summaries, or unrelated edits? -- Did the agent violate a literal scope constraint such as making only non-test changes? -- Do the last build/test/tool observations contradict the final claims or show an unrepaired broken repository? - -`final_model_patch` is the actual final diff left by this trajectory, not a reference answer. Use it to verify task -coverage and final scope. Test files, dependency files, generated files, and unrelated changes in that diff are real -final state even when the assistant says they were cleaned up. - -Passing existing tests is useful evidence, but it does not prove that a missing requirement or wrong public API is -correct. Re-read the request before choosing KEEP_FULL. - -Fill all four checks before the decision. KEEP_FULL is allowed only when they are COMPLETE, CLEAN, RESPECTED, and -CONSISTENT. If a check fails, locate the earliest causal assistant turn for TRUNCATE; use HOLD when no safe B is clear. - -Choose TRUNCATE only when both are true: - -1. Turn B clearly starts a serious problem, such as an unrepaired harmful edit, a persistent wrong implementation, - a literal user-constraint violation, a broken tool call, or an unsupported final claim. -2. Everything before B is still safe process training data. It must not contain an earlier unrepaired harmful state. - -Return KEEP_FULL when the whole trajectory has no unrepaired MAJOR or CRITICAL problem. It may still contain normal -failed experiments or minor inefficiency. Return HOLD only when the evidence is incomplete or contradictory enough -that you cannot establish either a safe full trajectory or a safe prefix. - -Never choose a command that merely reveals a problem as B. Choose the earlier assistant edit or decision that caused -the bad state. If you can see a major final defect but cannot identify a safe causal assistant turn, return HOLD. - -Keep normal debugging. A failed command, a plausible experiment, or temporarily broken code is not a boundary when -the agent later diagnoses, repairs, and verifies it. Do not punish harmless inefficiency. - -Find the cause, not the last symptom. If a bad patch remains and the final answer falsely claims success, choose the -assistant turn that created the bad patch. Do not merely remove the final answer while retaining failed code. - -Read user constraints literally. Do not invent stronger restrictions. Reading tests, running tests, or creating a -requested reproduction script is not the same as modifying repository tests. - -Evidence turn IDs must include assistant turn B. Supporting turn IDs may point to later tool results. Python will attach -exact source excerpts and derive the containing block, so do not reproduce quotes or block IDs. - -Positive example: turn 40 writes code that calls a nonexistent method, turn 46 shows the resulting failure, and the -edit is never repaired. Choose turn 40. - -Recovery example: turn 40 tries a wrong edit, turn 44 diagnoses it, turn 48 reverts it, and turn 52 passes the relevant -tests. Do not choose turn 40. - -Late-claim example: tests fail after a patch and the final turn says "all tests pass." If the patch is visibly still -bad, choose the earlier patch turn. Choose the final claim only when the retained implementation is otherwise safe. diff --git a/src/swe_data_processing/prompts/classification.md b/src/swe_data_processing/prompts/classification.md deleted file mode 100644 index c760a81..0000000 --- a/src/swe_data_processing/prompts/classification.md +++ /dev/null @@ -1,37 +0,0 @@ -You are a conservative static quality auditor for Open-SWE-Traces. You cannot execute code, and you must not -assume any test result that is absent from the supplied evidence. Your task is classification only; do not -repair the sample. - -Mandatory policy: - -1. A sample with `resolved=0` can never become a successful full-trajectory SFT example. -2. A sample with `resolved=-1` cannot be promoted to a successful positive solely from your code intuition. -3. Only `resolved=1` with every static QC requirement satisfied may be `ACCEPT_SILVER_POSITIVE`. -4. Explicit test failures, compilation failures, timeouts, or trustworthy non-zero exit codes override an - assistant claim that the task succeeded. -5. Verification must occur after the final code-changing action. A later code change invalidates earlier tests. -6. A pipeline such as `test | grep`, `test | head`, `test | tail`, or `test | tee` has an untrustworthy exit - code unless `pipefail` or the left-side exit code is explicitly captured. -7. Build, syntax, and type checks are not functional tests. -8. A model patch need not be identical to the reference patch. Judge whether it addresses the same requirement - and core logic; do not reject solely because file sets differ. -9. Every critical conclusion must cite a turn ID or an exact metadata/patch field and include a quote no longer - than 240 characters. -10. If evidence is missing or conflicting, use `HOLD_UNVERIFIED`; do not guess. -11. `qc_passed` may be true only for `ACCEPT_SILVER_POSITIVE`. -12. The supplied trajectory is a turn-preserving API evidence view. A - `[COMPACTED ...]` marker is not dataset corruption and must not itself cause - a trajectory-integrity failure. Use the retained head/tail text, turn IDs, - static signals, character counts, and hashes; choose `HOLD_UNVERIFIED` when - decisive evidence falls outside a preview. - -Decision order: - -A. Map the immutable `resolved` value to `source_outcome_class`. -B. Validate trajectory and tool structure. -C. Locate the final state-changing turn. -D. Evaluate the final relevant verification after that turn, including scope and exit-code trustworthiness. -E. Compare files observed in tool calls, git status, assistant claims, and the model patch. -F. Check user constraints, patch contamination, and final-claim alignment. -G. Use the reference patch only as non-binding semantic evidence. -H. Return one schema-valid JSON object with decision, training use, failures, warnings, and cited evidence. diff --git a/src/swe_data_processing/prompts/prefix_quality.md b/src/swe_data_processing/prompts/prefix_quality.md deleted file mode 100644 index 6d7db17..0000000 --- a/src/swe_data_processing/prompts/prefix_quality.md +++ /dev/null @@ -1,50 +0,0 @@ -You are scoring process-SFT data. You can see only the exact prefix that would be kept for training. You cannot run -code, and you know nothing about any removed suffix or external outcome. - -First identify every concrete error or inefficiency and say whether it was recovered. Cite an unrecovered MAJOR or -CRITICAL issue in `evidence_turns`. Python derives prefix validity from this issue list and attaches exact excerpts. - -Check four things before scoring: - -- coverage: visible work follows the actual requested behavior and exact interfaces; -- current state: edits do not leave known compile errors, failing callers, or repository pollution; -- constraints: literal scope rules such as changing only non-test files are respected; -- honesty: conclusions match the latest relevant tool observations. - -A passing test does not excuse a missing requirement, wrong API, unrelated diff, or violated scope constraint. -When `final_model_patch` is present, it is the exact final diff for this full trajectory and should be checked. It is -omitted for a truncated prefix because later edits are intentionally hidden. - -If the user says test changes were already handled, an old visible test that expects the old behavior may be stale. -Do not call a correct source change unsafe only for that mismatch. Still treat modifying those test files as a scope -violation when the user asked for minimal non-test changes. - -Also list concrete erroneous or inefficient behavior. Anchor each issue to the most useful visible turn: usually the -assistant action, or its tool-result turn when the observed failure is the clearest evidence. An ERROR is a wrong tool -call, edit, interpretation, or claim. An INEFFICIENCY is avoidable repetition, noise, or a clearly wasteful detour. A -reasonable experiment that fails and is then read and handled correctly is not automatically an error. For every -issue, say whether the bad state was later recovered. Recovered mistakes may be listed, but they do not by themselves -make the prefix invalid. Any unrecovered MAJOR or CRITICAL issue makes the prefix invalid. - -An incomplete prefix can still be useful. It may teach problem understanding, repository exploration, reproduction, -tool use, or correct reading of tool results. It does not need to solve the issue. Ending after a tool result is valid. - -Score these five dimensions from 0 to 20: - -- planning: understands the task and forms sensible next steps; -- investigation: finds and reads relevant code or evidence; -- tool_use_and_observation: uses tools sensibly and interprets results honestly; -- progress: makes useful progress toward reproduction, diagnosis, or implementation; -- clarity_and_efficiency: avoids severe loops, noise, and unsupported claims. - -Judge only visible prefix behavior. Do not reduce the score for imagined later failures. Apply only literal user -constraints, not stronger paraphrases. - -Positive example: the prefix identifies the relevant files, reproduces the bug, reads the failure correctly, and -ends before implementation. It can be MEDIUM or HIGH even though it is incomplete. - -Negative example: the prefix itself contains an unreverted wrong-file edit and then ends. Mark it invalid even if -the earlier investigation was useful. - -Broken-state example: the prefix changes a function signature, has not updated known callers, and ends after a build -failure. It is invalid. Incomplete investigation with no harmful edit can still be valid. diff --git a/src/swe_data_processing/prompts/repair.md b/src/swe_data_processing/prompts/repair.md deleted file mode 100644 index 21b673f..0000000 --- a/src/swe_data_processing/prompts/repair.md +++ /dev/null @@ -1,47 +0,0 @@ -You are a static repair planner for Open-SWE-Traces. You have no execution environment. You may propose only -allowlisted repairs that preserve every execution fact, and you must return exactly one schema-valid JSON object. - -Immutable facts: - -- Do not change `resolved`, the model patch, or the reference patch. -- Preserve every existing tool output byte-for-byte. -- Do not create shell, test, build, lint, or tool results. -- Do not change tool-call semantics while retaining the old tool result. -- `resolved=0` cannot be repaired into a successful positive example. -- `resolved=-1` cannot be upgraded to `SFT_FULL`. -- Truncation or a corrected next action has a maximum use of `SFT_STEP_ONLY`. -- A rejected trajectory must remain unchanged when retained as an authentic - `DPO_REJECTED` example. Do not rewrite its failure or overconfidence away. - -The only allowed operations are: `CANONICALIZE_JSON`, `NORMALIZE_TOOL_ALIAS`, `REDACT_SECRET`, -`REMOVE_DUPLICATE_PROSE`, `REWRITE_FINAL_SUMMARY`, `DROP_REDUNDANT_READ_ONLY_PAIR`, -`TRUNCATE_BEFORE_TURN`, and `EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN`. - -For every proposed operation, provide exact target turns, machine-checkable preconditions, an exact replacement, -and a reason. If a precondition cannot be proven from the input, do not propose the operation. - -Choose `REQUIRES_EXECUTION` or `DROP` instead of full-trajectory static repair when the problem involves code -semantics, failed tests, timeouts, missing dependencies, patch/trajectory inconsistency, prohibited test-file -changes, removal of stateful operations, source-code changes, or any need to create a new tool result. - -Step-only truncation is narrower than full-trajectory repair. Use `CREATE_STEP_EXAMPLE` only when all of the -following are true: - -1. A specific bad assistant turn is identifiable from evidence already visible before that turn, such as an - ignored test failure, deletion of an unexecuted reproduction, a prohibited test-file edit, or an unsupported - success claim. -2. The correct immediate next assistant action is uniquely defensible from that prior evidence without using the - hidden outcome or reference patch as privileged information. -3. The replacement is a complete assistant message and contains at most a tool call or an honest diagnostic - response. It must not claim the task is solved and must not include a fabricated tool result. -4. Operations first replace the bad assistant turn with `EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN`, then use - `TRUNCATE_BEFORE_TURN` on the following turn. The resulting record ends at the corrected assistant action. -5. `maximum_training_use` is exactly `SFT_STEP_ONLY`. - -Do not truncate merely to hide a wrong patch, failed test, or external `resolved=0` label. If no uniquely correct -next action exists, retain the authentic negative, require execution, or drop the record. - -Decisions `NO_CHANGE`, `REQUIRES_EXECUTION`, and `DROP` must have an empty operations array. `APPLY_STATIC_REPAIR` -and `CREATE_STEP_EXAMPLE` must have at least one operation. - -Before returning, confirm all evidence-preservation invariants are true and require independent second review. diff --git a/src/swe_data_processing/prompts/review.md b/src/swe_data_processing/prompts/review.md deleted file mode 100644 index 98282db..0000000 --- a/src/swe_data_processing/prompts/review.md +++ /dev/null @@ -1,16 +0,0 @@ -You are an independent reviewer of a static Open-SWE-Traces repair. You cannot execute code. Compare the original -sample, classification, repair plan, repaired sample, and structured diff. Approval means only that the static -repair is faithful, safe, and structurally valid; it does not prove code correctness. - -Reject the repair if any of the following is true: - -- `resolved`, the model patch, or the reference patch changed. -- Any tool output changed or a synthetic execution result was added. -- A file write, git operation, installation, service start, or other stateful turn was removed. -- Removal created a dangling tool pair or invalidated later state dependencies. -- Tool-call semantics changed without a newly executed result. -- A `resolved=0` or `resolved=-1` sample was upgraded to `SFT_FULL`. -- A truncated sample was represented as a complete successful trajectory. -- Training use exceeds the maximum supported by the original evidence and classification. - -Every invariant violation must set `approved=false`. Return exactly one JSON object matching the review schema. diff --git a/src/swe_data_processing/repair.py b/src/swe_data_processing/repair.py deleted file mode 100644 index 455c070..0000000 --- a/src/swe_data_processing/repair.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Deterministic application of allowlisted static repair operations.""" - -from __future__ import annotations - -import copy -import hashlib -import json -from typing import Any - -from .features import STATEFUL_SHELL_RE -from .io import get_sample_id -from .policy import PolicyViolation, enforce_repair_policy - - -def _hash(value: Any) -> str: - """Return a stable SHA-256 digest for audit records.""" - - encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - -def _assistant_command(message: dict[str, Any]) -> str: - """Extract a shell command from the first tool call in an assistant message.""" - - calls = message.get("tool_calls") or [] - if not calls or not isinstance(calls[0], dict): - return "" - function = calls[0].get("function") or {} - raw_arguments = function.get("arguments", "{}") if isinstance(function, dict) else "{}" - try: - arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments - except json.JSONDecodeError: - return "" - return str(arguments.get("command") or arguments.get("cmd") or "") if isinstance(arguments, dict) else "" - - -def _is_read_only_pair(assistant_message: dict[str, Any], tool_message: dict[str, Any]) -> bool: - """Conservatively determine whether a tool pair has no repository side effects.""" - - if assistant_message.get("role") != "assistant" or tool_message.get("role") != "tool": - return False - calls = assistant_message.get("tool_calls") or [] - if len(calls) != 1 or not isinstance(calls[0], dict): - return False - function = calls[0].get("function") or {} - name = function.get("name") if isinstance(function, dict) else None - if name == "str_replace_editor": - raw_arguments = function.get("arguments", "{}") - try: - arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments - except json.JSONDecodeError: - return False - return isinstance(arguments, dict) and arguments.get("command") == "view" - command = _assistant_command(assistant_message) - return bool(command) and not STATEFUL_SHELL_RE.search(command) - - -def _replace_message( - entry: dict[str, Any], - operation: str, - replacement: Any, -) -> None: - """Apply a non-deleting operation to one annotated trajectory entry.""" - - message = entry["message"] - if message.get("role") == "tool": - raise PolicyViolation(f"{operation} cannot modify a tool observation") - - if operation in {"REMOVE_DUPLICATE_PROSE", "REWRITE_FINAL_SUMMARY", "REDACT_SECRET"}: - if not isinstance(replacement, str): - raise PolicyViolation(f"{operation} requires a string replacement") - if message.get("role") != "assistant": - raise PolicyViolation(f"{operation} is restricted to assistant messages") - message["content"] = replacement - return - - if operation in { - "CANONICALIZE_JSON", - "NORMALIZE_TOOL_ALIAS", - "EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN", - }: - if not isinstance(replacement, dict) or replacement.get("role") != "assistant": - raise PolicyViolation(f"{operation} requires a complete assistant-message object") - entry["message"] = copy.deepcopy(replacement) - return - - raise PolicyViolation(f"Unsupported non-deleting operation: {operation}") - - -def apply_static_repair( - record: dict[str, Any], - repair_plan: dict[str, Any], -) -> tuple[dict[str, Any], dict[str, Any]]: - """Apply an approved allowlisted plan and return a structured audit diff. - - The function performs only syntactic operations. It never executes code and - never changes outcome labels or patch metadata. - """ - - enforce_repair_policy(repair_plan) - decision = repair_plan["repair_decision"] - if decision not in {"APPLY_STATIC_REPAIR", "CREATE_STEP_EXAMPLE"}: - raise PolicyViolation(f"Repair decision {decision} does not authorize mutation") - - original = copy.deepcopy(record) - repaired = copy.deepcopy(record) - entries = [ - {"original_turn_id": turn_id, "message": copy.deepcopy(message)} - for turn_id, message in enumerate(repaired.get("trajectory") or [], 1) - ] - removed_turns: list[int] = [] - replaced_turns: list[int] = [] - - def find_entry(turn_id: int) -> dict[str, Any]: - for item in entries: - if item["original_turn_id"] == turn_id: - return item - raise PolicyViolation(f"Repair plan references missing or already removed turn {turn_id}") - - for operation in repair_plan["operations"]: - op = operation["op"] - targets = operation["target_turns"] - replacement = operation["replacement"] - - if op == "TRUNCATE_BEFORE_TURN": - if len(targets) != 1: - raise PolicyViolation("TRUNCATE_BEFORE_TURN requires exactly one target turn") - cutoff = targets[0] - to_remove = [item["original_turn_id"] for item in entries if item["original_turn_id"] >= cutoff] - entries[:] = [item for item in entries if item["original_turn_id"] < cutoff] - removed_turns.extend(to_remove) - continue - - if op == "DROP_REDUNDANT_READ_ONLY_PAIR": - if len(targets) != 2 or targets[1] != targets[0] + 1: - raise PolicyViolation("A read-only pair must contain two consecutive original turns") - assistant_entry = find_entry(targets[0]) - tool_entry = find_entry(targets[1]) - if not _is_read_only_pair(assistant_entry["message"], tool_entry["message"]): - raise PolicyViolation("The requested pair is not provably read-only") - entries[:] = [item for item in entries if item["original_turn_id"] not in set(targets)] - removed_turns.extend(targets) - continue - - if len(targets) != 1: - raise PolicyViolation(f"{op} requires exactly one target turn") - entry = find_entry(targets[0]) - _replace_message(entry, op, replacement) - replaced_turns.append(targets[0]) - - repaired["trajectory"] = [item["message"] for item in entries] - - # Deterministic invariants protect immutable outcome and patch metadata. - if repaired.get("resolved") != original.get("resolved"): - raise PolicyViolation("Static repair changed resolved") - original_metadata = original.get("metadata") or {} - repaired_metadata = repaired.get("metadata") or {} - for key in ("model_patch", "reference_patch"): - if repaired_metadata.get(key) != original_metadata.get(key): - raise PolicyViolation(f"Static repair changed immutable metadata field {key}") - - # Every retained tool observation must remain byte-for-byte equivalent. - original_tool_hashes = { - turn_id: _hash(message) - for turn_id, message in enumerate(original.get("trajectory") or [], 1) - if isinstance(message, dict) and message.get("role") == "tool" - } - retained_entries = {item["original_turn_id"]: item["message"] for item in entries} - for turn_id, digest in original_tool_hashes.items(): - if turn_id in retained_entries and _hash(retained_entries[turn_id]) != digest: - raise PolicyViolation(f"Static repair modified retained tool output at turn {turn_id}") - - diff = { - "sample_id": get_sample_id(record), - "input_sha256": _hash(original), - "output_sha256": _hash(repaired), - "removed_original_turn_ids": sorted(set(removed_turns)), - "replaced_original_turn_ids": sorted(set(replaced_turns)), - "original_turn_count": len(original.get("trajectory") or []), - "repaired_turn_count": len(repaired.get("trajectory") or []), - "maximum_training_use": repair_plan["maximum_training_use"], - } - return repaired, diff diff --git a/src/swe_data_processing/resources.py b/src/swe_data_processing/resources.py deleted file mode 100644 index 883d0ec..0000000 --- a/src/swe_data_processing/resources.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Load packaged prompts and JSON Schemas by stable resource name.""" - -from __future__ import annotations - -import json -from importlib.resources import files -from typing import Any - - -def load_prompt(name: str) -> str: - """Load a UTF-8 prompt from the package's ``prompts`` directory.""" - - resource = files("swe_data_processing").joinpath("prompts", name) - return resource.read_text(encoding="utf-8") - - -def load_schema(name: str) -> dict[str, Any]: - """Load and decode a JSON Schema from packaged resources.""" - - resource = files("swe_data_processing").joinpath("schemas", name) - return json.loads(resource.read_text(encoding="utf-8")) diff --git a/src/swe_data_processing/sampling.py b/src/swe_data_processing/sampling.py new file mode 100644 index 0000000..cefa5d7 --- /dev/null +++ b/src/swe_data_processing/sampling.py @@ -0,0 +1,90 @@ +"""Deterministic human-review sampling from heuristic decisions.""" + +from __future__ import annotations + +import random +from collections import defaultdict +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pyarrow.parquet as pq + +from .io import get_sample_id, iter_jsonl, iter_records + + +def select_review_ids( + decisions_path: Path, per_group: int, seed: int +) -> tuple[dict[str, list[str]], dict[str, dict[str, Any]]]: + """Sample IDs independently from each rule and length bucket.""" + + groups: dict[str, list[str]] = defaultdict(list) + decisions: dict[str, dict[str, Any]] = {} + for decision in iter_jsonl(decisions_path): + sample_id = get_sample_id(decision) + decisions[sample_id] = decision + for reason in decision["hard_reject_reasons"]: + groups[f"hard:{reason}"].append(sample_id) + for flag in decision["review_flags"]: + groups[f"review:{flag}"].append(sample_id) + groups[f"length:{decision['length_bucket']}"].append(sample_id) + + rng = random.Random(seed) + selected: dict[str, list[str]] = defaultdict(list) + for group, sample_ids in sorted(groups.items()): + chosen = rng.sample(sample_ids, min(per_group, len(sample_ids))) + for sample_id in chosen: + selected[sample_id].append(group) + return dict(selected), decisions + + +def iter_review_records( + input_path: Path, + selected: dict[str, list[str]], + decisions: dict[str, dict[str, Any]], +) -> Iterator[dict[str, Any]]: + """Yield selected full records with their deterministic review metadata.""" + + remaining = set(selected) + source_paths = { + sample_id: decisions[sample_id].get("source_parquet") for sample_id in selected + } + if input_path.is_dir() and all(source_paths.values()): + records = _iter_selected_parquet_records(source_paths) + else: + records = iter_records(input_path) + + for record in records: + sample_id = get_sample_id(record) + if sample_id not in remaining: + continue + output = dict(record) + output["_qc_review"] = { + "groups": selected[sample_id], + "decision": decisions[sample_id], + } + yield output + remaining.remove(sample_id) + if not remaining: + break + if remaining: + missing = ", ".join(sorted(remaining)[:10]) + raise ValueError(f"Selected sample IDs were not found in input: {missing}") + + +def _iter_selected_parquet_records( + source_paths: dict[str, str | None], +) -> Iterator[dict[str, Any]]: + """Read only Parquet shards that contain selected sample IDs.""" + + ids_by_path: dict[Path, set[str]] = defaultdict(set) + for sample_id, raw_path in source_paths.items(): + if raw_path is not None: + ids_by_path[Path(raw_path)].add(sample_id) + + for path, sample_ids in sorted(ids_by_path.items(), key=lambda item: str(item[0])): + parquet_file = pq.ParquetFile(path) + for batch in parquet_file.iter_batches(batch_size=64): + for record in batch.to_pylist(): + if isinstance(record, dict) and get_sample_id(record) in sample_ids: + yield record diff --git a/src/swe_data_processing/schemas/boundary_output.schema.json b/src/swe_data_processing/schemas/boundary_output.schema.json deleted file mode 100644 index 6af6390..0000000 --- a/src/swe_data_processing/schemas/boundary_output.schema.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "additionalProperties": false, - "required": [ - "sample_id", "checks", "decision", "truncate_before_turn", "category", - "severity", "state_effect", "evidence_turns", "reason" - ], - "properties": { - "sample_id": {"type": "string", "minLength": 1}, - "checks": { - "type": "object", - "additionalProperties": false, - "required": [ - "task_coverage", "final_patch_scope", "constraints", "claims_vs_observations" - ], - "properties": { - "task_coverage": {"enum": ["COMPLETE", "INCOMPLETE", "UNCLEAR"]}, - "final_patch_scope": {"enum": ["CLEAN", "POLLUTED", "UNCLEAR"]}, - "constraints": {"enum": ["RESPECTED", "VIOLATED", "UNCLEAR"]}, - "claims_vs_observations": {"enum": ["CONSISTENT", "CONTRADICTED", "UNCLEAR"]} - } - }, - "decision": {"enum": ["TRUNCATE", "KEEP_FULL", "HOLD"]}, - "truncate_before_turn": {"type": ["integer", "null"], "minimum": 1}, - "category": { - "enum": [ - "NONE", "USER_CONSTRAINT_VIOLATION", "HARMFUL_STATE_CHANGE", - "PERSISTENT_WRONG_IMPLEMENTATION", "BROKEN_TOOL_STRUCTURE", - "UNGROUNDED_FINAL_CLAIM" - ] - }, - "severity": {"enum": ["NONE", "MAJOR", "CRITICAL"]}, - "state_effect": {"enum": ["NONE", "UNRECOVERED", "UNCLEAR"]}, - "evidence_turns": { - "type": "array", - "items": {"type": "integer", "minimum": 1}, - "uniqueItems": true, - "maxItems": 4, - "default": [] - }, - "reason": {"type": "string", "minLength": 1, "maxLength": 1000} - } -} diff --git a/src/swe_data_processing/schemas/classification_output.schema.json b/src/swe_data_processing/schemas/classification_output.schema.json deleted file mode 100644 index 4069b25..0000000 --- a/src/swe_data_processing/schemas/classification_output.schema.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "additionalProperties": false, - "required": [ - "sample_id", "source_outcome_class", "qc_decision", "training_use", "qc_passed", - "confidence", "verification", "dimensions", "hard_fail_codes", "warning_codes", - "evidence", "repairability", "summary" - ], - "properties": { - "sample_id": {"type": "string", "minLength": 1}, - "source_outcome_class": { - "enum": ["POSITIVE_CANDIDATE", "EXPLICIT_NEGATIVE", "UNVERIFIED"] - }, - "qc_decision": { - "enum": [ - "ACCEPT_SILVER_POSITIVE", "ACCEPT_NEGATIVE", "STATIC_REPAIR", - "HOLD_UNVERIFIED", "REJECT" - ] - }, - "training_use": { - "enum": ["SFT_FULL", "SFT_STEP_ONLY", "DPO_REJECTED", "ERROR_ANALYSIS", "HOLD", "DROP"] - }, - "qc_passed": {"type": "boolean"}, - "confidence": {"type": "number", "minimum": 0, "maximum": 1}, - "verification": { - "type": "object", - "additionalProperties": false, - "required": [ - "status", "level", "last_code_change_turn", "last_relevant_test_turn", - "test_after_final_change", "scope_relevant", "exit_code_trustworthy" - ], - "properties": { - "status": { - "enum": ["PASS_RELIABLE", "FAIL_EXPLICIT", "MASKED_EXIT", "ENV_BLOCKED", "INSUFFICIENT", "NONE"] - }, - "level": {"enum": ["NONE", "SYNTAX", "BUILD", "TARGETED", "MODULE", "FULL"]}, - "last_code_change_turn": {"type": ["integer", "null"], "minimum": 1}, - "last_relevant_test_turn": {"type": ["integer", "null"], "minimum": 1}, - "test_after_final_change": {"type": "boolean"}, - "scope_relevant": {"type": "boolean"}, - "exit_code_trustworthy": {"type": "boolean"} - } - }, - "dimensions": { - "type": "object", - "additionalProperties": false, - "required": [ - "trajectory_integrity", "tool_integrity", "patch_presence", - "patch_trajectory_consistency", "instruction_compliance", "verification_consistency", - "final_claim_alignment", "patch_hygiene", "issue_patch_alignment" - ], - "properties": { - "trajectory_integrity": {"$ref": "#/$defs/dimension"}, - "tool_integrity": {"$ref": "#/$defs/dimension"}, - "patch_presence": {"$ref": "#/$defs/dimension"}, - "patch_trajectory_consistency": {"$ref": "#/$defs/dimension"}, - "instruction_compliance": {"$ref": "#/$defs/dimension"}, - "verification_consistency": {"$ref": "#/$defs/dimension"}, - "final_claim_alignment": {"$ref": "#/$defs/dimension"}, - "patch_hygiene": {"$ref": "#/$defs/dimension"}, - "issue_patch_alignment": {"$ref": "#/$defs/dimension"} - } - }, - "hard_fail_codes": { - "type": "array", "uniqueItems": true, - "items": {"$ref": "#/$defs/hardFailCode"} - }, - "warning_codes": { - "type": "array", "uniqueItems": true, - "items": {"$ref": "#/$defs/warningCode"} - }, - "evidence": { - "type": "array", "minItems": 1, "maxItems": 20, - "items": { - "type": "object", "additionalProperties": false, - "required": ["turn_id", "field", "code", "quote"], - "properties": { - "turn_id": {"type": ["integer", "null"], "minimum": 1}, - "field": {"type": "string", "minLength": 1, "maxLength": 80}, - "code": {"type": "string", "minLength": 1, "maxLength": 80}, - "quote": {"type": "string", "minLength": 1, "maxLength": 240} - } - } - }, - "repairability": { - "type": "object", "additionalProperties": false, - "required": ["decision", "safe_operations"], - "properties": { - "decision": { - "enum": ["NOT_NEEDED", "SAFE_STATIC", "STEP_ONLY", "REQUIRES_EXECUTION", "NOT_REPAIRABLE"] - }, - "safe_operations": { - "type": "array", "uniqueItems": true, - "items": {"$ref": "#/$defs/repairOperation"} - } - } - }, - "summary": {"type": "string", "minLength": 1, "maxLength": 1200} - }, - "$defs": { - "dimension": {"enum": ["PASS", "FAIL", "UNKNOWN"]}, - "hardFailCode": { - "enum": [ - "RESOLVED_ZERO_FOR_POSITIVE", "FINAL_RELEVANT_TEST_FAILED", "SUCCESS_LOG_CONTRADICTION", - "MALFORMED_TOOL_ARGUMENTS", "UNKNOWN_TOOL", "UNPAIRED_TOOL_RESULT", "MISSING_PATCH", - "PATCH_TRAJECTORY_MISMATCH", "INSTRUCTION_VIOLATION", "PATCH_CONTAMINATION_SEVERE", - "INCOMPLETE_TRAJECTORY", "UNSAFE_OR_UNAUTHORIZED_ACTION", "ISSUE_PATCH_MISMATCH", - "POST_TEST_CODE_CHANGE", "UNPROVEN_PREEXISTING_FAILURE" - ] - }, - "warningCode": { - "enum": [ - "UNTRUSTWORTHY_TEST_PIPELINE", "NO_VERIFICATION", "BUILD_ONLY", "TARGETED_TEST_ONLY", - "ENVIRONMENT_FAILURE", "REFERENCE_PATCH_LOW_OVERLAP", "PATCH_SIZE_OUTLIER", - "REDUNDANT_TOOL_CALLS", "REPETITIVE_SUCCESS_CLAIMS", "TEMPORARY_FILES_OBSERVED", - "TEST_FILE_MODIFIED", "VENDOR_OR_GENERATED_FILE_MODIFIED" - ] - }, - "repairOperation": { - "enum": [ - "CANONICALIZE_JSON", "NORMALIZE_TOOL_ALIAS", "REDACT_SECRET", "REMOVE_DUPLICATE_PROSE", - "REWRITE_FINAL_SUMMARY", "DROP_REDUNDANT_READ_ONLY_PAIR", "TRUNCATE_BEFORE_TURN", - "EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN" - ] - } - } -} diff --git a/src/swe_data_processing/schemas/prefix_quality_output.schema.json b/src/swe_data_processing/schemas/prefix_quality_output.schema.json deleted file mode 100644 index 4ddc1c8..0000000 --- a/src/swe_data_processing/schemas/prefix_quality_output.schema.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "additionalProperties": false, - "required": [ - "sample_id", "behavior_issues", "dimensions", "evidence_turns", "reason" - ], - "properties": { - "sample_id": {"type": "string", "minLength": 1}, - "behavior_issues": { - "type": "array", - "maxItems": 20, - "items": {"$ref": "#/$defs/behavior_issue"} - }, - "dimensions": { - "type": "object", - "additionalProperties": false, - "required": [ - "planning", "investigation", "tool_use_and_observation", "progress", - "clarity_and_efficiency" - ], - "properties": { - "planning": {"type": "integer", "minimum": 0, "maximum": 20}, - "investigation": {"type": "integer", "minimum": 0, "maximum": 20}, - "tool_use_and_observation": {"type": "integer", "minimum": 0, "maximum": 20}, - "progress": {"type": "integer", "minimum": 0, "maximum": 20}, - "clarity_and_efficiency": {"type": "integer", "minimum": 0, "maximum": 20} - } - }, - "evidence_turns": { - "type": "array", - "items": {"type": "integer", "minimum": 1}, - "uniqueItems": true, - "maxItems": 4, - "default": [] - }, - "reason": {"type": "string", "minLength": 1, "maxLength": 800} - }, - "$defs": { - "behavior_issue": { - "type": "object", - "additionalProperties": false, - "required": ["turn_id", "kind", "severity", "recovered", "reason"], - "properties": { - "turn_id": {"type": "integer", "minimum": 1}, - "kind": {"enum": ["ERROR", "INEFFICIENCY"]}, - "severity": {"enum": ["MINOR", "MAJOR", "CRITICAL"]}, - "recovered": {"type": "boolean"}, - "reason": {"type": "string", "minLength": 1, "maxLength": 500} - } - } - } -} diff --git a/src/swe_data_processing/schemas/repair_output.schema.json b/src/swe_data_processing/schemas/repair_output.schema.json deleted file mode 100644 index 7caa70b..0000000 --- a/src/swe_data_processing/schemas/repair_output.schema.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "additionalProperties": false, - "required": [ - "sample_id", "repair_decision", "maximum_training_use", "invariants", - "operations", "requires_second_review", "summary" - ], - "properties": { - "sample_id": {"type": "string", "minLength": 1}, - "repair_decision": { - "enum": ["NO_CHANGE", "APPLY_STATIC_REPAIR", "CREATE_STEP_EXAMPLE", "REQUIRES_EXECUTION", "DROP"] - }, - "maximum_training_use": { - "enum": ["SFT_FULL", "SFT_STEP_ONLY", "DPO_REJECTED", "ERROR_ANALYSIS", "HOLD", "DROP"] - }, - "invariants": { - "type": "object", "additionalProperties": false, - "required": [ - "resolved_unchanged", "tool_outputs_unchanged", "model_patch_unchanged", - "reference_patch_unchanged", "no_synthetic_execution_result" - ], - "properties": { - "resolved_unchanged": {"const": true}, - "tool_outputs_unchanged": {"const": true}, - "model_patch_unchanged": {"const": true}, - "reference_patch_unchanged": {"const": true}, - "no_synthetic_execution_result": {"const": true} - } - }, - "operations": { - "type": "array", "maxItems": 30, - "items": { - "type": "object", "additionalProperties": false, - "required": ["op", "target_turns", "preconditions", "replacement", "reason"], - "properties": { - "op": {"$ref": "#/$defs/repairOperation"}, - "target_turns": { - "type": "array", "uniqueItems": true, - "items": {"type": "integer", "minimum": 1} - }, - "preconditions": { - "type": "array", "minItems": 1, - "items": {"type": "string", "minLength": 1, "maxLength": 240} - }, - "replacement": {"type": ["string", "object", "array", "null"]}, - "reason": {"type": "string", "minLength": 1, "maxLength": 600} - } - } - }, - "requires_second_review": {"const": true}, - "summary": {"type": "string", "minLength": 1, "maxLength": 1200} - }, - "$defs": { - "repairOperation": { - "enum": [ - "CANONICALIZE_JSON", "NORMALIZE_TOOL_ALIAS", "REDACT_SECRET", "REMOVE_DUPLICATE_PROSE", - "REWRITE_FINAL_SUMMARY", "DROP_REDUNDANT_READ_ONLY_PAIR", "TRUNCATE_BEFORE_TURN", - "EMIT_STRUCTURALLY_CORRECT_ASSISTANT_TURN" - ] - } - } -} diff --git a/src/swe_data_processing/schemas/review_output.schema.json b/src/swe_data_processing/schemas/review_output.schema.json deleted file mode 100644 index 9254ba5..0000000 --- a/src/swe_data_processing/schemas/review_output.schema.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "additionalProperties": false, - "required": [ - "sample_id", "approved", "maximum_training_use", "invariant_violations", - "unsupported_changes", "evidence", "summary" - ], - "properties": { - "sample_id": {"type": "string", "minLength": 1}, - "approved": {"type": "boolean"}, - "maximum_training_use": { - "enum": ["SFT_FULL", "SFT_STEP_ONLY", "DPO_REJECTED", "ERROR_ANALYSIS", "HOLD", "DROP"] - }, - "invariant_violations": { - "type": "array", "uniqueItems": true, - "items": { - "enum": [ - "RESOLVED_CHANGED", "TOOL_OUTPUT_CHANGED", "MODEL_PATCH_CHANGED", "REFERENCE_PATCH_CHANGED", - "SYNTHETIC_EXECUTION_RESULT_ADDED", "STATEFUL_TURN_REMOVED", "DANGLING_TOOL_PAIR", - "SEMANTIC_TOOL_CALL_CHANGED_WITHOUT_EXECUTION", "TRAINING_USE_UPGRADED_WITHOUT_EVIDENCE" - ] - } - }, - "unsupported_changes": { - "type": "array", - "items": {"type": "string", "minLength": 1, "maxLength": 400} - }, - "evidence": { - "type": "array", "minItems": 1, "maxItems": 20, - "items": { - "type": "object", "additionalProperties": false, - "required": ["turn_id", "quote", "assessment"], - "properties": { - "turn_id": {"type": ["integer", "null"], "minimum": 1}, - "quote": {"type": "string", "minLength": 1, "maxLength": 240}, - "assessment": {"type": "string", "minLength": 1, "maxLength": 400} - } - } - }, - "summary": {"type": "string", "minLength": 1, "maxLength": 1000} - } -} diff --git a/src/swe_data_processing/summary.py b/src/swe_data_processing/summary.py new file mode 100644 index 0000000..b4e16b8 --- /dev/null +++ b/src/swe_data_processing/summary.py @@ -0,0 +1,178 @@ +"""Dataset-level aggregation for deterministic Open-SWE-Traces metrics.""" + +from __future__ import annotations + +import math +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + +from .heuristics import classify_metrics, length_bucket +from .io import iter_jsonl + + +def percentile(values: list[float | int], fraction: float) -> float | None: + """Return a linearly interpolated percentile for a finite value list.""" + + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return float(ordered[lower]) + weight = position - lower + return float(ordered[lower] * (1 - weight) + ordered[upper] * weight) + + +def _distribution(values: list[float | int]) -> dict[str, Any]: + """Return compact quantiles used in reports and threshold selection.""" + + if not values: + return {"count": 0} + return { + "count": len(values), + "min": min(values), + "p50": percentile(values, 0.5), + "p90": percentile(values, 0.9), + "p95": percentile(values, 0.95), + "p99": percentile(values, 0.99), + "max": max(values), + "mean": round(sum(values) / len(values), 6), + } + + +def build_summary(metrics_path: Path) -> tuple[dict[str, Any], int]: + """Aggregate one metrics JSONL and derive the global error threshold.""" + + turns: list[int] = [] + chars: list[int] = [] + tokens: list[int] = [] + error_counts: list[int] = [] + error_rates: list[float] = [] + resolved_counts: Counter[str] = Counter() + length_counts: Counter[str] = Counter() + length_tokens: Counter[str] = Counter() + outcome_length_counts: dict[str, Counter[str]] = defaultdict(Counter) + outcome_length_tokens: dict[str, Counter[str]] = defaultdict(Counter) + error_types: Counter[str] = Counter() + error_tools: Counter[str] = Counter() + source_counts: Counter[str] = Counter() + source_values: dict[str, dict[str, list[float | int]]] = defaultdict( + lambda: {"turns": [], "tokens": [], "errors": [], "error_rates": []} + ) + + for metrics in iter_jsonl(metrics_path): + length = metrics["length"] + tools = metrics["tools"] + token_count = length.get("token_count") + turns.append(int(length["turn_count"])) + chars.append(int(length["canonical_chars"])) + error_counts.append(int(tools["failed_tool_call_count"])) + error_rates.append(float(tools["failed_tool_call_rate"])) + resolved_counts[str(metrics.get("resolved"))] += 1 + source_counts[str(metrics.get("source_dataset") or "unknown")] += 1 + source_group = str(metrics.get("source_group") or "unknown") + source_values[source_group]["turns"].append(int(length["turn_count"])) + source_values[source_group]["errors"].append( + int(tools["failed_tool_call_count"]) + ) + source_values[source_group]["error_rates"].append( + float(tools["failed_tool_call_rate"]) + ) + error_types.update(tools["error_type_counts"]) + error_tools.update(tools["error_tool_counts"]) + bucket = length_bucket(token_count) + length_counts[bucket] += 1 + outcome_key = str(metrics.get("resolved")) + outcome_length_counts[outcome_key][bucket] += 1 + if token_count is not None: + token_value = int(token_count) + tokens.append(token_value) + length_tokens[bucket] += token_value + outcome_length_tokens[outcome_key][bucket] += token_value + source_values[source_group]["tokens"].append(token_value) + + p99 = percentile(error_counts, 0.99) or 0 + high_error_count_threshold = max(5, math.ceil(p99)) + total_tokens = sum(tokens) + bucket_summary = {} + total = len(turns) + for bucket, count in sorted(length_counts.items()): + bucket_tokens = length_tokens[bucket] + bucket_summary[bucket] = { + "samples": count, + "sample_share": round(count / total, 6) if total else 0.0, + "tokens": bucket_tokens if tokens else None, + "token_share": round(bucket_tokens / total_tokens, 6) + if total_tokens + else None, + } + + tail_contribution = {} + if tokens: + descending = sorted(tokens, reverse=True) + for fraction in (0.001, 0.01, 0.05): + count = max(1, math.ceil(len(descending) * fraction)) + tail_contribution[str(fraction)] = { + "samples": count, + "token_share": round(sum(descending[:count]) / total_tokens, 6), + } + + first_pass = { + "sample_count": total, + "resolved_counts": dict(sorted(resolved_counts.items())), + "source_dataset_counts": dict(source_counts.most_common()), + "source_group_profiles": { + group: { + "samples": len(values["turns"]), + "turns": _distribution(values["turns"]), + "tokens": _distribution(values["tokens"]), + "tool_failure_counts": _distribution(values["errors"]), + "tool_failure_rates": _distribution(values["error_rates"]), + } + for group, values in sorted(source_values.items()) + }, + "length": { + "turns": _distribution(turns), + "canonical_chars": _distribution(chars), + "tokens": _distribution(tokens), + "buckets": bucket_summary, + "buckets_by_resolved": { + outcome: { + bucket: { + "samples": count, + "tokens": outcome_length_tokens[outcome][bucket] + if tokens + else None, + } + for bucket, count in sorted(counts.items()) + } + for outcome, counts in sorted(outcome_length_counts.items()) + }, + "longest_sample_token_contribution": tail_contribution, + }, + "tool_failures": { + "count_distribution": _distribution(error_counts), + "rate_distribution": _distribution(error_rates), + "error_type_counts": dict(error_types.most_common()), + "error_tool_counts": dict(error_tools.most_common()), + "high_error_count_threshold_p99": high_error_count_threshold, + }, + } + + actions: Counter[str] = Counter() + hard_rules: Counter[str] = Counter() + review_rules: Counter[str] = Counter() + for metrics in iter_jsonl(metrics_path): + decision = classify_metrics(metrics, high_error_count_threshold) + actions[decision["recommended_action"]] += 1 + hard_rules.update(decision["hard_reject_reasons"]) + review_rules.update(decision["review_flags"]) + first_pass["heuristic_hits"] = { + "recommended_actions": dict(actions.most_common()), + "hard_reject_reasons": dict(hard_rules.most_common()), + "review_flags": dict(review_rules.most_common()), + } + return first_pass, high_error_count_threshold diff --git a/src/swe_data_processing/tokenization.py b/src/swe_data_processing/tokenization.py new file mode 100644 index 0000000..ae23cbe --- /dev/null +++ b/src/swe_data_processing/tokenization.py @@ -0,0 +1,36 @@ +"""Deterministic token counting for canonical trajectory serialization.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any + + +class TokenCounter: + """Count tokens with one explicit Hugging Face ``tokenizer.json`` file. + + The package deliberately has no implicit model download and no approximate + character-to-token conversion. A token count is emitted only when the user + supplies the exact tokenizer artifact used for the analysis. + """ + + def __init__(self, tokenizer_json: Path): + try: + from tokenizers import Tokenizer + except ImportError as exc: + raise RuntimeError( + "Token counting requires the optional 'tokens' dependency: " + "pip install -e '.[tokens]'" + ) from exc + + raw = tokenizer_json.read_bytes() + self.path = tokenizer_json + self.sha256 = hashlib.sha256(raw).hexdigest() + self._tokenizer: Any = Tokenizer.from_file(str(tokenizer_json)) + + def count(self, text: str) -> int: + """Return the exact token count for ``text`` without model downloads.""" + + return len(self._tokenizer.encode(text, add_special_tokens=False).ids) + diff --git a/src/swe_data_processing/workflow.py b/src/swe_data_processing/workflow.py deleted file mode 100644 index 263df52..0000000 --- a/src/swe_data_processing/workflow.py +++ /dev/null @@ -1,343 +0,0 @@ -"""High-level classification, repair planning, and review workflows.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from typing import Any - -from .audit import ( - compute_prefix_quality, - derive_prefix_safety, - effective_boundary_policy, - materialize_evidence, - materialize_prefix, - validate_boundary, - validate_prefix_quality, -) -from .client import GLMClient, GLMResponse -from .evidence import ( - build_trajectory_blocks, - compact_patch_object, - compact_text, - compact_trajectory, -) -from .features import extract_instruction_constraints, extract_static_signals, index_trajectory -from .io import get_sample_id -from .policy import enforce_classification_policy, enforce_repair_policy -from .resources import load_prompt, load_schema - - -def _stable_hash(value: dict[str, Any]) -> str: - """Hash canonical JSON for reproducible provenance records.""" - - encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - -def _first_user_issue(trajectory: list[dict[str, Any]]) -> str: - """Return the first user message, which contains the benchmark issue.""" - - for message in trajectory: - if message.get("role") == "user": - content = message.get("content") - if isinstance(content, str): - return content - return "" - - -def prepare_classification_payload(record: dict[str, Any]) -> dict[str, Any]: - """Create the complete evidence package consumed by the classifier.""" - - trajectory = index_trajectory(record) - metadata = record.get("metadata") or {} - static_signals = extract_static_signals(record) - trajectory_evidence, compaction = compact_trajectory(trajectory, static_signals) - return { - "sample_id": get_sample_id(record), - "resolved": int(record.get("resolved", -1)), - "issue": compact_text(_first_user_issue(trajectory), 16_000), - "instruction_constraints": extract_instruction_constraints(trajectory), - "tool_definition_summary": { - "allowed_tool_names": static_signals["allowed_tool_names"], - "malformed_definition_indexes": static_signals[ - "malformed_tool_definition_indexes" - ], - }, - "trajectory": trajectory_evidence, - "prompt_compaction": compaction, - "model_patch": compact_patch_object(metadata.get("model_patch") or {}), - "reference_patch": compact_patch_object(metadata.get("reference_patch") or {}), - "static_signals": static_signals, - "record_metadata": { - "instance_id": record.get("instance_id"), - "repo": record.get("repo"), - "language": record.get("language"), - "trajectory_id": record.get("trajectory_id"), - "category": metadata.get("category") if isinstance(metadata, dict) else None, - }, - } - - -def _provenance( - stage: str, - response: GLMResponse, - input_payload: dict[str, Any], - client: GLMClient, -) -> dict[str, Any]: - """Build non-sensitive provenance for one successful API stage.""" - - return { - "stage": stage, - "model": client.settings.model, - "endpoint": client.settings.endpoint, - "request_id": response.request_id, - "usage": response.usage, - "compatibility_fallback_used": response.compatibility_fallback_used, - "input_sha256": _stable_hash(input_payload), - "created_at": datetime.now(timezone.utc).isoformat(), - } - - -def classify_record(record: dict[str, Any], client: GLMClient) -> dict[str, Any]: - """Classify one trajectory and enforce immutable outcome policy locally.""" - - payload = prepare_classification_payload(record) - response = client.invoke_json( - system_prompt=load_prompt("classification.md"), - payload=payload, - schema=load_schema("classification_output.schema.json"), - ) - enforce_classification_policy(record, response.data) - result = dict(response.data) - result["provenance"] = _provenance("classification", response, payload, client) - return result - - -def _trajectory_only_record( - record: dict[str, Any], trajectory: list[dict[str, Any]] -) -> dict[str, Any]: - """Return the fields allowed to influence trajectory-only evidence.""" - - return {"trajectory": trajectory, "tools": record.get("tools") or []} - - -def prepare_boundary_payload(record: dict[str, Any]) -> dict[str, Any]: - """Build an outcome-blind, reference-patch-blind boundary payload.""" - - raw_trajectory = record.get("trajectory") or [] - indexed = index_trajectory(record) - signals = extract_static_signals(_trajectory_only_record(record, raw_trajectory)) - blocks, compaction = build_trajectory_blocks(indexed, signals) - return { - "sample_id": get_sample_id(record), - "user_request": compact_text(_first_user_issue(indexed), 16_000), - "allowed_tool_names": signals["allowed_tool_names"], - "trajectory_blocks": blocks, - "final_model_patch": compact_patch_object( - (record.get("metadata") or {}).get("model_patch") or {} - ), - "prompt_compaction": compaction, - } - - -def prepare_prefix_quality_payload( - record: dict[str, Any], prefix: list[dict[str, Any]] -) -> dict[str, Any]: - """Build a quality payload from the materialized prefix and nothing later.""" - - prefix_record = _trajectory_only_record(record, prefix) - indexed = index_trajectory(prefix_record) - signals = extract_static_signals(prefix_record) - trajectory, compaction = compact_trajectory(indexed, signals) - assistant_turns = sum(message.get("role") == "assistant" for message in prefix) - payload = { - "sample_id": get_sample_id(record), - "user_request": compact_text(_first_user_issue(indexed), 16_000), - "allowed_tool_names": signals["allowed_tool_names"], - "trajectory": trajectory, - "prefix_metadata": { - "turn_count": len(prefix), - "assistant_turn_count": assistant_turns, - "ends_with_role": prefix[-1].get("role") if prefix else None, - "trajectory_sha256": _stable_hash({"trajectory": prefix}), - }, - "prompt_compaction": compaction, - } - if len(prefix) == len(record.get("trajectory") or []): - payload["final_model_patch"] = compact_patch_object( - (record.get("metadata") or {}).get("model_patch") or {} - ) - return payload - - -def audit_trajectory(record: dict[str, Any], client: GLMClient) -> dict[str, Any]: - """Locate a boundary, materialize its prefix, then score only that prefix.""" - - sample_id = get_sample_id(record) - resolved = int(record.get("resolved", -1)) - source_trajectory = record.get("trajectory") or [] - boundary_result: dict[str, Any] | None = None - boundary_provenance: dict[str, Any] | None = None - - if resolved == 1: - prefix = materialize_prefix(record, None) - evaluation_mode = "FULL_TRAJECTORY" - effective_boundary = None - else: - boundary_payload = prepare_boundary_payload(record) - boundary_response = client.invoke_json( - system_prompt=load_prompt("boundary.md"), - payload=boundary_payload, - schema=load_schema("boundary_output.schema.json"), - ) - validate_boundary(record, boundary_response.data) - boundary_result = dict(boundary_response.data) - boundary = boundary_result["truncate_before_turn"] - boundary_result["candidate_block_id"] = next( - ( - block["block_id"] - for block in boundary_payload["trajectory_blocks"] - if boundary is not None - and block["start_turn"] <= boundary <= block["end_turn"] - ), - None, - ) - boundary_result["grounded_evidence"] = materialize_evidence( - source_trajectory, boundary_result["evidence_turns"] - ) - effective_boundary = effective_boundary_policy(record, boundary_result) - boundary_provenance = _provenance( - "boundary", boundary_response, boundary_payload, client - ) - evaluation_mode = "PROCESS_PREFIX" - if effective_boundary["decision"] == "HOLD": - return { - "sample_id": sample_id, - "pipeline_version": "2.0", - "evaluation_mode": evaluation_mode, - "boundary": {"model": boundary_result, "effective": effective_boundary}, - "prefix": None, - "quality": None, - "recommended_use": "HOLD", - "source_record": _audit_source_record(record), - "provenance": {"boundary": boundary_provenance, "quality": None}, - } - prefix = materialize_prefix(record, effective_boundary["truncate_before_turn"]) - - quality_payload = prepare_prefix_quality_payload(record, prefix) - quality_response = client.invoke_json( - system_prompt=load_prompt("prefix_quality.md"), - payload=quality_payload, - schema=load_schema("prefix_quality_output.schema.json"), - ) - validate_prefix_quality(sample_id, prefix, quality_response.data) - quality = dict(quality_response.data) - quality["grounded_evidence"] = materialize_evidence( - prefix, quality["evidence_turns"] - ) - quality["local_safety"] = derive_prefix_safety(quality) - quality["local_score"] = compute_prefix_quality(quality) - tier = quality["local_score"]["quality_tier"] - if tier == "REJECT": - recommended_use = "REJECT" - elif resolved == 1: - recommended_use = "FULL_TRAJECTORY_CANDIDATE" - elif resolved == 0 and effective_boundary["decision"] == "KEEP_FULL": - recommended_use = "HOLD" - else: - recommended_use = "PROCESS_PREFIX_CANDIDATE" - - boundary_turn = effective_boundary["truncate_before_turn"] if effective_boundary else None - return { - "sample_id": sample_id, - "pipeline_version": "2.0", - "evaluation_mode": evaluation_mode, - "boundary": ( - {"model": boundary_result, "effective": effective_boundary} - if boundary_result is not None - else None - ), - "prefix": { - "truncate_before_turn": boundary_turn, - "source_turn_count": len(source_trajectory), - "retained_turn_count": len(prefix), - "retained_assistant_turn_count": sum( - message.get("role") == "assistant" for message in prefix - ), - "ends_with_role": prefix[-1].get("role") if prefix else None, - "trajectory_sha256": quality_payload["prefix_metadata"]["trajectory_sha256"], - }, - "quality": quality, - "recommended_use": recommended_use, - "source_record": _audit_source_record(record), - "provenance": { - "boundary": boundary_provenance, - "quality": _provenance("prefix_quality", quality_response, quality_payload, client), - }, - } - - -def _audit_source_record(record: dict[str, Any]) -> dict[str, Any]: - """Return compact immutable routing metadata for the final manifest.""" - - return { - "resolved": record.get("resolved"), - "instance_id": record.get("instance_id"), - "repo": record.get("repo"), - "language": record.get("language"), - "sample_provenance": record.get("_sample"), - } - - -def plan_static_repair( - record: dict[str, Any], - classification: dict[str, Any], - client: GLMClient, -) -> dict[str, Any]: - """Produce a fact-preserving repair plan without applying changes.""" - - payload = { - "sample_id": get_sample_id(record), - "original_record": prepare_classification_payload(record), - "classification": classification, - } - response = client.invoke_json( - system_prompt=load_prompt("repair.md"), - payload=payload, - schema=load_schema("repair_output.schema.json"), - ) - enforce_repair_policy(response.data) - result = dict(response.data) - result["provenance"] = _provenance("repair_plan", response, payload, client) - return result - - -def review_repaired_record( - *, - original_record: dict[str, Any], - classification: dict[str, Any], - repair_plan: dict[str, Any], - repaired_record: dict[str, Any], - structured_diff: dict[str, Any], - client: GLMClient, -) -> dict[str, Any]: - """Independently review a repaired record against immutable evidence.""" - - payload = { - "sample_id": get_sample_id(original_record), - "original_record": original_record, - "classification": classification, - "repair_plan": repair_plan, - "repaired_record": repaired_record, - "structured_diff": structured_diff, - } - response = client.invoke_json( - system_prompt=load_prompt("review.md"), - payload=payload, - schema=load_schema("review_output.schema.json"), - ) - result = dict(response.data) - result["provenance"] = _provenance("repair_review", response, payload, client) - return result diff --git a/tests/test_audit.py b/tests/test_audit.py deleted file mode 100644 index 8959ad5..0000000 --- a/tests/test_audit.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Tests for the two-call boundary and prefix-quality policy.""" - -from __future__ import annotations - -import json - -import pytest - -from swe_data_processing.audit import ( - compute_prefix_quality, - derive_prefix_safety, - effective_boundary_policy, - materialize_prefix, - validate_boundary, - validate_prefix_quality, -) -from swe_data_processing.policy import PolicyViolation -from swe_data_processing.workflow import ( - prepare_boundary_payload, - prepare_prefix_quality_payload, -) - - -def _record() -> dict: - return { - "trajectory_id": "sample-1", - "resolved": 0, - "tools": [], - "trajectory": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "Fix the bug."}, - {"role": "assistant", "content": "I will inspect the code."}, - {"role": "tool", "content": "relevant.py"}, - {"role": "assistant", "content": "I will apply the harmful patch."}, - {"role": "tool", "content": "tests failed"}, - ], - } - - -def _boundary() -> dict: - return { - "sample_id": "sample-1", - "checks": { - "task_coverage": "COMPLETE", - "final_patch_scope": "CLEAN", - "constraints": "RESPECTED", - "claims_vs_observations": "CONSISTENT", - }, - "decision": "TRUNCATE", - "truncate_before_turn": 5, - "category": "PERSISTENT_WRONG_IMPLEMENTATION", - "severity": "MAJOR", - "state_effect": "UNRECOVERED", - "evidence_turns": [5], - "reason": "The patch fails and is not repaired.", - } - - -def _quality() -> dict: - return { - "sample_id": "sample-1", - "behavior_issues": [], - "dimensions": { - "planning": 14, - "investigation": 15, - "tool_use_and_observation": 14, - "progress": 12, - "clarity_and_efficiency": 10, - }, - "evidence_turns": [], - "reason": "Useful investigation.", - } - - -def test_valid_boundary_materializes_exact_prefix() -> None: - validate_boundary(_record(), _boundary()) - prefix = materialize_prefix(_record(), 5) - assert prefix == _record()["trajectory"][:4] - - -def test_boundary_must_be_an_assistant_turn() -> None: - result = _boundary() - result["truncate_before_turn"] = 4 - with pytest.raises(PolicyViolation, match="assistant turn"): - validate_boundary(_record(), result) - - -def test_boundary_evidence_must_include_boundary_turn() -> None: - result = _boundary() - result["evidence_turns"] = [6] - with pytest.raises(PolicyViolation, match="excluded assistant turn"): - validate_boundary(_record(), result) - - -def test_keep_full_cannot_contain_a_boundary() -> None: - result = _boundary() - result.update( - decision="KEEP_FULL", - truncate_before_turn=None, - category="NONE", - severity="NONE", - state_effect="NONE", - evidence_turns=[], - ) - validate_boundary(_record(), result) - result["truncate_before_turn"] = 5 - with pytest.raises(PolicyViolation, match="cannot contain a boundary"): - validate_boundary(_record(), result) - - -def test_prefix_quality_cannot_reference_suffix() -> None: - prefix = materialize_prefix(_record(), 5) - result = _quality() - result["evidence_turns"] = [5] - with pytest.raises(PolicyViolation, match="outside the prefix"): - validate_prefix_quality("sample-1", prefix, result) - - -def test_quality_score_and_tier_are_computed_locally() -> None: - score = compute_prefix_quality(_quality()) - assert score["educational_quality_score"] == 65 - assert score["quality_tier"] == "MEDIUM" - - -def test_unrecovered_major_issue_is_locally_invalid() -> None: - result = _quality() - result["behavior_issues"] = [ - { - "turn_id": 3, - "kind": "ERROR", - "severity": "MAJOR", - "recovered": False, - "reason": "The prefix leaves a known broken edit.", - } - ] - result["evidence_turns"] = [3] - validate_prefix_quality("sample-1", materialize_prefix(_record(), 5), result) - assert derive_prefix_safety(result)["prefix_valid"] is False - assert compute_prefix_quality(result)["quality_tier"] == "REJECT" - - -@pytest.mark.parametrize("resolved", [0, -1]) -def test_non_success_outcome_is_not_capped_at_first_stateful_turn(resolved: int) -> None: - record = _record() - record["resolved"] = resolved - record["tools"] = [ - {"type": "function", "function": {"name": "str_replace_editor"}} - ] - record["trajectory"][2]["tool_calls"] = [ - { - "id": "call-1", - "type": "function", - "function": { - "name": "str_replace_editor", - "arguments": '{"command":"str_replace","path":"src/a.py"}', - }, - } - ] - result = _boundary() - result.update(decision="KEEP_FULL", truncate_before_turn=None) - policy = effective_boundary_policy(record, result) - assert policy["decision"] == "KEEP_FULL" - assert policy["truncate_before_turn"] is None - assert policy["source"] == "MODEL_DECISION" - assert policy["first_stateful_turn"] == 3 - - -def test_semantic_boundary_is_not_replaced_by_earlier_stateful_turn() -> None: - record = _record() - record["resolved"] = 0 - record["tools"] = [ - {"type": "function", "function": {"name": "str_replace_editor"}} - ] - record["trajectory"][2]["tool_calls"] = [ - { - "id": "call-1", - "type": "function", - "function": { - "name": "str_replace_editor", - "arguments": '{"command":"str_replace","path":"src/a.py"}', - }, - } - ] - result = _boundary() - policy = effective_boundary_policy(record, result) - assert policy["decision"] == "TRUNCATE" - assert policy["truncate_before_turn"] == 5 - assert policy["source"] == "MODEL_BOUNDARY" - assert policy["first_stateful_turn"] == 3 - - -def test_boundary_payload_hides_outcome_and_patch_metadata() -> None: - record = _record() - record["resolved"] = -1 - record["metadata"] = { - "model_patch": {"patch": "MODEL_PATCH_SECRET"}, - "reference_patch": {"patch": "REFERENCE_PATCH_SECRET"}, - } - serialized = json.dumps(prepare_boundary_payload(record), ensure_ascii=False) - assert "resolved" not in serialized - assert "MODEL_PATCH_SECRET" in serialized - assert "REFERENCE_PATCH_SECRET" not in serialized - - -def test_prefix_payload_is_identical_when_only_suffix_and_labels_change() -> None: - first = _record() - second = _record() - second["resolved"] = -1 - second["trajectory"][4]["content"] = "DIFFERENT SUFFIX" - second["trajectory"][5]["content"] = "DIFFERENT TOOL RESULT" - second["metadata"] = {"model_patch": {"patch": "DIFFERENT PATCH"}} - first_prefix = materialize_prefix(first, 5) - second_prefix = materialize_prefix(second, 5) - assert prepare_prefix_quality_payload( - first, first_prefix - ) == prepare_prefix_quality_payload(second, second_prefix) diff --git a/tests/test_audit_workflow.py b/tests/test_audit_workflow.py deleted file mode 100644 index 0e0fdc4..0000000 --- a/tests/test_audit_workflow.py +++ /dev/null @@ -1,165 +0,0 @@ -"""End-to-end tests for call isolation in the audit workflow.""" - -from __future__ import annotations - -import json - -from swe_data_processing.client import GLMResponse -from swe_data_processing.config import Settings -from swe_data_processing.workflow import audit_trajectory - - -class FakeClient: - """Return deterministic responses while recording every API payload.""" - - def __init__(self, responses: list[dict]) -> None: - self.settings = Settings(api_key="test-secret") - self.responses = iter(responses) - self.calls: list[dict] = [] - - def invoke_json(self, *, system_prompt: str, payload: dict, schema: dict) -> GLMResponse: - self.calls.append( - {"system_prompt": system_prompt, "payload": payload, "schema": schema} - ) - return GLMResponse( - data=next(self.responses), - request_id=f"request-{len(self.calls)}", - usage={"total_tokens": 10}, - compatibility_fallback_used=False, - ) - - -def _record() -> dict: - return { - "trajectory_id": "sample-1", - "resolved": 0, - "trajectory": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "Fix the bug."}, - {"role": "assistant", "content": "Inspect relevant.py"}, - {"role": "tool", "content": "relevant code"}, - {"role": "assistant", "content": "BAD_SUFFIX_SENTINEL patch"}, - {"role": "tool", "content": "BAD_SUFFIX_SENTINEL failed"}, - ], - } - - -def test_workflow_calls_boundary_then_scores_only_materialized_prefix() -> None: - client = FakeClient( - [ - { - "sample_id": "sample-1", - "checks": { - "task_coverage": "INCOMPLETE", - "final_patch_scope": "POLLUTED", - "constraints": "RESPECTED", - "claims_vs_observations": "CONTRADICTED", - }, - "decision": "TRUNCATE", - "truncate_before_turn": 5, - "category": "PERSISTENT_WRONG_IMPLEMENTATION", - "severity": "MAJOR", - "state_effect": "UNRECOVERED", - "evidence_turns": [5], - "reason": "The patch is not repaired.", - }, - { - "sample_id": "sample-1", - "behavior_issues": [], - "dimensions": { - "planning": 15, - "investigation": 15, - "tool_use_and_observation": 14, - "progress": 12, - "clarity_and_efficiency": 10, - }, - "evidence_turns": [], - "reason": "The prefix is useful.", - }, - ] - ) - result = audit_trajectory(_record(), client) # type: ignore[arg-type] - - assert len(client.calls) == 2 - quality_payload = json.dumps(client.calls[1]["payload"], ensure_ascii=False) - assert "BAD_SUFFIX_SENTINEL" not in quality_payload - assert "truncate_before_turn" not in quality_payload - assert result["prefix"]["retained_turn_count"] == 4 - assert result["quality"]["local_score"]["quality_tier"] == "MEDIUM" - assert result["recommended_use"] == "PROCESS_PREFIX_CANDIDATE" - - -def test_hold_boundary_skips_quality_call() -> None: - client = FakeClient( - [ - { - "sample_id": "sample-1", - "checks": { - "task_coverage": "COMPLETE", - "final_patch_scope": "CLEAN", - "constraints": "RESPECTED", - "claims_vs_observations": "CONSISTENT", - }, - "decision": "HOLD", - "truncate_before_turn": None, - "category": "NONE", - "severity": "NONE", - "state_effect": "UNCLEAR", - "evidence_turns": [], - "reason": "No clear unrepaired defect.", - } - ] - ) - result = audit_trajectory(_record(), client) # type: ignore[arg-type] - assert len(client.calls) == 1 - assert result["recommended_use"] == "HOLD" - assert result["quality"] is None - - -def test_keep_full_process_trajectory_is_scored() -> None: - client = FakeClient( - [ - { - "sample_id": "sample-1", - "checks": { - "task_coverage": "COMPLETE", - "final_patch_scope": "CLEAN", - "constraints": "RESPECTED", - "claims_vs_observations": "CONSISTENT", - }, - "decision": "KEEP_FULL", - "truncate_before_turn": None, - "category": "NONE", - "severity": "NONE", - "state_effect": "NONE", - "evidence_turns": [], - "reason": "No unrepaired severe problem is visible.", - }, - { - "sample_id": "sample-1", - "behavior_issues": [ - { - "turn_id": 3, - "kind": "INEFFICIENCY", - "severity": "MINOR", - "recovered": True, - "reason": "The inspection was somewhat broad.", - } - ], - "dimensions": { - "planning": 15, - "investigation": 15, - "tool_use_and_observation": 14, - "progress": 12, - "clarity_and_efficiency": 10, - }, - "evidence_turns": [], - "reason": "Useful despite minor inefficiency.", - }, - ] - ) - result = audit_trajectory(_record(), client) # type: ignore[arg-type] - assert len(client.calls) == 2 - assert result["prefix"]["retained_turn_count"] == 6 - assert result["quality"]["local_score"]["issue_counts"]["inefficiencies"] == 1 - assert result["recommended_use"] == "HOLD" diff --git a/tests/test_cli.py b/tests/test_cli.py index dfa3db1..93f184a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,54 +1,38 @@ -"""Tests for the streaming CLI runner.""" +"""Tests for CLI parsing and streaming profile execution.""" from __future__ import annotations import json -import threading -import time from pathlib import Path -from swe_data_processing.cli import _run_streaming_stage +from swe_data_processing.cli import _run_streaming_stage, build_parser -def test_streaming_stage_processes_records_concurrently(tmp_path: Path) -> None: - """Multiple workers run processors in parallel while one thread writes JSONL.""" - +def test_streaming_stage_writes_each_record(tmp_path: Path) -> None: input_path = tmp_path / "input.jsonl" output_path = tmp_path / "output.jsonl" input_path.write_text( - "".join( - json.dumps({"trajectory_id": f"sample-{index}"}) + "\n" for index in range(20) - ), + "".join(json.dumps({"trajectory_id": f"s-{i}"}) + "\n" for i in range(10)), encoding="utf-8", ) - lock = threading.Lock() - active = 0 - peak_active = 0 - - def processor(record: dict) -> dict: - nonlocal active, peak_active - with lock: - active += 1 - peak_active = max(peak_active, active) - time.sleep(0.01) - with lock: - active -= 1 - return {"sample_id": record["trajectory_id"]} - status = _run_streaming_stage( input_path=input_path, output_path=output_path, errors_path=None, limit=None, resume=False, - workers=4, - stage_name="test", - processor=processor, + workers=3, + processor=lambda record: {"sample_id": record["trajectory_id"]}, ) - - written = [json.loads(line) for line in output_path.read_text().splitlines()] assert status == 0 - assert peak_active == 4 - assert {value["sample_id"] for value in written} == { - f"sample-{index}" for index in range(20) - } + assert len(output_path.read_text().splitlines()) == 10 + + +def test_cli_has_only_deterministic_pipeline_commands() -> None: + parser = build_parser() + assert parser.parse_args( + ["profile", "--input", "in.jsonl", "--output", "out.jsonl"] + ).command == "profile" + help_text = parser.format_help() + assert "classify" not in help_text + assert "repair" not in help_text diff --git a/tests/test_client.py b/tests/test_client.py deleted file mode 100644 index e691ce3..0000000 --- a/tests/test_client.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Tests for API request compatibility and structured-response validation.""" - -from __future__ import annotations - -import json - -import httpx -import pytest - -from swe_data_processing.client import GLMClient -from swe_data_processing.config import Settings - - -def test_client_parses_fenced_json_and_never_sends_key_in_body() -> None: - captured_body = {} - - def handler(request: httpx.Request) -> httpx.Response: - captured_body.update(json.loads(request.content)) - assert request.headers["authorization"] == "Bearer test-secret" - return httpx.Response( - 200, - headers={"x-request-id": "request-123"}, - json={ - "choices": [ - {"message": {"role": "assistant", "content": "```json\n{\"status\":\"ok\"}\n```"}} - ], - "usage": {"total_tokens": 10}, - }, - ) - - schema = { - "type": "object", - "additionalProperties": False, - "required": ["status"], - "properties": {"status": {"const": "ok"}}, - } - settings = Settings(api_key="test-secret", max_retries=0) - with GLMClient(settings, transport=httpx.MockTransport(handler)) as client: - response = client.invoke_json(system_prompt="Return JSON.", payload={"ping": True}, schema=schema) - - assert response.data == {"status": "ok"} - assert response.request_id == "request-123" - assert response.usage == {"total_tokens": 10} - assert "test-secret" not in json.dumps(captured_body) - assert captured_body["model"] == "glm-5.2" - - -def test_client_falls_back_when_vendor_extensions_are_rejected() -> None: - calls = [] - - def handler(request: httpx.Request) -> httpx.Response: - body = json.loads(request.content) - calls.append(body) - if len(calls) == 1: - return httpx.Response(400, json={"error": {"message": "unknown parameter: thinking"}}) - return httpx.Response( - 200, - json={"choices": [{"message": {"content": "{\"status\":\"ok\"}"}}]}, - ) - - schema = { - "type": "object", - "required": ["status"], - "properties": {"status": {"const": "ok"}}, - } - settings = Settings(api_key="test-secret", max_retries=1) - with GLMClient(settings, transport=httpx.MockTransport(handler)) as client: - response = client.invoke_json(system_prompt="Return JSON.", payload={}, schema=schema) - - assert "thinking" in calls[0] - assert "thinking" not in calls[1] - assert response.compatibility_fallback_used is True - - -def test_client_retries_five_failures_before_succeeding( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Five retryable failures are followed by a sixth and final attempt.""" - - calls = 0 - - def handler(_: httpx.Request) -> httpx.Response: - nonlocal calls - calls += 1 - if calls <= 5: - return httpx.Response(503, json={"error": {"message": "temporarily unavailable"}}) - return httpx.Response( - 200, - json={"choices": [{"message": {"content": "{\"status\":\"ok\"}"}}]}, - ) - - monkeypatch.setattr("swe_data_processing.client.time.sleep", lambda _: None) - schema = { - "type": "object", - "required": ["status"], - "properties": {"status": {"const": "ok"}}, - } - settings = Settings(api_key="test-secret", max_retries=5) - with GLMClient(settings, transport=httpx.MockTransport(handler)) as client: - response = client.invoke_json(system_prompt="Return JSON.", payload={}, schema=schema) - - assert calls == 6 - assert response.data == {"status": "ok"} diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index f0762a1..0000000 --- a/tests/test_config.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Tests for strict environment configuration and endpoint normalization.""" - -from __future__ import annotations - -import pytest - -from swe_data_processing.config import Settings - - -def test_endpoint_normalization() -> None: - settings = Settings( - api_key="test-secret", - api_base="https://llm-api.cowin.run/", - api_path="v1/chat/completions", - ) - assert settings.endpoint == "https://llm-api.cowin.run/v1/chat/completions" - - -def test_missing_api_key_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("GLM_API_KEY", raising=False) - with pytest.raises(ValueError, match="GLM_API_KEY"): - Settings.from_env() - - -def test_feature_only_settings_can_omit_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("GLM_API_KEY", raising=False) - settings = Settings.from_env(require_api_key=False) - assert settings.api_key == "" - assert settings.max_retries == 5 diff --git a/tests/test_evidence.py b/tests/test_evidence.py deleted file mode 100644 index 3e86eba..0000000 --- a/tests/test_evidence.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Tests for prompt-only trajectory compaction.""" - -from swe_data_processing.evidence import ( - BOUNDARY_BLOCK_TURNS, - build_trajectory_blocks, - compact_text, - compact_trajectory, -) - - -def test_compact_text_preserves_short_values() -> None: - """Short evidence must remain byte-for-byte identical.""" - - assert compact_text("short evidence", 100) == "short evidence" - - -def test_compact_text_hashes_long_values() -> None: - """Long evidence must expose its size and digest for auditability.""" - - result = compact_text("a" * 1_000, 200) - assert len(result) == 200 - assert "COMPACTED original_chars=1000 sha256=" in result - - -def test_compact_trajectory_prioritizes_test_observations() -> None: - """Important test output receives a larger preview than ordinary output.""" - - long_content = "x" * 3_000 - trajectory = [ - {"turn_id": 1, "role": "user", "content": "issue"}, - {"turn_id": 2, "role": "assistant", "content": "", "tool_calls": []}, - {"turn_id": 3, "role": "tool", "content": long_content}, - {"turn_id": 4, "role": "assistant", "content": "done"}, - ] - signals = { - "stateful_turns": [], - "test_events": [{"command_turn": 2, "output_turn": 3}], - "malformed_tool_turns": [], - "unknown_tool_turns": [], - } - compacted, metadata = compact_trajectory(trajectory, signals) - assert compacted[2]["content"] == long_content - assert 3 in metadata["important_turn_ids"] - - -def test_boundary_blocks_preserve_absolute_turns_without_overlap() -> None: - trajectory = [ - {"turn_id": turn, "role": "assistant", "content": f"turn {turn}"} - for turn in range(1, BOUNDARY_BLOCK_TURNS + 3) - ] - blocks, metadata = build_trajectory_blocks(trajectory, {}) - assert len(blocks) == 2 - assert blocks[0]["start_turn"] == 1 - assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS - assert blocks[1]["start_turn"] == BOUNDARY_BLOCK_TURNS + 1 - assert blocks[1]["end_turn"] == BOUNDARY_BLOCK_TURNS + 2 - turn_ids = [ - message["turn_id"] for block in blocks for message in block["messages"] - ] - assert turn_ids == list(range(1, BOUNDARY_BLOCK_TURNS + 3)) - assert metadata["block_count"] == 2 - - -def test_boundary_block_keeps_immediate_tool_result_with_assistant() -> None: - trajectory = [ - {"turn_id": turn, "role": "user", "content": f"turn {turn}"} - for turn in range(1, BOUNDARY_BLOCK_TURNS + 2) - ] - trajectory[BOUNDARY_BLOCK_TURNS - 1]["role"] = "assistant" - trajectory[BOUNDARY_BLOCK_TURNS]["role"] = "tool" - blocks, _ = build_trajectory_blocks(trajectory, {}) - assert blocks[0]["end_turn"] == BOUNDARY_BLOCK_TURNS + 1 - assert len(blocks) == 1 diff --git a/tests/test_features.py b/tests/test_features.py deleted file mode 100644 index b844fb9..0000000 --- a/tests/test_features.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Tests for deterministic static trajectory evidence extraction.""" - -from __future__ import annotations - -import json - -from swe_data_processing.features import extract_patch_files, extract_static_signals - - -def _tool(name: str) -> str: - return json.dumps({"type": "function", "function": {"name": name, "parameters": {}}}) - - -def test_detects_masked_failing_test_after_edit() -> None: - record = { - "trajectory_id": "sample-1", - "resolved": 0, - "tools": [_tool("str_replace_editor"), _tool("execute_bash"), _tool("finish")], - "trajectory": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "Do not modify test files."}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "edit-1", - "type": "function", - "function": { - "name": "str_replace_editor", - "arguments": json.dumps( - {"command": "str_replace", "path": "/workspace/src/main.py"} - ), - }, - } - ], - }, - {"role": "tool", "content": "The file was edited."}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "test-1", - "type": "function", - "function": { - "name": "execute_bash", - "arguments": json.dumps({"command": "pytest tests | tail -20"}), - }, - } - ], - }, - { - "role": "tool", - "content": "1 failed, 10 passed\n[Command finished with exit code 0]", - }, - ], - "metadata": { - "model_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"}, - "reference_patch": {"patch": "diff --git a/src/main.py b/src/main.py\n+++ b/src/main.py\n"}, - }, - } - - signals = extract_static_signals(record) - assert signals["last_stateful_turn"] == 3 - assert signals["instruction_constraints"] == ["Do not modify test files."] - assert signals["test_events"][0]["masked_pipeline"] is True - assert signals["test_events"][0]["explicit_failure"] is True - assert signals["test_events"][0]["explicit_exit_codes"] == [0] - - -def test_detects_malformed_and_unknown_tool_calls() -> None: - record = { - "trajectory_id": "sample-2", - "tools": [_tool("execute_bash")], - "trajectory": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "issue"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "bad", - "function": {"name": "made_up_tool", "arguments": "{bad json"}, - } - ], - }, - {"role": "tool", "content": "error"}, - ], - "metadata": {"model_patch": {}, "reference_patch": {}}, - } - signals = extract_static_signals(record) - assert signals["malformed_tool_turns"][0]["turn_id"] == 3 - assert signals["unknown_tool_turns"] == [{"turn_id": 3, "tool_name": "made_up_tool"}] - - -def test_extract_patch_files_is_stable_and_unique() -> None: - patch = "+++ b/src/a.py\n+++ b/src/b.py\n+++ b/src/a.py\n+++ /dev/null\n" - assert extract_patch_files(patch) == ["src/a.py", "src/b.py"] diff --git a/tests/test_heuristics.py b/tests/test_heuristics.py new file mode 100644 index 0000000..af5f08e --- /dev/null +++ b/tests/test_heuristics.py @@ -0,0 +1,84 @@ +"""Tests for transparent filtering rules.""" + +from __future__ import annotations + +from swe_data_processing.heuristics import classify_metrics, length_bucket + + +def _metrics(*, failed: int = 0, calls: int = 10, streak: int = 0, tokens=50_000): + return { + "sample_id": "sample-1", + "instance_id": "repo-1", + "source_group": "example_group", + "source_parquet": "/data/example.parquet", + "resolved": 1, + "length": {"turn_count": 30, "token_count": tokens}, + "tools": { + "tool_call_count": calls, + "failed_tool_call_count": failed, + "failed_tool_call_rate": failed / calls if calls else 0.0, + "longest_consecutive_failure_run": streak, + "error_positions": { + "early_count": failed, + "early_fraction": 1.0 if failed else 0.0, + "occupied_bins_5": min(failed, 5), + }, + "error_type_counts": {}, + "error_tool_counts": {}, + }, + "structure": { + "invalid_turn_count": 0, + "malformed_tool_definition_count": 0, + "malformed_tool_call_count": 0, + "unknown_tool_call_count": 0, + "missing_tool_result_count": 0, + "orphan_tool_result_count": 0, + }, + } + + +def test_length_buckets_match_training_limits() -> None: + assert length_bucket(None) == "TOKENIZER_REQUIRED" + assert length_bucket(81_920) == "LE_81920" + assert length_bucket(81_921) == "81921_TO_131072" + assert length_bucket(131_073) == "131073_TO_262144" + assert length_bucket(262_145) == "GT_262144" + + +def test_five_consecutive_failures_is_hard_reject() -> None: + decision = classify_metrics(_metrics(failed=5, calls=20, streak=5), 10) + assert "FIVE_CONSECUTIVE_TOOL_FAILURES" in decision["hard_reject_reasons"] + assert decision["recommended_action"] == "DROP_DEFINITE_TOOL_PROBLEM" + + +def test_early_cluster_is_review_only() -> None: + decision = classify_metrics(_metrics(failed=3, calls=20, streak=2), 10) + assert decision["hard_reject_reasons"] == [] + assert decision["review_flags"] == ["EARLY_FAILURE_CLUSTER"] + assert decision["recommended_action"] == "REVIEW_HEURISTIC_HIT" + + +def test_relative_count_outlier_is_not_automatic_drop() -> None: + decision = classify_metrics(_metrics(failed=10, calls=100, streak=2), 10) + assert decision["hard_reject_reasons"] == [] + assert "EXTREME_ERROR_COUNT" in decision["review_flags"] + + +def test_distributed_failures_require_review_instead_of_automatic_drop() -> None: + decision = classify_metrics(_metrics(failed=8, calls=40, streak=2), 20) + assert decision["hard_reject_reasons"] == [] + assert "PERSISTENT_DISTRIBUTED_FAILURES" in decision["review_flags"] + + +def test_failed_outcome_is_never_recommended_for_training() -> None: + metrics = _metrics() + metrics["resolved"] = 0 + decision = classify_metrics(metrics, 10) + assert decision["outcome_use"] == "EXCLUDE_FROM_SUCCESS_SFT" + assert decision["recommended_action"] == "EXCLUDE_FAILED_OUTCOME" + + +def test_decision_keeps_source_provenance_for_targeted_review() -> None: + decision = classify_metrics(_metrics(), 10) + assert decision["source_group"] == "example_group" + assert decision["source_parquet"] == "/data/example.parquet" diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..4695d7f --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,123 @@ +"""Tests for deterministic trajectory metrics.""" + +from __future__ import annotations + +import json + +from swe_data_processing.metrics import canonical_training_text, extract_metrics + + +def _record(outputs: list[str]) -> dict: + trajectory = [{"role": "user", "content": "Fix the bug."}] + for index, output in enumerate(outputs, 1): + trajectory.extend( + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": f"call-{index}", + "type": "function", + "function": { + "name": "bash", + "arguments": json.dumps({"command": "pytest tests"}), + }, + } + ], + }, + {"role": "tool", "content": output}, + ] + ) + return { + "trajectory_id": "sample-1", + "instance_id": "repo-1", + "resolved": 1, + "tools": [{"type": "function", "function": {"name": "bash"}}], + "trajectory": trajectory, + } + + +def test_canonical_text_is_stable_and_excludes_metadata() -> None: + record = _record(["exit code 0"]) + record["metadata"] = {"reference_patch": {"patch": "SECRET"}} + text = canonical_training_text(record) + assert "SECRET" not in text + assert text == canonical_training_text(record) + + +def test_profile_keeps_source_parquet_out_of_canonical_text() -> None: + record = _record(["exit code 0"]) + record["_source_parquet"] = "/data/openhands/train.parquet" + metrics = extract_metrics(record) + assert metrics["source_group"] == "openhands" + assert "/data/openhands" not in canonical_training_text(record) + + +def test_extracts_failure_count_type_distribution_and_streak() -> None: + record = _record( + [ + "2 failed\n[The command completed with exit code 1.]", + "permission denied\nexit status 2", + "tests passed\nexit code 0", + "timeout while waiting", + ] + ) + metrics = extract_metrics(record) + tools = metrics["tools"] + assert tools["tool_call_count"] == 4 + assert tools["failed_tool_call_count"] == 3 + assert tools["longest_consecutive_failure_run"] == 2 + assert tools["error_type_counts"]["nonzero_exit"] == 2 + assert tools["error_type_counts"]["test_failure"] == 1 + assert tools["error_type_counts"]["permission_denied"] == 1 + assert tools["error_type_counts"]["timeout"] == 1 + assert tools["error_positions"]["bins_5"] == [1, 1, 0, 0, 1] + + +def test_explicit_success_suppresses_incidental_error_words() -> None: + record = _record(["Read fixture containing 'permission denied'.\nexit code 0"]) + assert extract_metrics(record)["tools"]["failed_tool_call_count"] == 0 + + +def test_detects_broken_tool_structure() -> None: + record = _record(["exit code 0"]) + call = record["trajectory"][1]["tool_calls"][0] + call["function"]["arguments"] = "{broken" + record["trajectory"].pop() + metrics = extract_metrics(record) + assert metrics["structure"]["malformed_tool_call_count"] == 1 + assert metrics["structure"]["missing_tool_result_count"] == 1 + assert metrics["tools"]["failed_tool_call_count"] == 1 + + +def test_terminal_finish_without_result_is_valid() -> None: + record = _record([]) + record["trajectory"].append( + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "function": {"name": "finish", "arguments": "{}"}, + "id": "finish-1", + "type": "function", + } + ], + } + ) + record["tools"].append({"type": "function", "function": {"name": "finish"}}) + metrics = extract_metrics(record) + assert metrics["structure"]["missing_tool_result_count"] == 0 + assert metrics["tools"]["failed_tool_call_count"] == 0 + + +def test_editor_file_content_does_not_trigger_shell_patterns() -> None: + record = _record(["source code contains timeout and permission denied"]) + call = record["trajectory"][1]["tool_calls"][0] + call["function"]["name"] = "str_replace_editor" + call["function"]["arguments"] = '{"command":"view","path":"src/a.py"}' + record["tools"] = [ + {"type": "function", "function": {"name": "str_replace_editor"}} + ] + assert extract_metrics(record)["tools"]["failed_tool_call_count"] == 0 diff --git a/tests/test_policy.py b/tests/test_policy.py deleted file mode 100644 index c3624f1..0000000 --- a/tests/test_policy.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Tests for immutable classification and repair policy enforcement.""" - -from __future__ import annotations - -import pytest - -from swe_data_processing.policy import ( - PolicyViolation, - enforce_classification_policy, - enforce_repair_policy, -) - - -def _classification() -> dict: - dimensions = { - "trajectory_integrity": "PASS", - "tool_integrity": "PASS", - "patch_presence": "PASS", - "patch_trajectory_consistency": "PASS", - "instruction_compliance": "PASS", - "verification_consistency": "PASS", - "final_claim_alignment": "PASS", - "patch_hygiene": "PASS", - "issue_patch_alignment": "PASS", - } - return { - "source_outcome_class": "POSITIVE_CANDIDATE", - "qc_decision": "ACCEPT_SILVER_POSITIVE", - "training_use": "SFT_FULL", - "qc_passed": True, - "hard_fail_codes": [], - "verification": {"status": "PASS_RELIABLE"}, - "dimensions": dimensions, - } - - -def test_valid_silver_positive_passes() -> None: - enforce_classification_policy({"resolved": 1}, _classification()) - - -def test_resolved_zero_cannot_be_promoted() -> None: - result = _classification() - result["source_outcome_class"] = "EXPLICIT_NEGATIVE" - with pytest.raises(PolicyViolation, match="resolved=0"): - enforce_classification_policy({"resolved": 0}, result) - - -def test_unknown_cannot_be_full_sft() -> None: - result = _classification() - result["source_outcome_class"] = "UNVERIFIED" - result["qc_decision"] = "HOLD_UNVERIFIED" - result["qc_passed"] = False - with pytest.raises(PolicyViolation, match="resolved=-1"): - enforce_classification_policy({"resolved": -1}, result) - - -def _repair_result() -> dict: - """Return a minimal non-mutating repair result for policy tests.""" - - return { - "repair_decision": "NO_CHANGE", - "maximum_training_use": "HOLD", - "invariants": { - "resolved_unchanged": True, - "tool_outputs_unchanged": True, - "model_patch_unchanged": True, - "reference_patch_unchanged": True, - "no_synthetic_execution_result": True, - }, - "operations": [], - "requires_second_review": True, - } - - -def test_nonmutating_repair_decision_rejects_operations() -> None: - """A hold decision cannot smuggle in an operation the applier will ignore.""" - - result = _repair_result() - result["repair_decision"] = "REQUIRES_EXECUTION" - result["operations"] = [ - { - "op": "REWRITE_FINAL_SUMMARY", - "target_turns": [3], - "preconditions": ["Turn 3 is an assistant summary"], - "replacement": "Honest summary", - "reason": "Align the claim with evidence", - } - ] - with pytest.raises(PolicyViolation, match="must not contain"): - enforce_repair_policy(result) - - -def test_step_example_requires_operations() -> None: - """Step-only salvage must include a concrete correction and truncation.""" - - result = _repair_result() - result["repair_decision"] = "CREATE_STEP_EXAMPLE" - result["maximum_training_use"] = "SFT_STEP_ONLY" - with pytest.raises(PolicyViolation, match="requires at least one"): - enforce_repair_policy(result) diff --git a/tests/test_repair.py b/tests/test_repair.py deleted file mode 100644 index 19568a4..0000000 --- a/tests/test_repair.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Tests for deterministic static repair application and evidence preservation.""" - -from __future__ import annotations - -from swe_data_processing.repair import apply_static_repair - - -def _plan() -> dict: - return { - "sample_id": "sample-1", - "repair_decision": "APPLY_STATIC_REPAIR", - "maximum_training_use": "ERROR_ANALYSIS", - "invariants": { - "resolved_unchanged": True, - "tool_outputs_unchanged": True, - "model_patch_unchanged": True, - "reference_patch_unchanged": True, - "no_synthetic_execution_result": True, - }, - "operations": [ - { - "op": "REWRITE_FINAL_SUMMARY", - "target_turns": [5], - "preconditions": ["Turn 5 is an assistant-only summary."], - "replacement": "The build passed, but functional behavior remains unverified.", - "reason": "Remove an unsupported success claim.", - } - ], - "requires_second_review": True, - "summary": "Correct the final claim without changing execution evidence.", - } - - -def test_rewrite_summary_preserves_tool_output_and_patch() -> None: - record = { - "trajectory_id": "sample-1", - "resolved": -1, - "trajectory": [ - {"role": "system", "content": "system"}, - {"role": "user", "content": "issue"}, - {"role": "assistant", "content": "build", "tool_calls": []}, - {"role": "tool", "content": "build completed with exit code 0"}, - {"role": "assistant", "content": "All tests pass.", "tool_calls": []}, - ], - "metadata": { - "model_patch": {"patch": "+++ b/src/a.py\n"}, - "reference_patch": {"patch": "+++ b/src/a.py\n"}, - }, - } - repaired, diff = apply_static_repair(record, _plan()) - assert repaired["trajectory"][3] == record["trajectory"][3] - assert repaired["trajectory"][4]["content"].startswith("The build passed") - assert repaired["metadata"] == record["metadata"] - assert diff["replaced_original_turn_ids"] == [5] diff --git a/tests/test_sampling.py b/tests/test_sampling.py new file mode 100644 index 0000000..1d29cb1 --- /dev/null +++ b/tests/test_sampling.py @@ -0,0 +1,47 @@ +"""Tests for deterministic review sampling.""" + +from __future__ import annotations + +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +from swe_data_processing.sampling import iter_review_records + + +def test_review_reads_only_provenance_shards(tmp_path: Path) -> None: + selected_shard = tmp_path / "selected.parquet" + unrelated_shard = tmp_path / "unrelated.parquet" + pq.write_table( + pa.Table.from_pylist( + [ + {"trajectory_id": "wanted", "trajectory": "selected"}, + {"trajectory_id": "other", "trajectory": "same shard"}, + ] + ), + selected_shard, + ) + pq.write_table( + pa.Table.from_pylist( + [{"trajectory_id": "unrelated", "trajectory": "must not be read"}] + ), + unrelated_shard, + ) + selected = {"wanted": ["hard:FIVE_CONSECUTIVE_TOOL_FAILURES"]} + decision = { + "sample_id": "wanted", + "source_parquet": str(selected_shard), + "hard_reject_reasons": ["FIVE_CONSECUTIVE_TOOL_FAILURES"], + } + + records = list( + iter_review_records( + tmp_path, + selected, + {"wanted": decision}, + ) + ) + + assert [record["trajectory_id"] for record in records] == ["wanted"] + assert records[0]["_qc_review"]["decision"] == decision diff --git a/tests/test_summary.py b/tests/test_summary.py new file mode 100644 index 0000000..8ab2cd8 --- /dev/null +++ b/tests/test_summary.py @@ -0,0 +1,61 @@ +"""Tests for dataset aggregation and derived thresholds.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from swe_data_processing.summary import build_summary, percentile + + +def _metric(sample_id: str, tokens: int, errors: int) -> dict: + return { + "sample_id": sample_id, + "resolved": 1, + "source_dataset": "test", + "length": { + "turn_count": 10, + "canonical_chars": 100, + "token_count": tokens, + }, + "tools": { + "tool_call_count": 10, + "failed_tool_call_count": errors, + "failed_tool_call_rate": errors / 10, + "longest_consecutive_failure_run": 0, + "error_type_counts": {"nonzero_exit": errors} if errors else {}, + "error_tool_counts": {"bash": errors} if errors else {}, + "error_positions": { + "early_count": 0, + "early_fraction": 0.0, + "occupied_bins_5": 0, + }, + }, + "structure": { + "invalid_turn_count": 0, + "malformed_tool_definition_count": 0, + "malformed_tool_call_count": 0, + "unknown_tool_call_count": 0, + "missing_tool_result_count": 0, + "orphan_tool_result_count": 0, + }, + } + + +def test_percentile_interpolates() -> None: + assert percentile([0, 10], 0.5) == 5 + + +def test_summary_reports_token_tail_and_p99_threshold(tmp_path: Path) -> None: + path = tmp_path / "metrics.jsonl" + rows = [_metric(f"s-{index}", 10_000, index) for index in range(10)] + rows.append(_metric("long", 300_000, 0)) + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + summary, threshold = build_summary(path) + assert summary["sample_count"] == 11 + assert summary["length"]["buckets"]["GT_262144"]["samples"] == 1 + assert summary["length"]["longest_sample_token_contribution"]["0.001"][ + "token_share" + ] > 0.7 + assert threshold >= 8 +