Document audit policy and add full kept export

scripts/repurposing/build_swift_full_kept.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import gzip
+import json
+import sys
+from collections import Counter
+from contextlib import ExitStack
+from pathlib import Path
+from typing import Any, TextIO
+
+import pyarrow.parquet as pq
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "filtering"))
+import audit_native_traces as audit  # noqa: E402
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "repurposing"))
+import build_swift_training_probe_5k as train_builder  # noqa: E402
+
+
+class AuditArgs:
+    long_turn_threshold = 300
+    long_char_threshold = 900_000
+    repeat_window = 24
+    repeat_threshold = 10
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Export all hard-filter-kept Open-SWE-Traces rows to ModelScope-SWIFT JSONL."
+    )
+    parser.add_argument("--input-root", type=Path, default=Path("data/Open-SWE-Traces"))
+    parser.add_argument("--output-dir", type=Path, default=Path("runs/training_full_kept_swift"))
+    parser.add_argument("--batch-size", type=int, default=128)
+    parser.add_argument("--limit", type=int, default=0, help="Optional total row limit for smoke tests.")
+    parser.add_argument("--write-gzip", action="store_true", help="Also write train.jsonl.gz.")
+    return parser.parse_args()
+
+
+def normalize_json(value: Any) -> Any:
+    return json.loads(json.dumps(value, ensure_ascii=False, default=str))
+
+
+def build_item(
+    row: dict[str, Any],
+    config: str,
+    stats: Counter,
+    issues: list[str],
+    details: Counter,
+) -> dict[str, Any]:
+    model_family, scaffold, thinking_mode = train_builder.CONFIG_META[config]
+    messages = train_builder.convert_messages(row.get("trajectory") or [], thinking_mode, stats)
+    return {
+        "messages": messages,
+        "source_dataset": "nvidia/Open-SWE-Traces",
+        "source_config": config,
+        "model_family": model_family,
+        "scaffold": scaffold,
+        "thinking_mode": thinking_mode,
+        "instance_id": row.get("instance_id"),
+        "repo": row.get("repo"),
+        "language": row.get("language"),
+        "license": row.get("license"),
+        "trajectory_id": row.get("trajectory_id"),
+        "resolved": int(row.get("resolved") or 0),
+        "model_patch": row.get("model_patch") or "",
+        "metadata": normalize_json(row.get("metadata") or {}),
+        "audit_flags": issues,
+        "audit_details": dict(details),
+    }
+
+
+def write_jsonl_row(handle: TextIO, item: dict[str, Any]) -> None:
+    handle.write(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n")
+
+
+def main() -> int:
+    args = parse_args()
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    data_root = args.input_root / "data"
+    audit_args = AuditArgs()
+    stats = Counter()
+    remaining = args.limit or None
+
+    with ExitStack() as stack:
+        jsonl = stack.enter_context((args.output_dir / "train.jsonl").open("w", encoding="utf-8"))
+        gz = None
+        if args.write_gzip:
+            gz = stack.enter_context(gzip.open(args.output_dir / "train.jsonl.gz", "wt", encoding="utf-8"))
+
+        for config in audit.CONFIGS:
+            for file in sorted((data_root / config).glob("*.parquet")):
+                if remaining is not None and remaining <= 0:
+                    break
+                parquet = pq.ParquetFile(file)
+                for batch in parquet.iter_batches(batch_size=args.batch_size):
+                    rows = batch.to_pylist()
+                    if remaining is not None:
+                        rows = rows[:remaining]
+                    for row in rows:
+                        stats[f"seen:{config}"] += 1
+                        issues, _details = audit.audit_row(row, audit_args)
+                        hard = audit.hard_filter_issues(issues)
+                        if hard:
+                            stats[f"filtered:{config}"] += 1
+                            for issue in hard:
+                                stats[f"hard_issue:{issue}"] += 1
+                            continue
+                        item = build_item(row, config, stats, issues, _details)
+                        write_jsonl_row(jsonl, item)
+                        if gz is not None:
+                            write_jsonl_row(gz, item)
+                        stats[f"kept:{config}"] += 1
+                        stats[f"resolved:{config}:{item['resolved']}"] += 1
+
+                    if remaining is not None:
+                        remaining -= len(rows)
+                        if remaining <= 0:
+                            break
+
+                print(
+                    json.dumps(
+                        {
+                            "file": str(file),
+                            "seen": stats[f"seen:{config}"],
+                            "kept": stats[f"kept:{config}"],
+                            "filtered": stats[f"filtered:{config}"],
+                        },
+                        ensure_ascii=False,
+                    ),
+                    flush=True,
+                )
+
+    metadata = {
+        "name": "open-swe-traces-swift-full-kept",
+        "source_dataset": "nvidia/Open-SWE-Traces",
+        "format": "modelscope-swift messages JSONL",
+        "selection": "all rows without hard-filter issues",
+        "stats": dict(stats),
+        "thinking_policy": {
+            "minimax": "reasoning_content is wrapped as <think>...</think> in assistant content",
+            "qwen": "non-thinking export; reasoning_content is not emitted; unexpected nonempty reasoning is counted",
+            "tool_response_mask": "tool messages have loss=false",
+        },
+        "files": ["train.jsonl", "metadata.json"] + (["train.jsonl.gz"] if args.write_gzip else []),
+    }
+    (args.output_dir / "metadata.json").write_text(json.dumps(metadata, indent=2, ensure_ascii=False), encoding="utf-8")
+    (args.output_dir / "README.md").write_text(build_readme(metadata), encoding="utf-8")
+    print(json.dumps(metadata, indent=2, ensure_ascii=False))
+    return 0
+
+
+def build_readme(metadata: dict[str, Any]) -> str:
+    return f"""# Open-SWE-Traces Swift Full Kept
+
+This is the full hard-filter-kept export from `nvidia/Open-SWE-Traces` for ModelScope-SWIFT style SFT.
+
+Selection:
+
+- Scan all four Open-SWE-Traces source configs.
+- Drop rows with hard-filter issues from `scripts/filtering/audit_native_traces.py`.
+- Preserve native scaffold semantics.
+- MiniMax rows are exported as thinking examples by wrapping `reasoning_content` in `<think>...</think>`.
+- Qwen rows are exported as non-thinking examples.
+- Tool responses are included with `loss: false`.
+
+Stats:
+
+```json
+{json.dumps(metadata["stats"], indent=2, ensure_ascii=False)}
+```
+"""
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())