Organize Open-SWE-Traces data prep project

scripts/probing/analyze_and_decompose.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import random
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pyarrow.parquet as pq
+
+
+DATASET_DIR = Path("data/Open-SWE-Traces")
+OUT_DIR = Path("runs/subproblem_decomposition")
+SEED = 20260623
+
+
+CONFIGS = {
+    "minimax_m25_openhands": {
+        "path": "data/minimax_m25_openhands_trajectories",
+        "scaffold": "openhands",
+        "model_family": "minimax_m25",
+        "thinking_mode": "thinking",
+    },
+    "minimax_m25_sweagent": {
+        "path": "data/minimax_m25_sweagent_trajectories",
+        "scaffold": "sweagent",
+        "model_family": "minimax_m25",
+        "thinking_mode": "thinking",
+    },
+    "qwen35_openhands": {
+        "path": "data/qwen35_openhands_trajectories",
+        "scaffold": "openhands",
+        "model_family": "qwen35_122b",
+        "thinking_mode": "non_thinking",
+    },
+    "qwen35_sweagent": {
+        "path": "data/qwen35_sweagent_trajectories",
+        "scaffold": "sweagent",
+        "model_family": "qwen35_122b",
+        "thinking_mode": "non_thinking",
+    },
+}
+
+
+PHASE_ORDER = [
+    "understand_task",
+    "explore_repo",
+    "locate_relevant_code",
+    "inspect_code",
+    "reproduce_or_probe",
+    "edit_solution",
+    "verify_solution",
+    "cleanup",
+    "review_diff",
+    "submit",
+    "other",
+]
+
+
+def read_tool_call(message: dict[str, Any]) -> tuple[str | None, dict[str, Any]]:
+    tool_calls = message.get("tool_calls") or []
+    if not tool_calls:
+        return None, {}
+    call = tool_calls[0] or {}
+    fn = call.get("function") or {}
+    name = fn.get("name")
+    args = fn.get("arguments") or "{}"
+    try:
+        parsed = json.loads(args) if isinstance(args, str) and args else {}
+    except json.JSONDecodeError:
+        parsed = {"_raw": args}
+    return name, parsed
+
+
+def command_text(tool_name: str | None, args: dict[str, Any]) -> str:
+    if not tool_name:
+        return ""
+    if "command" in args:
+        return str(args["command"])
+    if "path" in args:
+        return f"{tool_name} {args.get('command', '')} {args.get('path', '')}".strip()
+    if "message" in args:
+        return str(args["message"])
+    return json.dumps(args, ensure_ascii=False)
+
+
+def classify_tool(tool_name: str | None, args: dict[str, Any], assistant_text: str = "") -> str:
+    cmd = command_text(tool_name, args).lower()
+    text = assistant_text.lower()
+    joined = f"{cmd}\n{text}"
+
+    if tool_name in {"finish", "submit"}:
+        return "submit"
+    if re.search(r"\b(git diff|git status|diff\b)", cmd):
+        return "review_diff"
+    if re.search(r"\b(rm|git restore|git checkout --)\b", cmd) or "clean up" in text:
+        return "cleanup"
+    if re.search(r"\b(pytest|go test|npm test|pnpm test|yarn test|cargo test|mvn test|gradle test|phpunit|rspec|tox|unittest)\b", cmd):
+        return "verify_solution"
+    if re.search(r"\bpython .*repro|reproduce|test_reproduce|final_verification|verify|run.*script\b", joined):
+        return "reproduce_or_probe"
+    if tool_name == "str_replace_editor":
+        editor_cmd = str(args.get("command", "")).lower()
+        if editor_cmd in {"str_replace", "insert", "create"} or "file_text" in args or "new_str" in args:
+            return "edit_solution"
+        if editor_cmd == "view":
+            path = str(args.get("path", "")).lower()
+            if path.endswith((".py", ".go", ".ts", ".tsx", ".js", ".jsx", ".rs", ".java", ".php", ".c", ".cc", ".cpp", ".h", ".hpp", ".md", ".rst")):
+                return "inspect_code"
+            return "explore_repo"
+    if re.search(r"\b(sed -n|cat |less |head |tail |nl |grep -n|rg |ripgrep)\b", cmd):
+        return "inspect_code"
+    if re.search(r"\b(find |ls |tree |grep -r|rg --files)\b", cmd):
+        return "locate_relevant_code"
+    if re.search(r"\b(grep |rg )", cmd):
+        return "locate_relevant_code"
+    if tool_name in {"execute_bash", "bash"}:
+        return "reproduce_or_probe"
+    return "other"
+
+
+def task_prompt(user_content: str) -> str:
+    markers = [
+        ("<issue_description>", "</issue_description>"),
+        ("<pr_description>", "</pr_description>"),
+    ]
+    for start, end in markers:
+        if start in user_content and end in user_content:
+            return user_content.split(start, 1)[1].split(end, 1)[0].strip()
+    return user_content[:2000].strip()
+
+
+def phase_goal(phase: str, repo: str, task: str, files: list[str], commands: list[str]) -> str:
+    file_hint = f" in {', '.join(files[:3])}" if files else ""
+    if phase == "understand_task":
+        return f"Understand the requested fix for {repo}."
+    if phase == "explore_repo":
+        return f"Map the repository structure and identify likely implementation areas for the task."
+    if phase == "locate_relevant_code":
+        return f"Search for symbols, files, or modules related to the issue."
+    if phase == "inspect_code":
+        return f"Read the relevant code{file_hint} and infer current behavior."
+    if phase == "reproduce_or_probe":
+        return "Run focused commands or reproduction snippets to observe the bug or current behavior."
+    if phase == "edit_solution":
+        return f"Apply code changes{file_hint} that address the issue."
+    if phase == "verify_solution":
+        return "Run targeted tests or validation commands to check the fix."
+    if phase == "cleanup":
+        return "Remove temporary reproduction files or revert unrelated artifacts."
+    if phase == "review_diff":
+        return "Inspect the final diff and working tree before submission."
+    if phase == "submit":
+        return "Submit or finish the solution with a concise summary."
+    return "Continue task-specific investigation."
+
+
+def extract_paths(text: str) -> list[str]:
+    candidates = re.findall(r"(?:(?:/workspace|/testbed)[^\s,:;`'\")]+|[\w./-]+\.(?:py|go|ts|tsx|js|jsx|rs|java|php|c|cc|cpp|h|hpp|md|rst))", text)
+    cleaned = []
+    for c in candidates:
+        c = c.rstrip(".,)]}")
+        if c not in cleaned:
+            cleaned.append(c)
+    return cleaned[:20]
+
+
+def compact_text(value: Any, limit: int = 500) -> str:
+    if value is None:
+        return ""
+    text = str(value).strip()
+    return text[:limit] + ("..." if len(text) > limit else "")
+
+
+def decompose(row: dict[str, Any], config_name: str, config: dict[str, str]) -> dict[str, Any]:
+    trajectory = row["trajectory"] or []
+    user = next((m for m in trajectory if m.get("role") == "user"), {})
+    task = task_prompt(user.get("content") or "")
+
+    segments: list[dict[str, Any]] = []
+    current: dict[str, Any] | None = None
+
+    def close(end_idx: int) -> None:
+        nonlocal current
+        if not current:
+            return
+        current["turn_range"][1] = end_idx
+        current["goal"] = phase_goal(
+            current["phase"],
+            row["repo"],
+            task,
+            current["files"],
+            current["commands"],
+        )
+        segments.append(current)
+        current = None
+
+    for idx, msg in enumerate(trajectory):
+        role = msg.get("role")
+        if role in {"system", "user"}:
+            phase = "understand_task"
+            assistant_text = compact_text(msg.get("content"), 400)
+            tool_name, args = None, {}
+        elif role == "assistant":
+            assistant_text = compact_text(msg.get("content") or msg.get("reasoning_content"), 700)
+            tool_name, args = read_tool_call(msg)
+            phase = classify_tool(tool_name, args, assistant_text)
+        else:
+            if current:
+                obs = compact_text(msg.get("content"), 700)
+                if obs:
+                    current["observations"].append(obs)
+                current["files"].extend(p for p in extract_paths(obs) if p not in current["files"])
+            continue
+
+        if current is None or phase != current["phase"]:
+            close(idx - 1)
+            current = {
+                "phase": phase,
+                "goal": "",
+                "turn_range": [idx, idx],
+                "commands": [],
+                "tool_names": [],
+                "files": [],
+                "assistant_intents": [],
+                "observations": [],
+            }
+
+        current["turn_range"][1] = idx
+        if assistant_text:
+            current["assistant_intents"].append(assistant_text)
+        if tool_name:
+            current["tool_names"].append(tool_name)
+        cmd = command_text(tool_name, args)
+        if cmd:
+            current["commands"].append(compact_text(cmd, 600))
+            current["files"].extend(p for p in extract_paths(cmd) if p not in current["files"])
+
+    close(len(trajectory) - 1)
+
+    # Merge very small adjacent "other" segments into neighbors where possible.
+    merged: list[dict[str, Any]] = []
+    for seg in segments:
+        if seg["phase"] == "other" and merged:
+            prev = merged[-1]
+            prev["turn_range"][1] = seg["turn_range"][1]
+            prev["commands"].extend(seg["commands"])
+            prev["tool_names"].extend(seg["tool_names"])
+            prev["files"].extend(p for p in seg["files"] if p not in prev["files"])
+            prev["assistant_intents"].extend(seg["assistant_intents"])
+            prev["observations"].extend(seg["observations"])
+        else:
+            merged.append(seg)
+
+    qa_pairs = []
+    for seg in merged:
+        qa_pairs.append(
+            {
+                "question": f"In repository {row['repo']}, how can we {seg['goal'][0].lower() + seg['goal'][1:]}",
+                "answer_outline": {
+                    "phase": seg["phase"],
+                    "files": seg["files"][:8],
+                    "commands": seg["commands"][:8],
+                    "observations": seg["observations"][:4],
+                },
+            }
+        )
+
+    return {
+        "config": config_name,
+        "scaffold": config["scaffold"],
+        "model_family": config["model_family"],
+        "thinking_mode": config["thinking_mode"],
+        "instance_id": row["instance_id"],
+        "repo": row["repo"],
+        "language": row["language"],
+        "license": row["license"],
+        "trajectory_id": row["trajectory_id"],
+        "resolved": row["resolved"],
+        "metadata": row.get("metadata"),
+        "trajectory_len": len(trajectory),
+        "task": task,
+        "model_patch_head": compact_text(row.get("model_patch"), 1200),
+        "segments": merged,
+        "qa_pairs": qa_pairs,
+        "quality_flags": quality_flags(merged, row),
+    }
+
+
+def quality_flags(segments: list[dict[str, Any]], row: dict[str, Any]) -> dict[str, Any]:
+    phases = {s["phase"] for s in segments}
+    return {
+        "has_locate_or_inspect": bool(phases & {"locate_relevant_code", "inspect_code", "explore_repo"}),
+        "has_edit": "edit_solution" in phases,
+        "has_verify": "verify_solution" in phases or "reproduce_or_probe" in phases,
+        "has_submit": "submit" in phases,
+        "has_patch": bool(row.get("model_patch")),
+        "usable_for_subproblem_training": bool(
+            phases & {"locate_relevant_code", "inspect_code", "edit_solution"}
+        )
+        and bool(row.get("model_patch")),
+    }
+
+
+def collect_unique_counts() -> dict[str, Any]:
+    global_instances: set[str] = set()
+    global_repos: set[str] = set()
+    by_config = {}
+    by_instance_configs: dict[str, set[str]] = defaultdict(set)
+
+    for name, cfg in CONFIGS.items():
+        inst: set[str] = set()
+        repos: set[str] = set()
+        resolved = Counter()
+        languages = Counter()
+        rows = 0
+        for file in sorted((DATASET_DIR / cfg["path"]).glob("*.parquet")):
+            table = pq.read_table(file, columns=["instance_id", "repo", "language", "resolved"])
+            for r in table.to_pylist():
+                rows += 1
+                inst.add(r["instance_id"])
+                repos.add(r["repo"])
+                global_instances.add(r["instance_id"])
+                global_repos.add(r["repo"])
+                by_instance_configs[r["instance_id"]].add(name)
+                resolved[str(r["resolved"])] += 1
+                languages[r["language"]] += 1
+        by_config[name] = {
+            "rows": rows,
+            "unique_instances": len(inst),
+            "unique_repos": len(repos),
+            "resolved_counts": dict(resolved),
+            "language_counts": dict(languages),
+            "thinking_mode": cfg["thinking_mode"],
+            "scaffold": cfg["scaffold"],
+            "model_family": cfg["model_family"],
+        }
+
+    overlap = Counter(len(v) for v in by_instance_configs.values())
+    return {
+        "total_rows": sum(v["rows"] for v in by_config.values()),
+        "global_unique_instances": len(global_instances),
+        "global_unique_repos": len(global_repos),
+        "instance_coverage_across_configs": dict(overlap),
+        "by_config": by_config,
+    }
+
+
+def load_random_rows(total: int = 20) -> list[tuple[str, dict[str, str], dict[str, Any]]]:
+    rng = random.Random(SEED)
+    per_config = total // len(CONFIGS)
+    remainder = total % len(CONFIGS)
+    output = []
+    for idx, (name, cfg) in enumerate(CONFIGS.items()):
+        need = per_config + (1 if idx < remainder else 0)
+        files = sorted((DATASET_DIR / cfg["path"]).glob("*.parquet"))
+        row_counts = [pq.ParquetFile(f).metadata.num_rows for f in files]
+        total_rows = sum(row_counts)
+        targets = sorted(rng.sample(range(total_rows), need))
+        cursor = 0
+        for file, count in zip(files, row_counts):
+            local_targets = [t - cursor for t in targets if cursor <= t < cursor + count]
+            if local_targets:
+                table = pq.read_table(file)
+                rows = table.to_pylist()
+                for local in local_targets:
+                    output.append((name, cfg, rows[local]))
+            cursor += count
+    rng.shuffle(output)
+    return output
+
+
+def main() -> int:
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    counts = collect_unique_counts()
+    (OUT_DIR / "unique_counts.json").write_text(json.dumps(counts, indent=2, ensure_ascii=False))
+
+    decomposed = [decompose(row, name, cfg) for name, cfg, row in load_random_rows(20)]
+    (OUT_DIR / "random20_decomposed.json").write_text(json.dumps(decomposed, indent=2, ensure_ascii=False))
+
+    phase_counts = Counter()
+    usable = 0
+    for item in decomposed:
+        usable += int(item["quality_flags"]["usable_for_subproblem_training"])
+        for seg in item["segments"]:
+            phase_counts[seg["phase"]] += 1
+
+    report = {
+        "seed": SEED,
+        "unique_counts": counts,
+        "random20_summary": {
+            "items": len(decomposed),
+            "usable_for_subproblem_training": usable,
+            "phase_counts": dict(phase_counts),
+            "avg_segments_per_trace": sum(len(x["segments"]) for x in decomposed) / len(decomposed),
+        },
+    }
+    (OUT_DIR / "report.json").write_text(json.dumps(report, indent=2, ensure_ascii=False))
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/check_tokenizer_env.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+mods = ["transformers", "tokenizers", "modelscope", "sentencepiece"]
+for name in mods:
+    try:
+        mod = __import__(name)
+        print(name, "OK", getattr(mod, "__version__", ""))
+    except Exception as exc:
+        print(name, "NO", type(exc).__name__, str(exc)[:120])

scripts/probing/compare_qwen_tokenizers.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from huggingface_hub import snapshot_download
+
+
+MODELS = ["Qwen/Qwen3.5-27B", "Qwen/Qwen3.6-27B"]
+ALLOW = [
+    "tokenizer.json",
+    "tokenizer_config.json",
+    "vocab.json",
+    "merges.txt",
+    "special_tokens_map.json",
+    "config.json",
+    "generation_config.json",
+]
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def main() -> int:
+    base = Path("runs/qwen_tokenizer_compare")
+    base.mkdir(parents=True, exist_ok=True)
+    snapshots = {}
+    for model in MODELS:
+        local = snapshot_download(
+            model,
+            allow_patterns=ALLOW,
+            local_dir=base / model.replace("/", "__"),
+            local_dir_use_symlinks=False,
+        )
+        snapshots[model] = Path(local)
+        print("DOWNLOADED", model, local)
+
+    report = {"models": {}, "comparisons": {}}
+    for model, path in snapshots.items():
+        files = {}
+        for name in ALLOW:
+            f = path / name
+            if f.exists():
+                files[name] = {"sha256": sha256(f), "size": f.stat().st_size}
+        tokenizer_config = {}
+        tc = path / "tokenizer_config.json"
+        if tc.exists():
+            tokenizer_config = json.loads(tc.read_text(encoding="utf-8"))
+        report["models"][model] = {
+            "path": str(path),
+            "files": files,
+            "tokenizer_class": tokenizer_config.get("tokenizer_class"),
+            "chat_template": tokenizer_config.get("chat_template"),
+            "eos_token": tokenizer_config.get("eos_token"),
+            "pad_token": tokenizer_config.get("pad_token"),
+            "additional_special_tokens": tokenizer_config.get("additional_special_tokens"),
+        }
+
+    a, b = MODELS
+    a_files = report["models"][a]["files"]
+    b_files = report["models"][b]["files"]
+    all_files = sorted(set(a_files) | set(b_files))
+    for name in all_files:
+        report["comparisons"][name] = {
+            "same": a_files.get(name, {}).get("sha256") == b_files.get(name, {}).get("sha256"),
+            a: a_files.get(name),
+            b: b_files.get(name),
+        }
+    report["same_tokenizer_core"] = all(
+        report["comparisons"].get(name, {}).get("same") for name in ["tokenizer.json", "vocab.json", "merges.txt"]
+    )
+    report["same_tokenizer_config"] = report["comparisons"].get("tokenizer_config.json", {}).get("same")
+    report["same_chat_template"] = report["models"][a].get("chat_template") == report["models"][b].get("chat_template")
+    out = base / "comparison_report.json"
+    out.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    print(json.dumps(report, indent=2, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/count_qwen_tokens_exact.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from collections import Counter
+from pathlib import Path
+
+import pyarrow.parquet as pq
+from transformers import AutoTokenizer
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "filtering"))
+import audit_native_traces as audit  # noqa: E402
+
+
+class AuditArgs:
+    long_turn_threshold = 300
+    long_char_threshold = 900_000
+    repeat_window = 24
+    repeat_threshold = 10
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Exact Qwen tokenizer count for kept Open-SWE traces.")
+    parser.add_argument("--input-root", type=Path, default=Path("data/Open-SWE-Traces"))
+    parser.add_argument("--output", type=Path, default=Path("runs/native_trace_audit/qwen_exact_token_count.json"))
+    parser.add_argument("--model", default="Qwen/Qwen3-32B")
+    parser.add_argument("--batch-size", type=int, default=512)
+    parser.add_argument("--tokenize-batch-size", type=int, default=2048)
+    parser.add_argument("--flush-interval", type=int, default=25_000)
+    return parser.parse_args()
+
+
+def message_text(message: dict) -> str:
+    text = ""
+    for key in ("content", "reasoning_content"):
+        value = message.get(key)
+        if isinstance(value, str):
+            text += value
+    if message.get("tool_calls"):
+        text += json.dumps(message.get("tool_calls"), ensure_ascii=False)
+    return text
+
+
+def add_tokenized_batch(tokenizer, texts: list[str], roles: list[str], stats: Counter) -> None:
+    if not texts:
+        return
+    encoded = tokenizer(texts, add_special_tokens=False, return_attention_mask=False, return_token_type_ids=False)
+    for role, ids in zip(roles, encoded["input_ids"], strict=True):
+        n = len(ids)
+        stats["tokens_total"] += n
+        stats[f"tokens_{role}"] += n
+
+
+def update_output(path: Path, report: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    tmp.replace(path)
+
+
+def summarize(stats: Counter) -> dict:
+    out = dict(stats)
+    kept = stats.get("kept", 0)
+    total = stats.get("tokens_total", 0)
+    tool = stats.get("tokens_tool", 0)
+    out["loss_tokens_if_tool_masked"] = total - tool
+    out["tool_response_token_share"] = tool / total if total else 0
+    out["avg_total_tokens_per_trace"] = total / kept if kept else 0
+    out["avg_loss_tokens_per_trace_if_tool_masked"] = (total - tool) / kept if kept else 0
+    return out
+
+
+def main() -> int:
+    args = parse_args()
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    audit_args = AuditArgs()
+    data_root = args.input_root / "data"
+    total = Counter()
+    configs: dict[str, Counter] = {}
+    start = time.time()
+    processed_since_flush = 0
+
+    for config in audit.CONFIGS:
+        cfg_stats = configs.setdefault(config, Counter())
+        for file in sorted((data_root / config).glob("*.parquet")):
+            file_stats = Counter()
+            parquet = pq.ParquetFile(file)
+            pending_texts: list[str] = []
+            pending_roles: list[str] = []
+
+            def flush_tokens() -> None:
+                add_tokenized_batch(tokenizer, pending_texts, pending_roles, file_stats)
+                pending_texts.clear()
+                pending_roles.clear()
+
+            for record_batch in parquet.iter_batches(batch_size=args.batch_size):
+                for row in record_batch.to_pylist():
+                    issues, _details = audit.audit_row(row, audit_args)
+                    hard = audit.hard_filter_issues(issues)
+                    if hard:
+                        file_stats["filtered"] += 1
+                        continue
+                    file_stats["kept"] += 1
+                    for message in row.get("trajectory") or []:
+                        role = message.get("role") or "unknown"
+                        text = message_text(message)
+                        file_stats["messages"] += 1
+                        file_stats[f"messages_{role}"] += 1
+                        file_stats["chars_total"] += len(text)
+                        file_stats[f"chars_{role}"] += len(text)
+                        if text:
+                            pending_texts.append(text)
+                            pending_roles.append(role)
+                        if len(pending_texts) >= args.tokenize_batch_size:
+                            flush_tokens()
+                    processed_since_flush += 1
+                    if processed_since_flush >= args.flush_interval:
+                        flush_tokens()
+                        processed_since_flush = 0
+                        elapsed = time.time() - start
+                        partial_total = Counter()
+                        for cstats in configs.values():
+                            partial_total.update(cstats)
+                        partial_total.update(total)
+                        report = {
+                            "model": args.model,
+                            "status": "running",
+                            "elapsed_seconds": elapsed,
+                            "configs": {k: summarize(v) for k, v in configs.items()},
+                            "total": summarize(partial_total),
+                        }
+                        update_output(args.output, report)
+            flush_tokens()
+            cfg_stats.update(file_stats)
+            print(
+                json.dumps(
+                    {
+                        "file": str(file),
+                        "stats": summarize(file_stats),
+                        "elapsed_seconds": round(time.time() - start, 2),
+                    },
+                    ensure_ascii=False,
+                ),
+                flush=True,
+            )
+        total.update(cfg_stats)
+
+    report = {
+        "model": args.model,
+        "status": "complete",
+        "elapsed_seconds": time.time() - start,
+        "configs": {k: summarize(v) for k, v in configs.items()},
+        "total": summarize(total),
+    }
+    update_output(args.output, report)
+    print(json.dumps({"output": str(args.output), "total": report["total"]}, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/count_qwen_tokens_exact_parallel.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from collections import Counter
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+import pyarrow.parquet as pq
+from transformers import AutoTokenizer
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "filtering"))
+import audit_native_traces as audit  # noqa: E402
+
+
+TOKENIZER = None
+MODEL_NAME = None
+
+
+class AuditArgs:
+    long_turn_threshold = 300
+    long_char_threshold = 900_000
+    repeat_window = 24
+    repeat_threshold = 10
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Parallel exact Qwen tokenizer count for Open-SWE traces.")
+    parser.add_argument("--input-root", type=Path, default=Path("data/Open-SWE-Traces"))
+    parser.add_argument("--output", type=Path, default=Path("runs/native_trace_audit/qwen_exact_token_count.json"))
+    parser.add_argument("--model", default="Qwen/Qwen3-32B")
+    parser.add_argument("--workers", type=int, default=12)
+    parser.add_argument("--batch-size", type=int, default=512)
+    parser.add_argument("--tokenize-batch-size", type=int, default=1024)
+    return parser.parse_args()
+
+
+def init_worker(model_name: str) -> None:
+    global TOKENIZER, MODEL_NAME
+    MODEL_NAME = model_name
+    os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+    TOKENIZER = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+
+def message_text(message: dict[str, Any]) -> str:
+    text = ""
+    for key in ("content", "reasoning_content"):
+        value = message.get(key)
+        if isinstance(value, str):
+            text += value
+    if message.get("tool_calls"):
+        text += json.dumps(message.get("tool_calls"), ensure_ascii=False)
+    return text
+
+
+def add_tokenized_batch(texts: list[str], roles: list[str], stats: Counter) -> None:
+    if not texts:
+        return
+    encoded = TOKENIZER(texts, add_special_tokens=False, return_attention_mask=False, return_token_type_ids=False)
+    for role, ids in zip(roles, encoded["input_ids"], strict=True):
+        n = len(ids)
+        stats["tokens_total"] += n
+        stats[f"tokens_{role}"] += n
+
+
+def count_file(task: tuple[str, str, int, int]) -> dict[str, Any]:
+    config, file_name, batch_size, tokenize_batch_size = task
+    file = Path(file_name)
+    audit_args = AuditArgs()
+    stats = Counter()
+    pending_texts: list[str] = []
+    pending_roles: list[str] = []
+
+    def flush() -> None:
+        add_tokenized_batch(pending_texts, pending_roles, stats)
+        pending_texts.clear()
+        pending_roles.clear()
+
+    parquet = pq.ParquetFile(file)
+    for record_batch in parquet.iter_batches(batch_size=batch_size):
+        for row in record_batch.to_pylist():
+            issues, _details = audit.audit_row(row, audit_args)
+            hard = audit.hard_filter_issues(issues)
+            if hard:
+                stats["filtered"] += 1
+                continue
+            stats["kept"] += 1
+            for message in row.get("trajectory") or []:
+                role = message.get("role") or "unknown"
+                text = message_text(message)
+                stats["messages"] += 1
+                stats[f"messages_{role}"] += 1
+                stats["chars_total"] += len(text)
+                stats[f"chars_{role}"] += len(text)
+                if text:
+                    pending_texts.append(text)
+                    pending_roles.append(role)
+                if len(pending_texts) >= tokenize_batch_size:
+                    flush()
+    flush()
+    return {"config": config, "file": str(file), "stats": dict(stats)}
+
+
+def summarize(stats: Counter) -> dict[str, Any]:
+    out = dict(stats)
+    kept = stats.get("kept", 0)
+    total = stats.get("tokens_total", 0)
+    tool = stats.get("tokens_tool", 0)
+    out["loss_tokens_if_tool_masked"] = total - tool
+    out["tool_response_token_share"] = tool / total if total else 0
+    out["avg_total_tokens_per_trace"] = total / kept if kept else 0
+    out["avg_loss_tokens_per_trace_if_tool_masked"] = (total - tool) / kept if kept else 0
+    return out
+
+
+def write_report(path: Path, report: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    tmp.replace(path)
+
+
+def main() -> int:
+    args = parse_args()
+    data_root = args.input_root / "data"
+    tasks = []
+    for config in audit.CONFIGS:
+        for file in sorted((data_root / config).glob("*.parquet")):
+            tasks.append((config, str(file), args.batch_size, args.tokenize_batch_size))
+
+    start = time.time()
+    configs = {config: Counter() for config in audit.CONFIGS}
+    total = Counter()
+    completed = 0
+    with ProcessPoolExecutor(max_workers=args.workers, initializer=init_worker, initargs=(args.model,)) as pool:
+        futures = [pool.submit(count_file, task) for task in tasks]
+        for future in as_completed(futures):
+            result = future.result()
+            completed += 1
+            config = result["config"]
+            file_stats = Counter(result["stats"])
+            configs[config].update(file_stats)
+            total.update(file_stats)
+            elapsed = time.time() - start
+            print(
+                json.dumps(
+                    {
+                        "completed": completed,
+                        "total_files": len(tasks),
+                        "file": result["file"],
+                        "stats": summarize(file_stats),
+                        "elapsed_seconds": round(elapsed, 2),
+                    },
+                    ensure_ascii=False,
+                ),
+                flush=True,
+            )
+            report = {
+                "model": args.model,
+                "status": "running" if completed < len(tasks) else "complete",
+                "workers": args.workers,
+                "completed_files": completed,
+                "total_files": len(tasks),
+                "elapsed_seconds": elapsed,
+                "configs": {key: summarize(value) for key, value in configs.items()},
+                "total": summarize(total),
+            }
+            write_report(args.output, report)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/estimate_native_trace_tokens.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+import pyarrow.parquet as pq
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "filtering"))
+import audit_native_traces as audit  # noqa: E402
+
+
+class Args:
+    long_turn_threshold = 300
+    long_char_threshold = 900_000
+    repeat_window = 24
+    repeat_threshold = 10
+
+
+def main() -> int:
+    args = Args()
+    base = Path("data/Open-SWE-Traces/data")
+    stats = Counter()
+    for cfg in audit.CONFIGS:
+        c = Counter()
+        for file in sorted((base / cfg).glob("*.parquet")):
+            parquet = pq.ParquetFile(file)
+            for batch in parquet.iter_batches(batch_size=512):
+                for row in batch.to_pylist():
+                    issues, _details = audit.audit_row(row, args)
+                    hard = audit.hard_filter_issues(issues)
+                    if hard:
+                        c["filtered"] += 1
+                        stats["filtered"] += 1
+                        continue
+                    c["kept"] += 1
+                    stats["kept"] += 1
+                    trajectory = row.get("trajectory") or []
+                    for message in trajectory:
+                        role = message.get("role") or "unknown"
+                        text = ""
+                        for key in ("content", "reasoning_content"):
+                            value = message.get(key)
+                            if isinstance(value, str):
+                                text += value
+                        if message.get("tool_calls"):
+                            text += json.dumps(message.get("tool_calls"), ensure_ascii=False)
+                        chars = len(text)
+                        c[f"chars_{role}"] += chars
+                        stats[f"chars_{role}"] += chars
+                        c["chars_total"] += chars
+                        stats["chars_total"] += chars
+                    c["messages"] += len(trajectory)
+                    stats["messages"] += len(trajectory)
+        print("CFG", cfg, json.dumps(dict(c), ensure_ascii=False), flush=True)
+    print("TOTAL", json.dumps(dict(stats), ensure_ascii=False))
+    for ratio in (3.0, 3.5, 4.0):
+        total_tokens = stats["chars_total"] / ratio
+        tool_tokens = stats["chars_tool"] / ratio
+        loss_tokens = (stats["chars_total"] - stats["chars_tool"]) / ratio
+        print(
+            "TOKENS_RATIO",
+            ratio,
+            json.dumps(
+                {
+                    "total_tokens": round(total_tokens),
+                    "tool_response_tokens": round(tool_tokens),
+                    "loss_tokens_if_tool_masked": round(loss_tokens),
+                    "avg_total_tokens_per_row": round(total_tokens / stats["kept"]),
+                    "avg_loss_tokens_per_row": round(loss_tokens / stats["kept"]),
+                    "tool_response_token_share": round(tool_tokens / total_tokens, 4),
+                },
+                ensure_ascii=False,
+            ),
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/inspect_sample.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+
+import pyarrow.parquet as pq
+
+
+BASE = Path("data/Open-SWE-Traces/data")
+CONFIGS = [
+    "minimax_m25_openhands_trajectories",
+    "minimax_m25_sweagent_trajectories",
+    "qwen35_openhands_trajectories",
+    "qwen35_sweagent_trajectories",
+]
+
+
+def main() -> int:
+    for cfg in CONFIGS:
+        file = sorted((BASE / cfg).glob("*.parquet"))[0]
+        row = pq.read_table(file).slice(0, 1).to_pylist()[0]
+        print("CFG", cfg)
+        print("cols", list(row.keys()))
+        trajectory = row["trajectory"]
+        print("len", len(trajectory))
+        for index, message in enumerate(trajectory[:10]):
+            print("MSG", index, json.dumps(message, ensure_ascii=False)[:1600])
+        print("---")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/load_qwen_tokenizer.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+from transformers import AutoTokenizer
+
+
+MODEL = "Qwen/Qwen3-32B"
+
+
+def main() -> int:
+    tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+    print(type(tokenizer).__name__, tokenizer.vocab_size, tokenizer.eos_token_id)
+    print(tokenizer.encode("hello world"))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/query_qwen_model_ids.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+from huggingface_hub import HfApi
+
+
+def main() -> int:
+    api = HfApi(endpoint="https://hf-mirror.com")
+    for query in ["Qwen3.5", "Qwen3.6", "Qwen3.5-27B", "Qwen3.6-27B", "Qwen3.6-35B"]:
+        print("QUERY", query)
+        try:
+            models = list(api.list_models(search=query, limit=50))
+            for model in models:
+                print(model.modelId)
+        except Exception as exc:
+            print("ERR", type(exc).__name__, str(exc)[:300])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/probing/sample_qwen_token_ratios.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+import pyarrow.parquet as pq
+from transformers import AutoTokenizer
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT / "scripts" / "filtering"))
+import audit_native_traces as audit  # noqa: E402
+
+
+class AuditArgs:
+    long_turn_threshold = 300
+    long_char_threshold = 900_000
+    repeat_window = 24
+    repeat_threshold = 10
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", default="Qwen/Qwen3-32B")
+    parser.add_argument("--samples-per-config", type=int, default=2000)
+    parser.add_argument("--output", type=Path, default=Path("runs/native_trace_audit/qwen_token_ratio_sample.json"))
+    return parser.parse_args()
+
+
+def msg_text(message):
+    text = ""
+    for key in ("content", "reasoning_content"):
+        value = message.get(key)
+        if isinstance(value, str):
+            text += value
+    if message.get("tool_calls"):
+        text += json.dumps(message.get("tool_calls"), ensure_ascii=False)
+    return text
+
+
+def main() -> int:
+    args = parse_args()
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    audit_args = AuditArgs()
+    base = Path("data/Open-SWE-Traces/data")
+    report = {"model": args.model, "configs": {}}
+    total = Counter()
+
+    for cfg in audit.CONFIGS:
+        stats = Counter()
+        for file in sorted((base / cfg).glob("*.parquet")):
+            if stats["rows"] >= args.samples_per_config:
+                break
+            parquet = pq.ParquetFile(file)
+            for batch in parquet.iter_batches(batch_size=256):
+                for row in batch.to_pylist():
+                    issues, _ = audit.audit_row(row, audit_args)
+                    if audit.hard_filter_issues(issues):
+                        continue
+                    stats["rows"] += 1
+                    for message in row.get("trajectory") or []:
+                        role = message.get("role") or "unknown"
+                        text = msg_text(message)
+                        chars = len(text)
+                        if not chars:
+                            continue
+                        toks = len(tokenizer.encode(text, add_special_tokens=False))
+                        stats[f"chars_{role}"] += chars
+                        stats[f"tokens_{role}"] += toks
+                        stats["chars_total"] += chars
+                        stats["tokens_total"] += toks
+                    if stats["rows"] >= args.samples_per_config:
+                        break
+                if stats["rows"] >= args.samples_per_config:
+                    break
+        report["configs"][cfg] = summarize(stats)
+        total.update(stats)
+        print("CFG", cfg, json.dumps(report["configs"][cfg], ensure_ascii=False), flush=True)
+
+    report["total_sample"] = summarize(total)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
+    print("REPORT", args.output)
+    return 0
+
+
+def summarize(stats: Counter) -> dict:
+    out = dict(stats)
+    if stats["chars_total"]:
+        out["tokens_per_char_total"] = stats["tokens_total"] / stats["chars_total"]
+        out["chars_per_token_total"] = stats["chars_total"] / stats["tokens_total"]
+    for role in ("system", "user", "assistant", "tool"):
+        c = stats.get(f"chars_{role}", 0)
+        t = stats.get(f"tokens_{role}", 0)
+        if c and t:
+            out[f"tokens_per_char_{role}"] = t / c
+            out[f"chars_per_token_{role}"] = c / t
+    return out
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())