Add SWIFT coding agent probe experiment scripts

scripts/download_dataset.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import gzip
+import os
+import shutil
+from pathlib import Path
+
+from huggingface_hub import HfApi, snapshot_download
+
+
+DEFAULT_REPO_NAME = "ti_coding_agent_training_probe_20260624"
+
+
+def resolve_dataset_id(raw: str, token: str | None, endpoint: str | None) -> str:
+    if "/" in raw:
+        return raw
+    if not token:
+        raise SystemExit(
+            "HF_DATASET_REPO_ID must be owner/name when HF_TOKEN is not set. "
+            f"Got unqualified repo name: {raw}"
+        )
+    owner = HfApi(token=token, endpoint=endpoint).whoami()["name"]
+    return f"{owner}/{raw}"
+
+
+def gunzip_if_needed(src: Path, dst: Path) -> None:
+    if dst.exists() and dst.stat().st_size > 0:
+        return
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    with gzip.open(src, "rb") as fin, dst.open("wb") as fout:
+        shutil.copyfileobj(fin, fout, length=1024 * 1024)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset-id", default=os.environ.get("HF_DATASET_REPO_ID", DEFAULT_REPO_NAME))
+    parser.add_argument("--raw-dir", default="data/raw/training_probe")
+    parser.add_argument("--out-dir", default="data/processed/training_probe")
+    args = parser.parse_args()
+
+    token = os.environ.get("HF_TOKEN")
+    endpoint = os.environ.get("HF_ENDPOINT")
+    dataset_id = resolve_dataset_id(args.dataset_id, token, endpoint)
+
+    raw_dir = Path(args.raw_dir)
+    out_dir = Path(args.out_dir)
+    raw_dir.mkdir(parents=True, exist_ok=True)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    local_path = snapshot_download(
+        repo_id=dataset_id,
+        repo_type="dataset",
+        local_dir=raw_dir,
+        token=token,
+        endpoint=endpoint,
+        allow_patterns=[
+            "README.md",
+            "metadata.json",
+            "train.parquet",
+            "validation.parquet",
+            "train.jsonl.gz",
+            "validation.jsonl.gz",
+        ],
+    )
+    local = Path(local_path)
+    for name in ("train", "validation"):
+        gz = local / f"{name}.jsonl.gz"
+        if gz.exists():
+            gunzip_if_needed(gz, out_dir / f"{name}.jsonl")
+        parquet = local / f"{name}.parquet"
+        if parquet.exists():
+            target = out_dir / f"{name}.parquet"
+            if not target.exists():
+                target.symlink_to(parquet.resolve())
+    print(f"DATASET_ID={dataset_id}")
+    print(f"TRAIN_JSONL={out_dir / 'train.jsonl'}")
+    print(f"VALIDATION_JSONL={out_dir / 'validation.jsonl'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/download_models.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT_DIR}"
+
+export http_proxy="${http_proxy:-http://100.72.0.101:8888}"
+export https_proxy="${https_proxy:-http://100.72.0.101:8888}"
+export HTTP_PROXY="${HTTP_PROXY:-${http_proxy}}"
+export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy}}"
+export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
+
+source .venv/bin/activate
+mkdir -p models
+
+QWEN35_9B_MODEL_ID="${QWEN35_9B_MODEL_ID:-Qwen/Qwen3.5-9B}"
+QWEN36_27B_MODEL_ID="${QWEN36_27B_MODEL_ID:-Qwen/Qwen3.6-27B}"
+
+huggingface-cli download "${QWEN35_9B_MODEL_ID}" --local-dir "models/qwen3.5-9b" --local-dir-use-symlinks False
+huggingface-cli download "${QWEN36_27B_MODEL_ID}" --local-dir "models/qwen3.6-27b" --local-dir-use-symlinks False

scripts/run_all_experiments.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT_DIR}"
+
+./scripts/download_dataset.py
+./scripts/download_models.sh
+
+./scripts/train_qwen35_9b_lora.sh
+./scripts/train_qwen35_9b_full.sh
+./scripts/train_qwen36_27b_lora.sh
+./scripts/train_qwen36_27b_full.sh
+
+echo "All experiments finished."
+echo "TensorBoard: tensorboard --logdir runs --host 0.0.0.0 --port 6006"

scripts/setup_env.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT_DIR}"
+
+export http_proxy="${http_proxy:-http://100.72.0.101:8888}"
+export https_proxy="${https_proxy:-http://100.72.0.101:8888}"
+export HTTP_PROXY="${HTTP_PROXY:-${http_proxy}}"
+export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy}}"
+export PIP_INDEX_URL="${PIP_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/}"
+
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install -U pip setuptools wheel
+python -m pip install -e .
+python -m pip install -e third_party/modelscope-swift
+
+mkdir -p data/raw data/processed models outputs runs logs
+python - <<'PY'
+import swift, sys
+print("python", sys.version)
+print("swift", getattr(swift, "__version__", "unknown"))
+PY

scripts/swift_train_common.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT_DIR}"
+
+export http_proxy="${http_proxy:-http://100.72.0.101:8888}"
+export https_proxy="${https_proxy:-http://100.72.0.101:8888}"
+export HTTP_PROXY="${HTTP_PROXY:-${http_proxy}}"
+export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy}}"
+export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
+export TOKENIZERS_PARALLELISM="${TOKENIZERS_PARALLELISM:-false}"
+
+if [[ -f .venv/bin/activate ]]; then
+  source .venv/bin/activate
+elif [[ "${DRY_RUN:-0}" != "1" ]]; then
+  echo "Missing .venv. Run ./scripts/setup_env.sh first." >&2
+  exit 2
+fi
+mkdir -p outputs runs logs
+
+TRAIN_JSONL="${TRAIN_JSONL:-data/processed/training_probe/train.jsonl}"
+VAL_JSONL="${VAL_JSONL:-data/processed/training_probe/validation.jsonl}"
+MAX_LENGTH="${MAX_LENGTH:-262144}"
+SAVE_STEPS="${SAVE_STEPS:-1000}"
+EVAL_STEPS="${EVAL_STEPS:-1000}"
+LOGGING_STEPS="${LOGGING_STEPS:-1}"
+GRAD_ACCUM_STEPS="${GRAD_ACCUM_STEPS:-1}"
+PER_DEVICE_BATCH_SIZE="${PER_DEVICE_BATCH_SIZE:-1}"
+NUM_EPOCHS="${NUM_EPOCHS:-1}"
+LEARNING_RATE="${LEARNING_RATE:-1e-5}"
+WARMUP_RATIO="${WARMUP_RATIO:-0.1}"
+LORA_RANK="${LORA_RANK:-32}"
+
+require_file() {
+  if [[ ! -f "$1" ]]; then
+    echo "Missing required file: $1" >&2
+    exit 2
+  fi
+}
+
+run_swift_train() {
+  local model_path="$1"
+  local train_type="$2"
+  local run_name="$3"
+  local output_dir="outputs/${run_name}"
+  local tb_dir="runs/${run_name}"
+  local log_file="logs/${run_name}.log"
+
+  require_file "${TRAIN_JSONL}"
+  require_file "${VAL_JSONL}"
+  mkdir -p "${output_dir}" "${tb_dir}" logs
+
+  local cmd=(
+    swift sft
+    --model "${model_path}"
+    --dataset "${TRAIN_JSONL}"
+    --val_dataset "${VAL_JSONL}"
+    --train_type "${train_type}"
+    --torch_dtype bfloat16
+    --num_train_epochs "${NUM_EPOCHS}"
+    --per_device_train_batch_size "${PER_DEVICE_BATCH_SIZE}"
+    --per_device_eval_batch_size 1
+    --gradient_accumulation_steps "${GRAD_ACCUM_STEPS}"
+    --learning_rate "${LEARNING_RATE}"
+    --warmup_ratio "${WARMUP_RATIO}"
+    --max_length "${MAX_LENGTH}"
+    --save_steps "${SAVE_STEPS}"
+    --eval_steps "${EVAL_STEPS}"
+    --logging_steps "${LOGGING_STEPS}"
+    --report_to tensorboard
+    --logging_dir "${tb_dir}"
+    --output_dir "${output_dir}"
+    --save_total_limit "${SAVE_TOTAL_LIMIT:-3}"
+    --dataloader_num_workers "${DATALOADER_NUM_WORKERS:-4}"
+  )
+
+  if [[ "${train_type}" == "lora" ]]; then
+    cmd+=(--lora_rank "${LORA_RANK}")
+  fi
+
+  printf '%q ' "${cmd[@]}" | tee "${log_file}.cmd"
+  echo
+  if [[ "${DRY_RUN:-0}" == "1" ]]; then
+    return 0
+  fi
+  "${cmd[@]}" 2>&1 | tee "${log_file}"
+}

scripts/tensorboard.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT_DIR}"
+source .venv/bin/activate
+tensorboard --logdir runs --host 0.0.0.0 --port "${TENSORBOARD_PORT:-6006}"

scripts/train_qwen35_9b_full.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/swift_train_common.sh"
+run_swift_train "${QWEN35_9B_MODEL_PATH:-models/qwen3.5-9b}" full qwen35_9b_full_bf16

scripts/train_qwen35_9b_lora.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/swift_train_common.sh"
+run_swift_train "${QWEN35_9B_MODEL_PATH:-models/qwen3.5-9b}" lora qwen35_9b_lora_r32

scripts/train_qwen36_27b_full.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/swift_train_common.sh"
+run_swift_train "${QWEN36_27B_MODEL_PATH:-models/qwen3.6-27b}" full qwen36_27b_full_bf16

scripts/train_qwen36_27b_lora.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/swift_train_common.sh"
+run_swift_train "${QWEN36_27B_MODEL_PATH:-models/qwen3.6-27b}" lora qwen36_27b_lora_r32

scripts/validate_setup.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${ROOT_DIR}"
+
+test -d third_party/modelscope-swift
+python3 -m py_compile scripts/download_dataset.py
+bash -n scripts/*.sh
+
+DRY_RUN=1 TRAIN_JSONL=/tmp/train.jsonl VAL_JSONL=/tmp/validation.jsonl bash -c '
+  echo "{}" > /tmp/train.jsonl
+  echo "{}" > /tmp/validation.jsonl
+  ./scripts/train_qwen35_9b_lora.sh
+  ./scripts/train_qwen35_9b_full.sh
+  ./scripts/train_qwen36_27b_lora.sh
+  ./scripts/train_qwen36_27b_full.sh
+'