Files
laoyao_2b_moe/dataset/pretrain/scripts/download_rebalanced_sources.py
T

464 lines
17 KiB
Python

#!/usr/bin/env python
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import random
import time
from typing import Iterable
import urllib.error
import urllib.parse
import urllib.request
from huggingface_hub import HfApi
DEFAULT_DATASETS = {
"english_web_fineweb": {
"repo_id": "HuggingFaceFW/fineweb",
"patterns": (".parquet",),
"default_max_files": 256,
},
"english_edu_fineweb_edu": {
"repo_id": "HuggingFaceFW/fineweb-edu",
"patterns": (".parquet",),
"default_max_files": 256,
},
"chinese_cci3_hq": {
"repo_id": "BAAI/CCI3-HQ",
"patterns": (".jsonl", ".parquet"),
"default_max_files": 128,
},
"chinese_skypile_150b": {
"repo_id": "Skywork/SkyPile-150B",
"patterns": (".jsonl", ".parquet"),
"default_max_files": 128,
},
"science_medmcqa": {
"repo_id": "openlifescienceai/medmcqa",
"patterns": (".parquet",),
"default_max_files": None,
},
"science_proofwriter": {
"repo_id": "tasksource/proofwriter",
"patterns": (".parquet",),
"default_max_files": None,
},
"science_scienceqa": {
"repo_id": "derek-thomas/ScienceQA",
"patterns": (".parquet",),
"default_max_files": None,
},
"science_sciq": {
"repo_id": "allenai/sciq",
"patterns": (".parquet",),
"default_max_files": None,
},
"science_qasc": {
"repo_id": "allenai/qasc",
"patterns": (".parquet",),
"default_max_files": None,
},
"science_openbookqa": {
"repo_id": "allenai/openbookqa",
"patterns": (".parquet",),
"default_max_files": None,
},
"ms_fineweb_edu": {
"provider": "modelscope",
"repo_id": "AI-ModelScope/fineweb-edu",
"roots": ("data",),
"patterns": (".parquet",),
"default_max_files": 96,
},
"ms_ultra_fineweb_en": {
"provider": "modelscope",
"repo_id": "OpenBMB/Ultra-FineWeb",
"roots": ("data/ultrafineweb_en",),
"patterns": (".parquet",),
"default_max_files": 128,
},
"ms_ultra_fineweb_zh": {
"provider": "modelscope",
"repo_id": "OpenBMB/Ultra-FineWeb",
"roots": ("data/ultrafineweb_zh",),
"patterns": (".parquet",),
"default_max_files": 32,
},
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Download rebalanced pretraining source datasets via hf-mirror.")
parser.add_argument("--output-root", default="/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset")
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--endpoint", default=os.environ.get("HF_ENDPOINT", "https://hf-mirror.com"))
parser.add_argument("--modelscope-endpoint", default=os.environ.get("MODELSCOPE_ENDPOINT", "https://modelscope.cn"))
parser.add_argument("--token", default=os.environ.get("HF_TOKEN"))
parser.add_argument("--include", action="append", default=[], help="Dataset key to include. Repeatable.")
parser.add_argument("--exclude", action="append", default=[], help="Dataset key to skip. Repeatable.")
parser.add_argument("--max-files", action="append", default=[], metavar="KEY=N")
parser.add_argument("--manifest-name", default="download_manifest.jsonl")
parser.add_argument("--socket-timeout", type=float, default=60.0)
parser.add_argument(
"--max-mbps",
type=float,
default=0.0,
help="Per-process download bandwidth cap in MiB/s. 0 disables throttling.",
)
parser.add_argument("--dry-run", action="store_true")
return parser.parse_args()
def main() -> None:
args = parse_args()
if not args.token:
raise ValueError("HF_TOKEN is required")
output_root = Path(args.output_root)
output_root.mkdir(parents=True, exist_ok=True)
max_files = parse_max_files(args.max_files)
selected_keys = set(args.include) if args.include else set(DEFAULT_DATASETS)
selected_keys -= set(args.exclude)
unknown = selected_keys - set(DEFAULT_DATASETS)
if unknown:
raise ValueError(f"unknown dataset keys: {sorted(unknown)}")
api = HfApi(endpoint=args.endpoint, token=args.token)
manifest_path = output_root / args.manifest_name
completed = load_completed_downloads(manifest_path)
for key in sorted(selected_keys):
spec = DEFAULT_DATASETS[key]
repo_id = spec["repo_id"]
provider = spec.get("provider", "hf")
limit = max_files.get(key, spec["default_max_files"])
if provider == "modelscope":
files = list_modelscope_dataset_files(
endpoint=args.modelscope_endpoint,
repo_id=repo_id,
roots=spec.get("roots", ("",)),
suffixes=spec["patterns"],
max_files=limit,
seed=args.seed + stable_int(key),
)
else:
files = list_dataset_files(api, repo_id=repo_id, suffixes=spec["patterns"])
random.Random(args.seed + stable_int(key)).shuffle(files)
if limit is not None:
files = files[:limit]
dataset_dir = output_root / key
dataset_dir.mkdir(parents=True, exist_ok=True)
active_endpoint = args.modelscope_endpoint if provider == "modelscope" else args.endpoint
print(
"download_dataset_start "
f"key={key} repo_id={repo_id} provider={provider} files={len(files)} "
f"output={dataset_dir} endpoint={active_endpoint}",
flush=True,
)
for index, filename in enumerate(files, start=1):
completed_path = completed.get((key, filename))
if completed_path is not None and completed_path.exists():
record = {
"time": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"key": key,
"repo_id": repo_id,
"filename": filename,
"index": index,
"total": len(files),
"status": "skipped_existing",
"elapsed_sec": 0.0,
"local_path": str(completed_path),
"seed": args.seed,
"error": "",
}
append_jsonl(manifest_path, record)
print(
"download_file "
f"key={key} index={index}/{len(files)} status=skipped_existing file={filename}",
flush=True,
)
continue
started_at = time.time()
status = "ok"
error = ""
local_path = ""
if args.dry_run:
status = "dry_run"
local_path = str(dataset_dir / filename)
else:
try:
local_path = download_file_with_resume(
repo_id=repo_id,
filename=filename,
output_dir=dataset_dir,
token=args.token,
endpoint=args.modelscope_endpoint if provider == "modelscope" else args.endpoint,
provider=provider,
timeout=args.socket_timeout,
max_mbps=args.max_mbps,
)
except Exception as exc: # keep overnight batch moving
status = "error"
error = f"{type(exc).__name__}: {exc}"
elapsed = time.time() - started_at
record = {
"time": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
"key": key,
"repo_id": repo_id,
"provider": provider,
"filename": filename,
"index": index,
"total": len(files),
"status": status,
"elapsed_sec": elapsed,
"local_path": local_path,
"seed": args.seed,
"error": error,
}
append_jsonl(manifest_path, record)
print(
"download_file "
f"key={key} index={index}/{len(files)} status={status} file={filename} elapsed={elapsed:.1f}s",
flush=True,
)
print(f"download_dataset_done key={key} files={len(files)}", flush=True)
def list_dataset_files(api: HfApi, *, repo_id: str, suffixes: Iterable[str]) -> list[str]:
suffixes = tuple(suffixes)
info = api.dataset_info(repo_id)
files = []
for sibling in info.siblings or []:
name = sibling.rfilename
if name.endswith(suffixes):
files.append(name)
if not files:
raise FileNotFoundError(f"no files matched {suffixes} in {repo_id}")
return sorted(files)
def download_file_with_resume(
*,
repo_id: str,
filename: str,
output_dir: Path,
token: str,
endpoint: str,
provider: str,
timeout: float,
max_mbps: float,
) -> str:
target = output_dir / filename
target.parent.mkdir(parents=True, exist_ok=True)
url = resolve_url(endpoint=endpoint, repo_id=repo_id, filename=filename, provider=provider)
remote_size = remote_content_length(url, token=token, timeout=timeout)
if target.exists() and remote_size is not None and target.stat().st_size == remote_size:
return str(target)
incomplete = target.with_name(target.name + ".incomplete")
offset = incomplete.stat().st_size if incomplete.exists() else 0
if remote_size is not None and offset > remote_size:
incomplete.unlink()
offset = 0
headers = {"Authorization": f"Bearer {token}"}
if offset:
headers["Range"] = f"bytes={offset}-"
request = urllib.request.Request(url, headers=headers)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
mode = "ab" if offset and response.status == 206 else "wb"
throttle = DownloadThrottle(max_mbps)
with incomplete.open(mode) as handle:
while True:
chunk = response.read(1024 * 1024)
if not chunk:
break
handle.write(chunk)
throttle.wait(len(chunk))
except urllib.error.HTTPError as exc:
if exc.code == 416 and remote_size is not None and incomplete.exists() and incomplete.stat().st_size == remote_size:
incomplete.replace(target)
return str(target)
raise
if remote_size is not None and incomplete.stat().st_size != remote_size:
raise IOError(f"incomplete download for {filename}: got {incomplete.stat().st_size}, expected {remote_size}")
incomplete.replace(target)
return str(target)
class DownloadThrottle:
def __init__(self, max_mbps: float) -> None:
self.max_bytes_per_sec = max(0.0, max_mbps) * 1024 * 1024
self.started_at = time.monotonic()
self.bytes_seen = 0
def wait(self, bytes_read: int) -> None:
if self.max_bytes_per_sec <= 0:
return
self.bytes_seen += bytes_read
expected_elapsed = self.bytes_seen / self.max_bytes_per_sec
actual_elapsed = time.monotonic() - self.started_at
sleep_sec = expected_elapsed - actual_elapsed
if sleep_sec > 0:
time.sleep(sleep_sec)
def resolve_url(*, endpoint: str, repo_id: str, filename: str, provider: str) -> str:
base = endpoint.rstrip("/")
quoted_filename = urllib.parse.quote(filename, safe="/")
if provider == "modelscope":
return f"{base}/datasets/{repo_id}/resolve/master/{quoted_filename}"
return f"{base}/datasets/{repo_id}/resolve/main/{quoted_filename}"
def remote_content_length(url: str, *, token: str, timeout: float) -> int | None:
request = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}, method="HEAD")
with urllib.request.urlopen(request, timeout=timeout) as response:
raw = response.headers.get("Content-Length")
return int(raw) if raw and raw.isdigit() else None
def list_modelscope_dataset_files(
*,
endpoint: str,
repo_id: str,
roots: Iterable[str],
suffixes: Iterable[str],
max_files: int | None,
seed: int,
) -> list[str]:
files: list[str] = []
suffixes = tuple(suffixes)
for root in roots:
remaining = None if max_files is None else max_files - len(files)
if remaining is not None and remaining <= 0:
break
files.extend(
_list_modelscope_tree(
endpoint=endpoint,
repo_id=repo_id,
root=root,
suffixes=suffixes,
max_files=remaining,
seed=seed + stable_int(root),
)
)
if not files:
raise FileNotFoundError(f"no ModelScope files matched {suffixes} in {repo_id} roots={tuple(roots)}")
return sorted(set(files))
def _list_modelscope_tree(
*,
endpoint: str,
repo_id: str,
root: str,
suffixes: tuple[str, ...],
max_files: int | None,
seed: int,
) -> list[str]:
files: list[str] = []
page = 1
page_size = 500
trees: list[str] = []
while True:
query = urllib.parse.urlencode(
{
"Revision": "master",
"Root": root,
"PageNumber": page,
"PageSize": page_size,
}
)
url = f"{endpoint.rstrip('/')}/api/v1/datasets/{repo_id}/repo/tree?{query}"
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0", "Accept": "application/json"})
with urllib.request.urlopen(request, timeout=60) as response:
payload = json.load(response)
data = payload.get("Data")
entries = data.get("Files") if isinstance(data, dict) else data
if not entries:
break
for entry in entries:
path = entry.get("Path")
entry_type = entry.get("Type")
if entry_type == "blob" and isinstance(path, str) and path.endswith(suffixes):
files.append(path)
if max_files is not None and len(files) >= max_files:
return files
elif entry_type == "tree" and isinstance(path, str):
trees.append(path)
total = int(payload.get("TotalCount") or (data or {}).get("TotalCount") or 0)
if page * page_size >= total:
break
page += 1
random.Random(seed).shuffle(trees)
for tree_root in trees:
remaining = None if max_files is None else max_files - len(files)
if remaining is not None and remaining <= 0:
break
files.extend(
_list_modelscope_tree(
endpoint=endpoint,
repo_id=repo_id,
root=tree_root,
suffixes=suffixes,
max_files=remaining,
seed=seed + stable_int(tree_root),
)
)
return files
def parse_max_files(items: list[str]) -> dict[str, int | None]:
parsed: dict[str, int | None] = {}
for item in items:
if "=" not in item:
raise ValueError(f"--max-files must use KEY=N: {item!r}")
key, raw = item.split("=", 1)
key = key.strip()
raw = raw.strip().lower()
if raw in {"all", "none", "-1"}:
parsed[key] = None
else:
value = int(raw)
if value <= 0:
raise ValueError(f"max files must be positive: {item!r}")
parsed[key] = value
return parsed
def append_jsonl(path: Path, record: dict[str, object]) -> None:
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
def load_completed_downloads(path: Path) -> dict[tuple[str, str], Path]:
completed: dict[tuple[str, str], Path] = {}
if not path.exists():
return completed
with path.open("r", encoding="utf-8") as handle:
for line in handle:
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if record.get("status") != "ok":
continue
key = record.get("key")
filename = record.get("filename")
local_path = record.get("local_path")
if isinstance(key, str) and isinstance(filename, str) and isinstance(local_path, str):
completed[(key, filename)] = Path(local_path)
return completed
def stable_int(value: str) -> int:
import hashlib
digest = hashlib.sha256(value.encode("utf-8")).digest()
return int.from_bytes(digest[:4], "big")
if __name__ == "__main__":
main()