#!/usr/bin/env bash
set -euo pipefail

SRC="${SRC:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
DST="${DST:-/mnt/beegfs/yi/laoyao_2b_moe/dataset/pretrain/data/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
LOG="${LOG:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/logs/wait_build_rebalanced_200b_20260701_130058.log}"

mkdir -p "$(dirname "$DST")"

if pgrep -f "build_rebalanced_pretrain_dataset.py --target-tokens 200000000000" >/dev/null; then
  echo "pretrain builder is still running; not syncing yet" >&2
  exit 2
fi

if [ ! -d "$SRC" ]; then
  echo "source dataset directory does not exist: $SRC" >&2
  exit 1
fi

if ! grep -q "rebalanced_pretrain_done\|completed\|wrote manifest\|finished" "$LOG" 2>/dev/null; then
  echo "warning: completion marker not found in log; syncing existing directory anyway" >&2
fi

rm -rf "$DST"
mkdir -p "$DST"
if cp -al "$SRC"/. "$DST"/ 2>/tmp/laoyao_sync_cp_al.err; then
  echo "synced with hardlinks: $DST"
else
  echo "hardlink copy failed, falling back to rsync copy" >&2
  cat /tmp/laoyao_sync_cp_al.err >&2 || true
  rsync -a --info=progress2 "$SRC"/ "$DST"/
fi