33 lines
1.2 KiB
Bash
Executable File
33 lines
1.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SRC="${SRC:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
|
|
DST="${DST:-/mnt/beegfs/yi/laoyao_2b_moe/dataset/pretrain/data/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
|
|
LOG="${LOG:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/logs/wait_build_rebalanced_200b_20260701_130058.log}"
|
|
|
|
mkdir -p "$(dirname "$DST")"
|
|
|
|
if pgrep -f "build_rebalanced_pretrain_dataset.py --target-tokens 200000000000" >/dev/null; then
|
|
echo "pretrain builder is still running; not syncing yet" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [ ! -d "$SRC" ]; then
|
|
echo "source dataset directory does not exist: $SRC" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! grep -q "rebalanced_pretrain_done\|completed\|wrote manifest\|finished" "$LOG" 2>/dev/null; then
|
|
echo "warning: completion marker not found in log; syncing existing directory anyway" >&2
|
|
fi
|
|
|
|
rm -rf "$DST"
|
|
mkdir -p "$DST"
|
|
if cp -al "$SRC"/. "$DST"/ 2>/tmp/laoyao_sync_cp_al.err; then
|
|
echo "synced with hardlinks: $DST"
|
|
else
|
|
echo "hardlink copy failed, falling back to rsync copy" >&2
|
|
cat /tmp/laoyao_sync_cp_al.err >&2 || true
|
|
rsync -a --info=progress2 "$SRC"/ "$DST"/
|
|
fi
|