yi_lu/laoyao_2b_moe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add proof-pile logic inputs to pretrain builder

Browse Source
This commit is contained in:
路易
2026-07-01 22:42:18 +08:00
parent cdd8e2f85b
commit f0f72767f5
11 changed files with 122 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
scripts/sync_pretrain_data_into_repo.sh
View File

@@ -1,32 +1,15 @@
#!/usr/bin/env bash
set -euo pipefail
SRC="${SRC:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
DST="${DST:-/mnt/beegfs/yi/laoyao_2b_moe/dataset/pretrain/data/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
LOG="${LOG:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/logs/wait_build_rebalanced_200b_20260701_130058.log}"
SOURCE_DATA="${SOURCE_DATA:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
mkdir -p "$(dirname "$DST")"
echo "Large pretraining data is intentionally not copied into this repo."
echo "Use source data path directly: $SOURCE_DATA"
if pgrep -f "build_rebalanced_pretrain_dataset.py --target-tokens 200000000000" >/dev/null; then
echo "pretrain builder is still running; not syncing yet" >&2
exit 2
fi
if [ ! -d "$SRC" ]; then
echo "source dataset directory does not exist: $SRC" >&2
if [ ! -d "$SOURCE_DATA" ]; then
echo "missing source data: $SOURCE_DATA" >&2
exit 1
fi
if ! grep -q "rebalanced_pretrain_done\|completed\|wrote manifest\|finished" "$LOG" 2>/dev/null; then
echo "warning: completion marker not found in log; syncing existing directory anyway" >&2
fi
rm -rf "$DST"
mkdir -p "$DST"
if cp -al "$SRC"/. "$DST"/ 2>/tmp/laoyao_sync_cp_al.err; then
echo "synced with hardlinks: $DST"
else
echo "hardlink copy failed, falling back to rsync copy" >&2
cat /tmp/laoyao_sync_cp_al.err >&2 || true
rsync -a --info=progress2 "$SRC"/ "$DST"/
fi
du -sh "$SOURCE_DATA" || true
find "$SOURCE_DATA" -maxdepth 1 -type f | wc -l | awk "{print \"file_count=\" \$1}"

4
scripts/train_nemo_megatron_2b_moe.sh
View File

@@ -4,10 +4,10 @@ set -euo pipefail
REPO_ROOT="${REPO_ROOT:-/mnt/beegfs/yi/laoyao_2b_moe}"
IMAGE="${IMAGE:-nvcr.io/nvidia/nemo:26.06}"
CONFIG="${CONFIG:-$REPO_ROOT/training/nemo_megatron/pretrain_2b_moe_200b.yaml}"
DATA_DIR="$REPO_ROOT/dataset/pretrain/data/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701"
DATA_DIR="/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701"
if [ ! -d "$DATA_DIR" ]; then
echo "pretrain data is missing under repo; run scripts/sync_pretrain_data_into_repo.sh after the 200B build finishes" >&2
echo "pretrain source data is missing; check dataset/pretrain/manifests/rebalanced_182b_20260701_summary.json" >&2
exit 1
fi