16 lines
547 B
Bash
Executable File
16 lines
547 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SOURCE_DATA="${SOURCE_DATA:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"
|
|
|
|
echo "Large pretraining data is intentionally not copied into this repo."
|
|
echo "Use source data path directly: $SOURCE_DATA"
|
|
|
|
if [ ! -d "$SOURCE_DATA" ]; then
|
|
echo "missing source data: $SOURCE_DATA" >&2
|
|
exit 1
|
|
fi
|
|
|
|
du -sh "$SOURCE_DATA" || true
|
|
find "$SOURCE_DATA" -maxdepth 1 -type f | wc -l | awk "{print \"file_count=\" \$1}"
|