#!/usr/bin/env bash
set -euo pipefail

SOURCE_DATA="${SOURCE_DATA:-/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701}"

echo "Large pretraining data is intentionally not copied into this repo."
echo "Use source data path directly: $SOURCE_DATA"

if [ ! -d "$SOURCE_DATA" ]; then
  echo "missing source data: $SOURCE_DATA" >&2
  exit 1
fi

du -sh "$SOURCE_DATA" || true
find "$SOURCE_DATA" -maxdepth 1 -type f | wc -l | awk "{print \"file_count=\" \$1}"
