35 lines
1.2 KiB
Bash
Executable File
35 lines
1.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT_DIR"
|
|
|
|
export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
|
|
export HF_HOME="${HF_HOME:-$ROOT_DIR/dataset_building/generated/.hf_home}"
|
|
export HF_DATASETS_CACHE="${HF_DATASETS_CACHE:-$ROOT_DIR/dataset_building/generated/.hf_datasets_cache}"
|
|
|
|
OUT="${OUT:-$ROOT_DIR/dataset_building/generated/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611}"
|
|
TOKENIZER="${TOKENIZER:-$ROOT_DIR/model_building/generated_models/Qwen3-0.6B-DSV4-tokenizer-remap-v2}"
|
|
TRAIN_TOTAL="${TRAIN_TOTAL:-1000000}"
|
|
TEST_PER_GROUP="${TEST_PER_GROUP:-400}"
|
|
SEED="${SEED:-20260611}"
|
|
MAX_SEEN_PER_SOURCE="${MAX_SEEN_PER_SOURCE:-300000}"
|
|
|
|
mkdir -p "$OUT" "$HF_HOME" "$HF_DATASETS_CACHE"
|
|
|
|
python dataset_building/build_training_and_test_mix_v3.py \
|
|
--out "$OUT" \
|
|
--tokenizer "$TOKENIZER" \
|
|
--train-total "$TRAIN_TOTAL" \
|
|
--test-per-group "$TEST_PER_GROUP" \
|
|
--seed "$SEED" \
|
|
--hf-endpoint "$HF_ENDPOINT" \
|
|
--use-hf \
|
|
--allow-shortfall \
|
|
--trust-metadata-lengths \
|
|
--max-seen-per-source "$MAX_SEEN_PER_SOURCE"
|
|
|
|
echo "Wrote SFT heldout and train mix to: $OUT"
|
|
echo "Heldout: $OUT/heldout_2p8k.jsonl"
|
|
echo "Exclusion hashes: $OUT/heldout_exclusion_hashes.jsonl"
|