From 069f891ceb516405ed5f6801b2c282052c09ae2a Mon Sep 17 00:00:00 2001
From: Tom Lu <Luyi1720839132@Gmail.com>
Date: Wed, 1 Jul 2026 15:23:25 +0800
Subject: [PATCH] Add SFT heldout 2.8K build entrypoint

---
 dataset_building/README.md                    | 38 +++++++++++++++++--
 .../build_sft_heldout_2p8k_20260611.sh        | 34 +++++++++++++++++
 2 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100755 dataset_building/build_sft_heldout_2p8k_20260611.sh

diff --git a/dataset_building/README.md b/dataset_building/README.md
index 5a3f439..8173c98 100644
--- a/dataset_building/README.md
+++ b/dataset_building/README.md
@@ -16,16 +16,48 @@ dataset_building/generated/
 
 That directory is ignored by git.
 
-## Kept Dataset
+## Kept Datasets
 
-Only the latest public heldout 2K set is versioned here:
+The latest public heldout 2K set is versioned here:
 
 ```text
 heldout_public_mcq_2k_20260607/heldout_public_mcq_2k.jsonl
 heldout_public_mcq_2k_20260607/heldout_public_mcq_2k_stats.json
 ```
 
-Older in-domain heldout and earlier ratio-imbalanced 2K heldout datasets are intentionally not included.
+The 2.8K in-domain SFT heldout is kept as a reproducible build recipe rather
+than checked-in generated data. It contains seven ability dimensions with 400
+examples each:
+
+```text
+science_reasoning
+logic
+code
+chinese_exam
+math
+chinese_dialogue
+english_dialogue
+```
+
+Build it with:
+
+```bash
+bash dataset_building/build_sft_heldout_2p8k_20260611.sh
+```
+
+By default this writes:
+
+```text
+dataset_building/generated/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/
+  heldout_2p8k.jsonl
+  heldout_exclusion_hashes.jsonl
+  train_1m.jsonl
+  build_stats.json
+```
+
+Use `OUT=/path/to/output` and `TOKENIZER=/path/to/tokenizer` to place the
+validation set outside the repo or build it with a specific tokenizer. Generated
+outputs stay ignored by git.
 
 ## SFT Builders
 
diff --git a/dataset_building/build_sft_heldout_2p8k_20260611.sh b/dataset_building/build_sft_heldout_2p8k_20260611.sh
new file mode 100755
index 0000000..ead751a
--- /dev/null
+++ b/dataset_building/build_sft_heldout_2p8k_20260611.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
+export HF_HOME="${HF_HOME:-$ROOT_DIR/dataset_building/generated/.hf_home}"
+export HF_DATASETS_CACHE="${HF_DATASETS_CACHE:-$ROOT_DIR/dataset_building/generated/.hf_datasets_cache}"
+
+OUT="${OUT:-$ROOT_DIR/dataset_building/generated/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611}"
+TOKENIZER="${TOKENIZER:-$ROOT_DIR/model_building/generated_models/Qwen3-0.6B-DSV4-tokenizer-remap-v2}"
+TRAIN_TOTAL="${TRAIN_TOTAL:-1000000}"
+TEST_PER_GROUP="${TEST_PER_GROUP:-400}"
+SEED="${SEED:-20260611}"
+MAX_SEEN_PER_SOURCE="${MAX_SEEN_PER_SOURCE:-300000}"
+
+mkdir -p "$OUT" "$HF_HOME" "$HF_DATASETS_CACHE"
+
+python dataset_building/build_training_and_test_mix_v3.py \
+  --out "$OUT" \
+  --tokenizer "$TOKENIZER" \
+  --train-total "$TRAIN_TOTAL" \
+  --test-per-group "$TEST_PER_GROUP" \
+  --seed "$SEED" \
+  --hf-endpoint "$HF_ENDPOINT" \
+  --use-hf \
+  --allow-shortfall \
+  --trust-metadata-lengths \
+  --max-seen-per-source "$MAX_SEEN_PER_SOURCE"
+
+echo "Wrote SFT heldout and train mix to: $OUT"
+echo "Heldout: $OUT/heldout_2p8k.jsonl"
+echo "Exclusion hashes: $OUT/heldout_exclusion_hashes.jsonl"