{ "scale": 5.0, "budgets": { "english_web": 1250000000, "english_edu": 1000000000, "chinese_clean": 1250000000, "code": 750000000, "math": 500000000, "science": 150000000, "qa_as_text": 100000000 }, "tokens_by_category": { "english_web": 1250000445, "english_edu": 1000000462, "code": 750001624, "math": 96992014, "science": 30016910, "qa_as_text": 100000055 }, "docs_by_category": { "english_web": 1571857, "english_edu": 973081, "code": 589463, "math": 51576, "science": 48872, "qa_as_text": 244246 }, "tokens_by_source": { "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_000_00000.parquet": 684777289, "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00000.parquet": 627654914, "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_001_00000.parquet": 565223156, "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00001.parquet": 372345548, "data/raw_parquets/starcoder/python/train-00000-of-00059.parquet": 238778818, "data/raw_parquets/starcoder/python/train-00002-of-00059.parquet": 233459009, "data/raw_parquets/starcoder/python/train-00001-of-00059.parquet": 233210510, "/ssd/yi/tokenizer_swap_cepe/data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl": 100000055, "open-web-math/open-web-math": 96992014, "data/raw_parquets/starcoder/python/train-00003-of-00059.parquet": 44553287, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__train.jsonl": 10027949, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__train.jsonl": 6138410, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__train.jsonl": 3119670, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__test.jsonl": 1837710, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__GBaker_MedQA-USMLE-4-options__train.jsonl": 1572096, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__train.jsonl": 1182676, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__test.jsonl": 1033827, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__validation.jsonl": 1032383, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__validation.jsonl": 895384, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_qasc__train.jsonl": 688074, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__test.jsonl": 307416, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_folio__train.jsonl": 261306, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__validation.jsonl": 242852, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__train.jsonl": 221520, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__GBaker_MedQA-USMLE-4-options__test.jsonl": 203198, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__temporal_sequences__train.jsonl": 141867, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__test.jsonl": 139093, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__train.jsonl": 130752, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_deduction__train.jsonl": 118299, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__test.jsonl": 102393, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__validation.jsonl": 99297, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__test.jsonl": 79458, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_qasc__validation.jsonl": 79086, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__train.jsonl": 74207, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_folio__validation.jsonl": 53607, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__temporal_sequences__validation.jsonl": 35418, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__validation.jsonl": 33149, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_deduction__validation.jsonl": 31008, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__validation.jsonl": 23331, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__test.jsonl": 22745, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__qiaojin_PubMedQA__pqa_labeled__train.jsonl": 22169, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__validation.jsonl": 19963, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__disambiguation_qa__train.jsonl": 15771, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__date_understanding__train.jsonl": 14547, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__disambiguation_qa__validation.jsonl": 3843, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__date_understanding__validation.jsonl": 3629, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_args__validation.jsonl": 3136, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_args__train.jsonl": 2729, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__analytic_entailment__train.jsonl": 2277, "/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__analytic_entailment__validation.jsonl": 665 }, "rejected": { "english_web": { "too_short": 168039, "too_long": 179 }, "english_edu": { "too_short": 18375, "too_long": 221 }, "code": { "too_short": 106203, "too_long": 816 }, "math": { "too_short": 3669, "too_long": 152 }, "science": { "pack_too_short": 5591 }, "qa_as_text": { "too_short": 108331 } }, "sources": [ { "category": "english_web", "kind": "local_parquet", "source": "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_000_00000.parquet", "rows_seen": 961000, "docs_written": 864351, "tokens": 684777289, "error": "" }, { "category": "english_web", "kind": "local_parquet", "source": "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_001_00000.parquet", "rows_seen": 779076, "docs_written": 707506, "tokens": 565223156, "error": "" }, { "category": "english_edu", "kind": "local_parquet", "source": "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00000.parquet", "rows_seen": 621759, "docs_written": 610163, "tokens": 627654914, "error": "" }, { "category": "english_edu", "kind": "local_parquet", "source": "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00001.parquet", "rows_seen": 369919, "docs_written": 362918, "tokens": 372345548, "error": "" }, { "category": "chinese_clean", "kind": "hf_stream", "source": "BAAI/CCI3-HQ", "rows_seen": 0, "docs_written": 0, "tokens": 0, "error": "DatasetNotFoundError(\"Dataset 'BAAI/CCI3-HQ' is a gated dataset on the Hub. You must be authenticated to access it.\")" }, { "category": "chinese_clean", "kind": "hf_stream", "source": "Skywork/SkyPile-150B", "rows_seen": 0, "docs_written": 0, "tokens": 0, "error": "RuntimeError('Cannot send a request, as the client has been closed.')" }, { "category": "code", "kind": "local_parquet", "source": "data/raw_parquets/starcoder/python/train-00000-of-00059.parquet", "rows_seen": 218079, "docs_written": 184958, "tokens": 238778818, "error": "" }, { "category": "code", "kind": "local_parquet", "source": "data/raw_parquets/starcoder/python/train-00001-of-00059.parquet", "rows_seen": 218079, "docs_written": 184495, "tokens": 233210510, "error": "" }, { "category": "code", "kind": "local_parquet", "source": "data/raw_parquets/starcoder/python/train-00002-of-00059.parquet", "rows_seen": 218079, "docs_written": 184331, "tokens": 233459009, "error": "" }, { "category": "code", "kind": "local_parquet", "source": "data/raw_parquets/starcoder/python/train-00003-of-00059.parquet", "rows_seen": 42246, "docs_written": 35679, "tokens": 44553287, "error": "" }, { "category": "math", "kind": "hf_stream", "source": "open-web-math/open-web-math", "rows_seen": 55397, "docs_written": 51576, "tokens": 96992014, "error": "ProxyError('503 Service Unavailable')" }, { "category": "math", "kind": "hf_stream", "source": "GAIR/MathPile", "rows_seen": 0, "docs_written": 0, "tokens": 0, "error": "DatasetNotFoundError(\"Dataset 'GAIR/MathPile' is a gated dataset on the Hub. You must be authenticated to access it.\")" } ], "elapsed_sec": 10098.068085193634 }