215 lines
11 KiB
JSON
215 lines
11 KiB
JSON
{
|
|
"scale": 5.0,
|
|
"budgets": {
|
|
"english_web": 1250000000,
|
|
"english_edu": 1000000000,
|
|
"chinese_clean": 1250000000,
|
|
"code": 750000000,
|
|
"math": 500000000,
|
|
"science": 150000000,
|
|
"qa_as_text": 100000000
|
|
},
|
|
"tokens_by_category": {
|
|
"english_web": 1250000445,
|
|
"english_edu": 1000000462,
|
|
"code": 750001624,
|
|
"math": 96992014,
|
|
"science": 30016910,
|
|
"qa_as_text": 100000055
|
|
},
|
|
"docs_by_category": {
|
|
"english_web": 1571857,
|
|
"english_edu": 973081,
|
|
"code": 589463,
|
|
"math": 51576,
|
|
"science": 48872,
|
|
"qa_as_text": 244246
|
|
},
|
|
"tokens_by_source": {
|
|
"data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_000_00000.parquet": 684777289,
|
|
"data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00000.parquet": 627654914,
|
|
"data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_001_00000.parquet": 565223156,
|
|
"data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00001.parquet": 372345548,
|
|
"data/raw_parquets/starcoder/python/train-00000-of-00059.parquet": 238778818,
|
|
"data/raw_parquets/starcoder/python/train-00002-of-00059.parquet": 233459009,
|
|
"data/raw_parquets/starcoder/python/train-00001-of-00059.parquet": 233210510,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl": 100000055,
|
|
"open-web-math/open-web-math": 96992014,
|
|
"data/raw_parquets/starcoder/python/train-00003-of-00059.parquet": 44553287,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__train.jsonl": 10027949,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__train.jsonl": 6138410,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__train.jsonl": 3119670,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__test.jsonl": 1837710,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__GBaker_MedQA-USMLE-4-options__train.jsonl": 1572096,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__train.jsonl": 1182676,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__test.jsonl": 1033827,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__validation.jsonl": 1032383,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__validation.jsonl": 895384,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_qasc__train.jsonl": 688074,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__test.jsonl": 307416,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_folio__train.jsonl": 261306,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__validation.jsonl": 242852,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__train.jsonl": 221520,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__GBaker_MedQA-USMLE-4-options__test.jsonl": 203198,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__temporal_sequences__train.jsonl": 141867,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__test.jsonl": 139093,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__train.jsonl": 130752,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_deduction__train.jsonl": 118299,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__test.jsonl": 102393,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__validation.jsonl": 99297,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__test.jsonl": 79458,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_qasc__validation.jsonl": 79086,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__train.jsonl": 74207,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_folio__validation.jsonl": 53607,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__temporal_sequences__validation.jsonl": 35418,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__validation.jsonl": 33149,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_deduction__validation.jsonl": 31008,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__validation.jsonl": 23331,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__test.jsonl": 22745,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__qiaojin_PubMedQA__pqa_labeled__train.jsonl": 22169,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__validation.jsonl": 19963,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__disambiguation_qa__train.jsonl": 15771,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__date_understanding__train.jsonl": 14547,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__disambiguation_qa__validation.jsonl": 3843,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__date_understanding__validation.jsonl": 3629,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_args__validation.jsonl": 3136,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_args__train.jsonl": 2729,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__analytic_entailment__train.jsonl": 2277,
|
|
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__analytic_entailment__validation.jsonl": 665
|
|
},
|
|
"rejected": {
|
|
"english_web": {
|
|
"too_short": 168039,
|
|
"too_long": 179
|
|
},
|
|
"english_edu": {
|
|
"too_short": 18375,
|
|
"too_long": 221
|
|
},
|
|
"code": {
|
|
"too_short": 106203,
|
|
"too_long": 816
|
|
},
|
|
"math": {
|
|
"too_short": 3669,
|
|
"too_long": 152
|
|
},
|
|
"science": {
|
|
"pack_too_short": 5591
|
|
},
|
|
"qa_as_text": {
|
|
"too_short": 108331
|
|
}
|
|
},
|
|
"sources": [
|
|
{
|
|
"category": "english_web",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_000_00000.parquet",
|
|
"rows_seen": 961000,
|
|
"docs_written": 864351,
|
|
"tokens": 684777289,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "english_web",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_001_00000.parquet",
|
|
"rows_seen": 779076,
|
|
"docs_written": 707506,
|
|
"tokens": 565223156,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "english_edu",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00000.parquet",
|
|
"rows_seen": 621759,
|
|
"docs_written": 610163,
|
|
"tokens": 627654914,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "english_edu",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00001.parquet",
|
|
"rows_seen": 369919,
|
|
"docs_written": 362918,
|
|
"tokens": 372345548,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "chinese_clean",
|
|
"kind": "hf_stream",
|
|
"source": "BAAI/CCI3-HQ",
|
|
"rows_seen": 0,
|
|
"docs_written": 0,
|
|
"tokens": 0,
|
|
"error": "DatasetNotFoundError(\"Dataset 'BAAI/CCI3-HQ' is a gated dataset on the Hub. You must be authenticated to access it.\")"
|
|
},
|
|
{
|
|
"category": "chinese_clean",
|
|
"kind": "hf_stream",
|
|
"source": "Skywork/SkyPile-150B",
|
|
"rows_seen": 0,
|
|
"docs_written": 0,
|
|
"tokens": 0,
|
|
"error": "RuntimeError('Cannot send a request, as the client has been closed.')"
|
|
},
|
|
{
|
|
"category": "code",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/starcoder/python/train-00000-of-00059.parquet",
|
|
"rows_seen": 218079,
|
|
"docs_written": 184958,
|
|
"tokens": 238778818,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "code",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/starcoder/python/train-00001-of-00059.parquet",
|
|
"rows_seen": 218079,
|
|
"docs_written": 184495,
|
|
"tokens": 233210510,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "code",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/starcoder/python/train-00002-of-00059.parquet",
|
|
"rows_seen": 218079,
|
|
"docs_written": 184331,
|
|
"tokens": 233459009,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "code",
|
|
"kind": "local_parquet",
|
|
"source": "data/raw_parquets/starcoder/python/train-00003-of-00059.parquet",
|
|
"rows_seen": 42246,
|
|
"docs_written": 35679,
|
|
"tokens": 44553287,
|
|
"error": ""
|
|
},
|
|
{
|
|
"category": "math",
|
|
"kind": "hf_stream",
|
|
"source": "open-web-math/open-web-math",
|
|
"rows_seen": 55397,
|
|
"docs_written": 51576,
|
|
"tokens": 96992014,
|
|
"error": "ProxyError('503 Service Unavailable')"
|
|
},
|
|
{
|
|
"category": "math",
|
|
"kind": "hf_stream",
|
|
"source": "GAIR/MathPile",
|
|
"rows_seen": 0,
|
|
"docs_written": 0,
|
|
"tokens": 0,
|
|
"error": "DatasetNotFoundError(\"Dataset 'GAIR/MathPile' is a gated dataset on the Hub. You must be authenticated to access it.\")"
|
|
}
|
|
],
|
|
"elapsed_sec": 10098.068085193634
|
|
} |