Clean tokenizer swap migration
This commit is contained in:
Codex
@@ -0,0 +1,118 @@
|
||||
{
|
||||
"tokenizer": "/ssd/yi/tokenizer_swap_cepe/models/Qwen3-0.6B-DSV4-tokenizer-remap-v2",
|
||||
"budgets": {
|
||||
"english_web": 1250000000,
|
||||
"english_edu": 1000000000,
|
||||
"chinese_clean": 1250000000,
|
||||
"code": 750000000,
|
||||
"math": 500000000,
|
||||
"science": 150000000,
|
||||
"qa_as_text": 100000000
|
||||
},
|
||||
"seq_len_for_later_packing": 8192,
|
||||
"stream_sources": {
|
||||
"english_web": [
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "HuggingFaceFW/fineweb",
|
||||
"config": "CC-MAIN-2025-26",
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
},
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "HuggingFaceFW/fineweb",
|
||||
"config": "CC-MAIN-2025-21",
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
}
|
||||
],
|
||||
"english_edu": [
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "HuggingFaceFW/fineweb-edu",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
}
|
||||
],
|
||||
"chinese_clean": [
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "BAAI/CCI3-HQ",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
},
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "Skywork/SkyPile-150B",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
}
|
||||
],
|
||||
"code": [
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "bigcode/starcoderdata",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
},
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "codeparrot/github-code",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
}
|
||||
],
|
||||
"math": [
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "open-web-math/open-web-math",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
},
|
||||
{
|
||||
"kind": "hf",
|
||||
"name": "GAIR/MathPile",
|
||||
"config": null,
|
||||
"split": "train",
|
||||
"max_rows": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"local_globs": {
|
||||
"science": [
|
||||
"data/offline_text_only_reasoning_sources_20260611/science_reasoning__*.jsonl",
|
||||
"data/offline_text_only_reasoning_sources_20260611/logic__*.jsonl"
|
||||
],
|
||||
"qa_as_text": [
|
||||
"data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl",
|
||||
"data/open_recovery_sft_mix_alt_sources_1m_parquet_20260607/normalized.jsonl"
|
||||
]
|
||||
},
|
||||
"local_parquet_globs": {
|
||||
"english_web": [
|
||||
"data/raw_parquets/fineweb_2025/*.parquet"
|
||||
],
|
||||
"english_edu": [
|
||||
"data/raw_parquets/fineweb_edu/*.parquet"
|
||||
],
|
||||
"code": [
|
||||
"data/raw_parquets/starcoder/python/*.parquet",
|
||||
"data/raw_parquets/starcoder/javascript/*.parquet",
|
||||
"data/raw_parquets/starcoder/typescript/*.parquet",
|
||||
"data/raw_parquets/starcoder/java/*.parquet",
|
||||
"data/raw_parquets/starcoder/cpp/*.parquet",
|
||||
"data/raw_parquets/starcoder/go/*.parquet",
|
||||
"data/raw_parquets/starcoder/rust/*.parquet",
|
||||
"data/raw_parquets/starcoder/shell/*.parquet"
|
||||
]
|
||||
},
|
||||
"min_tokens": 128,
|
||||
"max_doc_tokens": 32768
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"scale": 5.0,
|
||||
"budgets": {
|
||||
"english_web": 1250000000,
|
||||
"english_edu": 1000000000,
|
||||
"chinese_clean": 1250000000,
|
||||
"code": 750000000,
|
||||
"math": 500000000,
|
||||
"science": 150000000,
|
||||
"qa_as_text": 100000000
|
||||
},
|
||||
"tokens_by_category": {
|
||||
"english_web": 1250000445,
|
||||
"english_edu": 1000000462,
|
||||
"code": 750001624,
|
||||
"math": 96992014,
|
||||
"science": 30016910,
|
||||
"qa_as_text": 100000055
|
||||
},
|
||||
"docs_by_category": {
|
||||
"english_web": 1571857,
|
||||
"english_edu": 973081,
|
||||
"code": 589463,
|
||||
"math": 51576,
|
||||
"science": 48872,
|
||||
"qa_as_text": 244246
|
||||
},
|
||||
"tokens_by_source": {
|
||||
"data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_000_00000.parquet": 684777289,
|
||||
"data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00000.parquet": 627654914,
|
||||
"data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_001_00000.parquet": 565223156,
|
||||
"data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00001.parquet": 372345548,
|
||||
"data/raw_parquets/starcoder/python/train-00000-of-00059.parquet": 238778818,
|
||||
"data/raw_parquets/starcoder/python/train-00002-of-00059.parquet": 233459009,
|
||||
"data/raw_parquets/starcoder/python/train-00001-of-00059.parquet": 233210510,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl": 100000055,
|
||||
"open-web-math/open-web-math": 96992014,
|
||||
"data/raw_parquets/starcoder/python/train-00003-of-00059.parquet": 44553287,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__train.jsonl": 10027949,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__train.jsonl": 6138410,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__train.jsonl": 3119670,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__test.jsonl": 1837710,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__GBaker_MedQA-USMLE-4-options__train.jsonl": 1572096,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__train.jsonl": 1182676,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__test.jsonl": 1033827,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__derek-thomas_ScienceQA__validation.jsonl": 1032383,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_proofwriter__validation.jsonl": 895384,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_qasc__train.jsonl": 688074,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__test.jsonl": 307416,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_folio__train.jsonl": 261306,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__openlifescienceai_medmcqa__validation.jsonl": 242852,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__train.jsonl": 221520,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__GBaker_MedQA-USMLE-4-options__test.jsonl": 203198,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__temporal_sequences__train.jsonl": 141867,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__test.jsonl": 139093,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__train.jsonl": 130752,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_deduction__train.jsonl": 118299,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__test.jsonl": 102393,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__sciq__validation.jsonl": 99297,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__test.jsonl": 79458,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_qasc__validation.jsonl": 79086,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__train.jsonl": 74207,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_folio__validation.jsonl": 53607,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__temporal_sequences__validation.jsonl": 35418,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Easy__validation.jsonl": 33149,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_deduction__validation.jsonl": 31008,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__validation.jsonl": 23331,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_openbookqa__main__test.jsonl": 22745,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__qiaojin_PubMedQA__pqa_labeled__train.jsonl": 22169,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/science_reasoning__allenai_ai2_arc__ARC-Challenge__validation.jsonl": 19963,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__disambiguation_qa__train.jsonl": 15771,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__date_understanding__train.jsonl": 14547,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__disambiguation_qa__validation.jsonl": 3843,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__date_understanding__validation.jsonl": 3629,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_args__validation.jsonl": 3136,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__logical_args__train.jsonl": 2729,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__analytic_entailment__train.jsonl": 2277,
|
||||
"/ssd/yi/tokenizer_swap_cepe/data/offline_text_only_reasoning_sources_20260611/logic__tasksource_bigbench__analytic_entailment__validation.jsonl": 665
|
||||
},
|
||||
"rejected": {
|
||||
"english_web": {
|
||||
"too_short": 168039,
|
||||
"too_long": 179
|
||||
},
|
||||
"english_edu": {
|
||||
"too_short": 18375,
|
||||
"too_long": 221
|
||||
},
|
||||
"code": {
|
||||
"too_short": 106203,
|
||||
"too_long": 816
|
||||
},
|
||||
"math": {
|
||||
"too_short": 3669,
|
||||
"too_long": 152
|
||||
},
|
||||
"science": {
|
||||
"pack_too_short": 5591
|
||||
},
|
||||
"qa_as_text": {
|
||||
"too_short": 108331
|
||||
}
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"category": "english_web",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_000_00000.parquet",
|
||||
"rows_seen": 961000,
|
||||
"docs_written": 864351,
|
||||
"tokens": 684777289,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "english_web",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/fineweb_2025/CC-MAIN-2025-26_001_00000.parquet",
|
||||
"rows_seen": 779076,
|
||||
"docs_written": 707506,
|
||||
"tokens": 565223156,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "english_edu",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00000.parquet",
|
||||
"rows_seen": 621759,
|
||||
"docs_written": 610163,
|
||||
"tokens": 627654914,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "english_edu",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/fineweb_edu/CC-MAIN-2024-10_000_00001.parquet",
|
||||
"rows_seen": 369919,
|
||||
"docs_written": 362918,
|
||||
"tokens": 372345548,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "chinese_clean",
|
||||
"kind": "hf_stream",
|
||||
"source": "BAAI/CCI3-HQ",
|
||||
"rows_seen": 0,
|
||||
"docs_written": 0,
|
||||
"tokens": 0,
|
||||
"error": "DatasetNotFoundError(\"Dataset 'BAAI/CCI3-HQ' is a gated dataset on the Hub. You must be authenticated to access it.\")"
|
||||
},
|
||||
{
|
||||
"category": "chinese_clean",
|
||||
"kind": "hf_stream",
|
||||
"source": "Skywork/SkyPile-150B",
|
||||
"rows_seen": 0,
|
||||
"docs_written": 0,
|
||||
"tokens": 0,
|
||||
"error": "RuntimeError('Cannot send a request, as the client has been closed.')"
|
||||
},
|
||||
{
|
||||
"category": "code",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/starcoder/python/train-00000-of-00059.parquet",
|
||||
"rows_seen": 218079,
|
||||
"docs_written": 184958,
|
||||
"tokens": 238778818,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "code",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/starcoder/python/train-00001-of-00059.parquet",
|
||||
"rows_seen": 218079,
|
||||
"docs_written": 184495,
|
||||
"tokens": 233210510,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "code",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/starcoder/python/train-00002-of-00059.parquet",
|
||||
"rows_seen": 218079,
|
||||
"docs_written": 184331,
|
||||
"tokens": 233459009,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "code",
|
||||
"kind": "local_parquet",
|
||||
"source": "data/raw_parquets/starcoder/python/train-00003-of-00059.parquet",
|
||||
"rows_seen": 42246,
|
||||
"docs_written": 35679,
|
||||
"tokens": 44553287,
|
||||
"error": ""
|
||||
},
|
||||
{
|
||||
"category": "math",
|
||||
"kind": "hf_stream",
|
||||
"source": "open-web-math/open-web-math",
|
||||
"rows_seen": 55397,
|
||||
"docs_written": 51576,
|
||||
"tokens": 96992014,
|
||||
"error": "ProxyError('503 Service Unavailable')"
|
||||
},
|
||||
{
|
||||
"category": "math",
|
||||
"kind": "hf_stream",
|
||||
"source": "GAIR/MathPile",
|
||||
"rows_seen": 0,
|
||||
"docs_written": 0,
|
||||
"tokens": 0,
|
||||
"error": "DatasetNotFoundError(\"Dataset 'GAIR/MathPile' is a gated dataset on the Hub. You must be authenticated to access it.\")"
|
||||
}
|
||||
],
|
||||
"elapsed_sec": 10098.068085193634
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
{
|
||||
"docs_seen": 960549,
|
||||
"train_blocks": 121041,
|
||||
"eval_blocks": 1024,
|
||||
"train_tokens": 991567872,
|
||||
"eval_tokens": 8388608,
|
||||
"source_docs": {
|
||||
"english_web": 318271,
|
||||
"english_edu": 194585,
|
||||
"chinese_clean": 181655,
|
||||
"code": 116242,
|
||||
"math": 53125,
|
||||
"science": 47973,
|
||||
"qa_as_text": 48698
|
||||
},
|
||||
"source_tokens": {
|
||||
"english_web": 249999374,
|
||||
"english_edu": 199998420,
|
||||
"chinese_clean": 249999618,
|
||||
"code": 149999958,
|
||||
"math": 99987860,
|
||||
"science": 29999428,
|
||||
"qa_as_text": 19999497
|
||||
},
|
||||
"train_blocks_by_category": {
|
||||
"english_web": 30261,
|
||||
"english_edu": 24208,
|
||||
"chinese_clean": 30261,
|
||||
"code": 18156,
|
||||
"math": 12103,
|
||||
"science": 3631,
|
||||
"qa_as_text": 2421
|
||||
},
|
||||
"eval_blocks_by_category": {
|
||||
"english_web": 256,
|
||||
"english_edu": 205,
|
||||
"chinese_clean": 256,
|
||||
"code": 154,
|
||||
"math": 102,
|
||||
"science": 31,
|
||||
"qa_as_text": 20
|
||||
},
|
||||
"start_time": 1781429820.1093097,
|
||||
"leftover_tokens_by_category": {
|
||||
"english_web": 4110,
|
||||
"english_edu": 7124,
|
||||
"chinese_clean": 4354,
|
||||
"code": 4438,
|
||||
"math": 4500,
|
||||
"science": 324,
|
||||
"qa_as_text": 2825
|
||||
},
|
||||
"tokenizer": "/ssd/yi/tokenizer_swap_cepe/models/Qwen3-0.6B-DSV4-tokenizer-remap-v2",
|
||||
"seq_len": 8192,
|
||||
"seed": 42,
|
||||
"budgets": {
|
||||
"english_web": 250000000,
|
||||
"english_edu": 200000000,
|
||||
"chinese_clean": 250000000,
|
||||
"code": 150000000,
|
||||
"math": 100000000,
|
||||
"science": 30000000,
|
||||
"qa_as_text": 20000000
|
||||
},
|
||||
"sources": {
|
||||
"english_web": "data/cpt_docmix_parquet_sources_8192_20260613/documents/english_web.jsonl.gz",
|
||||
"english_edu": "data/cpt_docmix_parquet_sources_8192_20260613/documents/english_edu.jsonl.gz",
|
||||
"chinese_clean": "data/cpt_docmix_cci3_science_fixed_8192_20260614/documents/chinese_clean.jsonl.gz",
|
||||
"code": "data/cpt_docmix_parquet_sources_8192_20260613/documents/code.jsonl.gz",
|
||||
"math": "data/cpt_docmix_available_sources_8192_20260613/documents/math.jsonl.gz",
|
||||
"science": "data/cpt_docmix_cci3_science_fixed_8192_20260614/documents/science.jsonl.gz",
|
||||
"qa_as_text": "data/cpt_docmix_available_sources_8192_20260613/documents/qa_as_text.jsonl.gz"
|
||||
},
|
||||
"eval_quota_blocks": {
|
||||
"english_web": 256,
|
||||
"english_edu": 205,
|
||||
"chinese_clean": 256,
|
||||
"code": 154,
|
||||
"math": 102,
|
||||
"science": 31,
|
||||
"qa_as_text": 20
|
||||
},
|
||||
"train_shards": [
|
||||
{
|
||||
"path": "train_00000.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00001.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00002.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00003.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00004.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00005.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00006.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00007.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00008.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00009.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00010.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00011.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00012.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00013.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00014.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00015.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00016.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00017.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00018.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00019.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00020.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00021.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00022.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00023.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00024.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00025.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00026.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00027.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00028.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00029.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00030.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00031.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00032.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00033.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00034.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00035.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00036.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00037.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00038.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00039.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00040.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00041.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00042.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00043.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00044.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00045.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00046.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00047.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00048.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00049.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00050.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00051.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00052.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00053.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00054.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00055.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00056.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00057.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00058.npy",
|
||||
"blocks": 2048,
|
||||
"tokens": 16777216
|
||||
},
|
||||
{
|
||||
"path": "train_00059.npy",
|
||||
"blocks": 209,
|
||||
"tokens": 1712128
|
||||
}
|
||||
],
|
||||
"eval_shards": [
|
||||
{
|
||||
"path": "eval_00000.npy",
|
||||
"blocks": 1024,
|
||||
"tokens": 8388608
|
||||
}
|
||||
],
|
||||
"elapsed_sec": 2387.9453027248383
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,309 @@
|
||||
{
|
||||
"args": {
|
||||
"out": "data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611",
|
||||
"tokenizer": "/ssd/yi/tokenizer_swap_cepe/models/Qwen3-0.6B-DSV4-tokenizer-remap-v2",
|
||||
"train_total": 1000000,
|
||||
"test_per_group": 400,
|
||||
"seed": 20260611,
|
||||
"hf_endpoint": "https://hf-mirror.com",
|
||||
"use_hf": true,
|
||||
"allow_shortfall": true,
|
||||
"upsample_train_shortfall": false,
|
||||
"min_prompt_tokens": 4,
|
||||
"min_answer_tokens": 8,
|
||||
"max_prompt_tokens": 1024,
|
||||
"max_answer_tokens": 1024,
|
||||
"trust_metadata_lengths": true,
|
||||
"max_seen_per_source": 500000,
|
||||
"stop_empty_source_early": false
|
||||
},
|
||||
"train_ratios": {
|
||||
"chinese_exam": 0,
|
||||
"chinese_dialogue": 35,
|
||||
"code": 30,
|
||||
"math": 20,
|
||||
"logic": 5,
|
||||
"science_reasoning": 5,
|
||||
"english_dialogue": 5
|
||||
},
|
||||
"test_quotas": {
|
||||
"chinese_exam": 400,
|
||||
"chinese_dialogue": 400,
|
||||
"code": 400,
|
||||
"math": 400,
|
||||
"logic": 400,
|
||||
"science_reasoning": 400,
|
||||
"english_dialogue": 400
|
||||
},
|
||||
"loaders": {
|
||||
"test_hf": {
|
||||
"test:chinese_exam:ceval/ceval-exam": {
|
||||
"seen": 728,
|
||||
"accepted": 400,
|
||||
"too_short": 328
|
||||
},
|
||||
"test:chinese_dialogue:m-a-p/COIG-CQIA": {
|
||||
"seen": 414,
|
||||
"accepted": 400,
|
||||
"too_long": 14
|
||||
},
|
||||
"test:code:openai/openai_humaneval": {
|
||||
"seen": 164,
|
||||
"accepted": 154,
|
||||
"too_short": 8,
|
||||
"too_long": 2
|
||||
},
|
||||
"test:code:google-research-datasets/mbpp": {
|
||||
"seen": 246,
|
||||
"accepted": 246
|
||||
},
|
||||
"test:math:gsm8k": {
|
||||
"seen": 400,
|
||||
"accepted": 400
|
||||
},
|
||||
"test:logic:cais/mmlu": {
|
||||
"seen": 331,
|
||||
"accepted": 164,
|
||||
"too_short": 167
|
||||
},
|
||||
"test:logic:lighteval/bbh": {
|
||||
"load_error:test:logical_deduction_three_objects": 1,
|
||||
"error:ValueError(\"Bad split: test. Available splits: ['train']\")": 1,
|
||||
"seen": 296,
|
||||
"accepted": 236,
|
||||
"too_short": 60
|
||||
},
|
||||
"test:science_reasoning:Idavidrein/gpqa": {
|
||||
"seen": 448,
|
||||
"too_short": 195,
|
||||
"accepted": 252,
|
||||
"too_long": 1
|
||||
},
|
||||
"test:science_reasoning:allenai/ai2_arc": {
|
||||
"seen": 287,
|
||||
"accepted": 148,
|
||||
"too_short": 139
|
||||
},
|
||||
"test:english_dialogue:HuggingFaceH4/ultrachat_200k": {
|
||||
"seen": 420,
|
||||
"too_long": 20,
|
||||
"accepted": 400
|
||||
}
|
||||
},
|
||||
"train_local": {
|
||||
"data/open_recovery_sft_mix_alt_sources_1m_parquet_20260607/normalized.jsonl": {
|
||||
"accepted": 1051916,
|
||||
"heldout_excluded": 411,
|
||||
"duplicate": 30
|
||||
},
|
||||
"data/open_recovery_sft_mix_100k_4_3_2_1_20260604/normalized.jsonl": {
|
||||
"duplicate": 88687,
|
||||
"accepted": 9020,
|
||||
"heldout_excluded": 401
|
||||
},
|
||||
"data/modelscope_alt_sources_20260607/CodeAlpaca-20k.jsonl": {
|
||||
"duplicate": 18820,
|
||||
"too_short": 1193,
|
||||
"no_messages": 6,
|
||||
"too_long": 3
|
||||
},
|
||||
"data/modelscope_alt_sources_20260607/alpaca-gpt4-data-zh_train.csv": {
|
||||
"duplicate": 46476,
|
||||
"too_short": 2342
|
||||
}
|
||||
},
|
||||
"train_hf": {
|
||||
"train:chinese_dialogue:m-a-p/COIG-CQIA": {
|
||||
"seen": 10116,
|
||||
"duplicate": 7211,
|
||||
"too_short": 1138,
|
||||
"too_long": 1354,
|
||||
"no_messages": 1,
|
||||
"accepted": 1,
|
||||
"heldout_excluded": 411
|
||||
},
|
||||
"train:chinese_dialogue:BelleGroup/train_0.5M_CN": {
|
||||
"seen": 330837,
|
||||
"too_short": 34263,
|
||||
"duplicate": 2188,
|
||||
"accepted": 294334,
|
||||
"too_long": 32,
|
||||
"no_messages": 20
|
||||
},
|
||||
"train:code:ise-uiuc/Magicoder-OSS-Instruct-75K": {
|
||||
"seen": 75197,
|
||||
"duplicate": 69998,
|
||||
"bad_code_fence": 34,
|
||||
"too_long": 48,
|
||||
"accepted": 5117
|
||||
},
|
||||
"train:code:bigcode/self-oss-instruct-sc2-exec-filter-50k": {
|
||||
"seen": 50661,
|
||||
"duplicate": 31578,
|
||||
"accepted": 19076,
|
||||
"too_long": 6,
|
||||
"bad_code_fence": 1
|
||||
},
|
||||
"train:code:HuggingFaceTB/smoltalk": {
|
||||
"seen": 131271,
|
||||
"duplicate": 78743,
|
||||
"too_long": 1498,
|
||||
"bad_code_fence": 1,
|
||||
"accepted": 51029
|
||||
},
|
||||
"train:code:nvidia/OpenCodeInstruct": {
|
||||
"seen": 43455,
|
||||
"duplicate": 6678,
|
||||
"too_long": 39,
|
||||
"accepted": 36738
|
||||
},
|
||||
"train:logic:tasksource/bigbench": {
|
||||
"seen": 8291,
|
||||
"too_short": 5767,
|
||||
"accepted": 2338,
|
||||
"heldout_excluded": 186
|
||||
},
|
||||
"train:logic:tau/commonsense_qa": {
|
||||
"seen": 10962,
|
||||
"too_short": 10961,
|
||||
"accepted": 1
|
||||
},
|
||||
"train:logic:metaeval/reclor": {
|
||||
"seen": 5138,
|
||||
"accepted": 5117,
|
||||
"too_short": 21
|
||||
},
|
||||
"train:science_reasoning:allenai/ai2_arc": {
|
||||
"seen": 1418,
|
||||
"too_short": 739,
|
||||
"accepted": 679
|
||||
},
|
||||
"train:science_reasoning:allenai/qasc": {
|
||||
"iter_error:train:None": 1,
|
||||
"error:ProxyError('503 Service Unavailable')": 2,
|
||||
"iter_error:validation:None": 1
|
||||
},
|
||||
"train:science_reasoning:allenai/openbookqa": {
|
||||
"iter_error:train:main": 1,
|
||||
"error:ProxyError('503 Service Unavailable')": 4,
|
||||
"iter_error:validation:main": 1,
|
||||
"iter_error:train:additional": 1,
|
||||
"iter_error:validation:additional": 1
|
||||
},
|
||||
"train:science_reasoning:sciq": {
|
||||
"seen": 12679,
|
||||
"accepted": 11370,
|
||||
"too_short": 1309
|
||||
},
|
||||
"train:science_reasoning:qiaojin/PubMedQA": {
|
||||
"seen": 1000,
|
||||
"accepted": 1000
|
||||
}
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"failed": false,
|
||||
"test_total": 2800,
|
||||
"train_total": 921360,
|
||||
"test_counts": {
|
||||
"science_reasoning": 400,
|
||||
"logic": 400,
|
||||
"code": 400,
|
||||
"chinese_exam": 400,
|
||||
"math": 400,
|
||||
"chinese_dialogue": 400,
|
||||
"english_dialogue": 400
|
||||
},
|
||||
"train_counts": {
|
||||
"chinese_dialogue": 350000,
|
||||
"code": 300000,
|
||||
"english_dialogue": 50000,
|
||||
"math": 200000,
|
||||
"science_reasoning": 13904,
|
||||
"logic": 7456
|
||||
},
|
||||
"test_shortfall": {},
|
||||
"train_shortfall": {
|
||||
"logic": 42544,
|
||||
"science_reasoning": 36096
|
||||
},
|
||||
"train_upsampled": {},
|
||||
"train_quotas": {
|
||||
"chinese_exam": 0,
|
||||
"chinese_dialogue": 350000,
|
||||
"code": 300000,
|
||||
"math": 200000,
|
||||
"logic": 50000,
|
||||
"science_reasoning": 50000,
|
||||
"english_dialogue": 50000
|
||||
},
|
||||
"train_source_top": {
|
||||
"BelleGroup/train_0.5M_CN": 296334,
|
||||
"nvidia/OpenMathInstruct-2": 89260,
|
||||
"TIGER-Lab/MathInstruct": 85173,
|
||||
"ise-uiuc/Magicoder-OSS-Instruct-75K": 75115,
|
||||
"bigcode/self-oss-instruct-sc2-exec-filter-50k": 66632,
|
||||
"HuggingFaceTB/smoltalk:apigen-80k": 51029,
|
||||
"AI-ModelScope/alpaca-gpt4-data-zh:file": 46476,
|
||||
"nvidia/OpenCodeInstruct": 43404,
|
||||
"AI-ModelScope/smoltalk:self-oss-instruct-file": 25000,
|
||||
"AI-ModelScope/smoltalk:apigen-80k-file": 20000,
|
||||
"AI-ModelScope/CodeAlpaca-20k:file": 18820,
|
||||
"AI-ModelScope/smoltalk:numina-cot-100k-file": 17080,
|
||||
"HuggingFaceTB/smoltalk:all": 15260,
|
||||
"HuggingFaceH4/ultrachat_200k": 15238,
|
||||
"allenai/tulu-3-sft-mixture": 15198,
|
||||
"sciq": 11370,
|
||||
"AI-ModelScope/smoltalk:metamathqa-50k-file": 8487,
|
||||
"metaeval/reclor": 5117,
|
||||
"m-a-p/COIG-CQIA:zhihu": 4978,
|
||||
"AI-ModelScope/smoltalk:openhermes-100k": 4304,
|
||||
"m-a-p/COIG-CQIA:coig_pc": 1877,
|
||||
"qiaojin/PubMedQA:pqa_labeled": 1000,
|
||||
"allenai/ai2_arc:ARC-Easy": 855,
|
||||
"tasksource/bigbench:logical_deduction": 812,
|
||||
"tasksource/bigbench:temporal_sequences": 800,
|
||||
"allenai/ai2_arc:ARC-Challenge": 679,
|
||||
"tasksource/bigbench:social_iqa": 343,
|
||||
"m-a-p/COIG-CQIA:wikihow": 335,
|
||||
"tasksource/bigbench:date_understanding": 296,
|
||||
"tasksource/bigbench:disambiguation_qa": 71,
|
||||
"tasksource/bigbench:logical_args": 16,
|
||||
"tau/commonsense_qa": 1
|
||||
},
|
||||
"test_source_top": {
|
||||
"gsm8k:main": 400,
|
||||
"m-a-p/COIG-CQIA:zhihu": 400,
|
||||
"HuggingFaceH4/ultrachat_200k": 400,
|
||||
"Idavidrein/gpqa": 252,
|
||||
"google-research-datasets/mbpp": 246,
|
||||
"lighteval/bbh:logical_deduction_three_objects": 236,
|
||||
"openai/openai_humaneval": 154,
|
||||
"allenai/ai2_arc:ARC-Challenge": 148,
|
||||
"cais/mmlu:formal_logic": 98,
|
||||
"cais/mmlu:logical_fallacies": 66,
|
||||
"ceval/ceval-exam:civil_servant": 38,
|
||||
"ceval/ceval-exam:college_economics": 36,
|
||||
"ceval/ceval-exam:accountant": 31,
|
||||
"ceval/ceval-exam:college_programming": 23,
|
||||
"ceval/ceval-exam:fire_engineer": 22,
|
||||
"ceval/ceval-exam:advanced_mathematics": 22,
|
||||
"ceval/ceval-exam:high_school_chinese": 21,
|
||||
"ceval/ceval-exam:high_school_biology": 19,
|
||||
"ceval/ceval-exam:college_chemistry": 18,
|
||||
"ceval/ceval-exam:college_physics": 18,
|
||||
"ceval/ceval-exam:high_school_chemistry": 17,
|
||||
"ceval/ceval-exam:environmental_impact_assessment_engineer": 16,
|
||||
"ceval/ceval-exam:business_administration": 15,
|
||||
"ceval/ceval-exam:discrete_mathematics": 14,
|
||||
"ceval/ceval-exam:education_science": 13,
|
||||
"ceval/ceval-exam:clinical_medicine": 13,
|
||||
"ceval/ceval-exam:electrical_engineer": 12,
|
||||
"ceval/ceval-exam:computer_network": 11,
|
||||
"ceval/ceval-exam:art_studies": 11,
|
||||
"ceval/ceval-exam:computer_architecture": 11,
|
||||
"ceval/ceval-exam:chinese_language_and_literature": 10,
|
||||
"ceval/ceval-exam:basic_medicine": 9
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"tokenizer": "/ssd/yi/tokenizer_swap_cepe/models/dsv4_chat_full_sft_remap_v2_alt1m_5epoch_bsz8_accum16_20260610",
|
||||
"encoding_dir": "/ssd/yi/tokenizer_swap_cepe/external/deepseek_v4_encoding",
|
||||
"cutoff_len": 2048,
|
||||
"eos_token": "<｜end▁of▁sentence｜>",
|
||||
"eos_token_id": 1,
|
||||
"splits": {
|
||||
"train": {
|
||||
"split": "train",
|
||||
"source": "/ssd/yi/tokenizer_swap_cepe/data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl",
|
||||
"output": "/ssd/yi/tokenizer_swap_cepe/data/dsv4_chat_tokenized_v4_noupsample_nobbh_921k_20260611/train_dsv4_chat_tokenized.jsonl.gz",
|
||||
"cutoff_len": 2048,
|
||||
"rows_seen": 921360,
|
||||
"rows_written": 921360,
|
||||
"skipped_no_messages": 0,
|
||||
"truncated": 0,
|
||||
"eos_in_labels": 921360,
|
||||
"prefix_mismatch": 0,
|
||||
"capability_counts": {
|
||||
"chinese_dialogue": 350000,
|
||||
"code": 300000,
|
||||
"math": 200000,
|
||||
"english_dialogue": 50000,
|
||||
"science_reasoning": 13904,
|
||||
"logic": 7456
|
||||
},
|
||||
"prompt_tokens": {
|
||||
"p50": 65,
|
||||
"p90": 461,
|
||||
"p95": 698,
|
||||
"p99": 951,
|
||||
"max": 1028
|
||||
},
|
||||
"response_tokens": {
|
||||
"p50": 101,
|
||||
"p90": 340,
|
||||
"p95": 457,
|
||||
"p99": 748,
|
||||
"max": 1025
|
||||
},
|
||||
"total_tokens": {
|
||||
"p50": 223,
|
||||
"p90": 699,
|
||||
"p95": 889,
|
||||
"p99": 1165,
|
||||
"max": 1957
|
||||
},
|
||||
"truncated_rate": 0.0,
|
||||
"eos_label_rate": 1.0,
|
||||
"source_counts_top50": {
|
||||
"BelleGroup/train_0.5M_CN": 296334,
|
||||
"nvidia/OpenMathInstruct-2": 89260,
|
||||
"TIGER-Lab/MathInstruct": 85173,
|
||||
"ise-uiuc/Magicoder-OSS-Instruct-75K": 75115,
|
||||
"bigcode/self-oss-instruct-sc2-exec-filter-50k": 66632,
|
||||
"HuggingFaceTB/smoltalk:apigen-80k": 51029,
|
||||
"AI-ModelScope/alpaca-gpt4-data-zh:file": 46476,
|
||||
"nvidia/OpenCodeInstruct": 43404,
|
||||
"AI-ModelScope/smoltalk:self-oss-instruct-file": 25000,
|
||||
"AI-ModelScope/smoltalk:apigen-80k-file": 20000,
|
||||
"AI-ModelScope/CodeAlpaca-20k:file": 18820,
|
||||
"AI-ModelScope/smoltalk:numina-cot-100k-file": 17080,
|
||||
"HuggingFaceTB/smoltalk:all": 15260,
|
||||
"HuggingFaceH4/ultrachat_200k": 15238,
|
||||
"allenai/tulu-3-sft-mixture": 15198,
|
||||
"sciq": 11370,
|
||||
"AI-ModelScope/smoltalk:metamathqa-50k-file": 8487,
|
||||
"metaeval/reclor": 5117,
|
||||
"m-a-p/COIG-CQIA:zhihu": 4978,
|
||||
"AI-ModelScope/smoltalk:openhermes-100k": 4304,
|
||||
"m-a-p/COIG-CQIA:coig_pc": 1877,
|
||||
"qiaojin/PubMedQA:pqa_labeled": 1000,
|
||||
"allenai/ai2_arc:ARC-Easy": 855,
|
||||
"tasksource/bigbench:logical_deduction": 812,
|
||||
"tasksource/bigbench:temporal_sequences": 800,
|
||||
"allenai/ai2_arc:ARC-Challenge": 679,
|
||||
"tasksource/bigbench:social_iqa": 343,
|
||||
"m-a-p/COIG-CQIA:wikihow": 335,
|
||||
"tasksource/bigbench:date_understanding": 296,
|
||||
"tasksource/bigbench:disambiguation_qa": 71,
|
||||
"tasksource/bigbench:logical_args": 16,
|
||||
"tau/commonsense_qa": 1
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"split": "validation",
|
||||
"source": "/ssd/yi/tokenizer_swap_cepe/data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/heldout_2p8k.jsonl",
|
||||
"output": "/ssd/yi/tokenizer_swap_cepe/data/dsv4_chat_tokenized_v4_noupsample_nobbh_921k_20260611/validation_dsv4_chat_tokenized.jsonl.gz",
|
||||
"cutoff_len": 2048,
|
||||
"rows_seen": 2800,
|
||||
"rows_written": 2800,
|
||||
"skipped_no_messages": 0,
|
||||
"truncated": 0,
|
||||
"eos_in_labels": 2800,
|
||||
"prefix_mismatch": 0,
|
||||
"capability_counts": {
|
||||
"science_reasoning": 400,
|
||||
"logic": 400,
|
||||
"code": 400,
|
||||
"chinese_exam": 400,
|
||||
"math": 400,
|
||||
"chinese_dialogue": 400,
|
||||
"english_dialogue": 400
|
||||
},
|
||||
"prompt_tokens": {
|
||||
"p50": 85,
|
||||
"p90": 238,
|
||||
"p95": 346,
|
||||
"p99": 687,
|
||||
"max": 1020
|
||||
},
|
||||
"response_tokens": {
|
||||
"p50": 53,
|
||||
"p90": 416,
|
||||
"p95": 547,
|
||||
"p99": 837,
|
||||
"max": 1019
|
||||
},
|
||||
"total_tokens": {
|
||||
"p50": 177,
|
||||
"p90": 543,
|
||||
"p95": 688,
|
||||
"p99": 981,
|
||||
"max": 1162
|
||||
},
|
||||
"truncated_rate": 0.0,
|
||||
"eos_label_rate": 1.0,
|
||||
"source_counts_top50": {
|
||||
"gsm8k:main": 400,
|
||||
"m-a-p/COIG-CQIA:zhihu": 400,
|
||||
"HuggingFaceH4/ultrachat_200k": 400,
|
||||
"Idavidrein/gpqa": 252,
|
||||
"google-research-datasets/mbpp": 246,
|
||||
"lighteval/bbh:logical_deduction_three_objects": 236,
|
||||
"openai/openai_humaneval": 154,
|
||||
"allenai/ai2_arc:ARC-Challenge": 148,
|
||||
"cais/mmlu:formal_logic": 98,
|
||||
"cais/mmlu:logical_fallacies": 66,
|
||||
"ceval/ceval-exam:civil_servant": 38,
|
||||
"ceval/ceval-exam:college_economics": 36,
|
||||
"ceval/ceval-exam:accountant": 31,
|
||||
"ceval/ceval-exam:college_programming": 23,
|
||||
"ceval/ceval-exam:fire_engineer": 22,
|
||||
"ceval/ceval-exam:advanced_mathematics": 22,
|
||||
"ceval/ceval-exam:high_school_chinese": 21,
|
||||
"ceval/ceval-exam:high_school_biology": 19,
|
||||
"ceval/ceval-exam:college_chemistry": 18,
|
||||
"ceval/ceval-exam:college_physics": 18,
|
||||
"ceval/ceval-exam:high_school_chemistry": 17,
|
||||
"ceval/ceval-exam:environmental_impact_assessment_engineer": 16,
|
||||
"ceval/ceval-exam:business_administration": 15,
|
||||
"ceval/ceval-exam:discrete_mathematics": 14,
|
||||
"ceval/ceval-exam:education_science": 13,
|
||||
"ceval/ceval-exam:clinical_medicine": 13,
|
||||
"ceval/ceval-exam:electrical_engineer": 12,
|
||||
"ceval/ceval-exam:computer_network": 11,
|
||||
"ceval/ceval-exam:art_studies": 11,
|
||||
"ceval/ceval-exam:computer_architecture": 11,
|
||||
"ceval/ceval-exam:chinese_language_and_literature": 10,
|
||||
"ceval/ceval-exam:basic_medicine": 9
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user