118 lines
2.9 KiB
JSON
118 lines
2.9 KiB
JSON
{
|
|
"tokenizer": "/ssd/yi/tokenizer_swap_cepe/models/Qwen3-0.6B-DSV4-tokenizer-remap-v2",
|
|
"budgets": {
|
|
"english_web": 1250000000,
|
|
"english_edu": 1000000000,
|
|
"chinese_clean": 1250000000,
|
|
"code": 750000000,
|
|
"math": 500000000,
|
|
"science": 150000000,
|
|
"qa_as_text": 100000000
|
|
},
|
|
"seq_len_for_later_packing": 8192,
|
|
"stream_sources": {
|
|
"english_web": [
|
|
{
|
|
"kind": "hf",
|
|
"name": "HuggingFaceFW/fineweb",
|
|
"config": "CC-MAIN-2025-26",
|
|
"split": "train",
|
|
"max_rows": 0
|
|
},
|
|
{
|
|
"kind": "hf",
|
|
"name": "HuggingFaceFW/fineweb",
|
|
"config": "CC-MAIN-2025-21",
|
|
"split": "train",
|
|
"max_rows": 0
|
|
}
|
|
],
|
|
"english_edu": [
|
|
{
|
|
"kind": "hf",
|
|
"name": "HuggingFaceFW/fineweb-edu",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
}
|
|
],
|
|
"chinese_clean": [
|
|
{
|
|
"kind": "hf",
|
|
"name": "BAAI/CCI3-HQ",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
},
|
|
{
|
|
"kind": "hf",
|
|
"name": "Skywork/SkyPile-150B",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
}
|
|
],
|
|
"code": [
|
|
{
|
|
"kind": "hf",
|
|
"name": "bigcode/starcoderdata",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
},
|
|
{
|
|
"kind": "hf",
|
|
"name": "codeparrot/github-code",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
}
|
|
],
|
|
"math": [
|
|
{
|
|
"kind": "hf",
|
|
"name": "open-web-math/open-web-math",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
},
|
|
{
|
|
"kind": "hf",
|
|
"name": "GAIR/MathPile",
|
|
"config": null,
|
|
"split": "train",
|
|
"max_rows": 0
|
|
}
|
|
]
|
|
},
|
|
"local_globs": {
|
|
"science": [
|
|
"data/offline_text_only_reasoning_sources_20260611/science_reasoning__*.jsonl",
|
|
"data/offline_text_only_reasoning_sources_20260611/logic__*.jsonl"
|
|
],
|
|
"qa_as_text": [
|
|
"data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl",
|
|
"data/open_recovery_sft_mix_alt_sources_1m_parquet_20260607/normalized.jsonl"
|
|
]
|
|
},
|
|
"local_parquet_globs": {
|
|
"english_web": [
|
|
"data/raw_parquets/fineweb_2025/*.parquet"
|
|
],
|
|
"english_edu": [
|
|
"data/raw_parquets/fineweb_edu/*.parquet"
|
|
],
|
|
"code": [
|
|
"data/raw_parquets/starcoder/python/*.parquet",
|
|
"data/raw_parquets/starcoder/javascript/*.parquet",
|
|
"data/raw_parquets/starcoder/typescript/*.parquet",
|
|
"data/raw_parquets/starcoder/java/*.parquet",
|
|
"data/raw_parquets/starcoder/cpp/*.parquet",
|
|
"data/raw_parquets/starcoder/go/*.parquet",
|
|
"data/raw_parquets/starcoder/rust/*.parquet",
|
|
"data/raw_parquets/starcoder/shell/*.parquet"
|
|
]
|
|
},
|
|
"min_tokens": 128,
|
|
"max_doc_tokens": 32768
|
|
} |