{
  "tokenizer": "/ssd/yi/tokenizer_swap_cepe/models/Qwen3-0.6B-DSV4-tokenizer-remap-v2",
  "budgets": {
    "english_web": 1250000000,
    "english_edu": 1000000000,
    "chinese_clean": 1250000000,
    "code": 750000000,
    "math": 500000000,
    "science": 150000000,
    "qa_as_text": 100000000
  },
  "seq_len_for_later_packing": 8192,
  "stream_sources": {
    "english_web": [
      {
        "kind": "hf",
        "name": "HuggingFaceFW/fineweb",
        "config": "CC-MAIN-2025-26",
        "split": "train",
        "max_rows": 0
      },
      {
        "kind": "hf",
        "name": "HuggingFaceFW/fineweb",
        "config": "CC-MAIN-2025-21",
        "split": "train",
        "max_rows": 0
      }
    ],
    "english_edu": [
      {
        "kind": "hf",
        "name": "HuggingFaceFW/fineweb-edu",
        "config": null,
        "split": "train",
        "max_rows": 0
      }
    ],
    "chinese_clean": [
      {
        "kind": "hf",
        "name": "BAAI/CCI3-HQ",
        "config": null,
        "split": "train",
        "max_rows": 0
      },
      {
        "kind": "hf",
        "name": "Skywork/SkyPile-150B",
        "config": null,
        "split": "train",
        "max_rows": 0
      }
    ],
    "code": [
      {
        "kind": "hf",
        "name": "bigcode/starcoderdata",
        "config": null,
        "split": "train",
        "max_rows": 0
      },
      {
        "kind": "hf",
        "name": "codeparrot/github-code",
        "config": null,
        "split": "train",
        "max_rows": 0
      }
    ],
    "math": [
      {
        "kind": "hf",
        "name": "open-web-math/open-web-math",
        "config": null,
        "split": "train",
        "max_rows": 0
      },
      {
        "kind": "hf",
        "name": "GAIR/MathPile",
        "config": null,
        "split": "train",
        "max_rows": 0
      }
    ]
  },
  "local_globs": {
    "science": [
      "data/offline_text_only_reasoning_sources_20260611/science_reasoning__*.jsonl",
      "data/offline_text_only_reasoning_sources_20260611/logic__*.jsonl"
    ],
    "qa_as_text": [
      "data/training_mix_v4_train1m_test2p8k_noupsample_nobbh_20260611/train_1m.jsonl",
      "data/open_recovery_sft_mix_alt_sources_1m_parquet_20260607/normalized.jsonl"
    ]
  },
  "local_parquet_globs": {
    "english_web": [
      "data/raw_parquets/fineweb_2025/*.parquet"
    ],
    "english_edu": [
      "data/raw_parquets/fineweb_edu/*.parquet"
    ],
    "code": [
      "data/raw_parquets/starcoder/python/*.parquet",
      "data/raw_parquets/starcoder/javascript/*.parquet",
      "data/raw_parquets/starcoder/typescript/*.parquet",
      "data/raw_parquets/starcoder/java/*.parquet",
      "data/raw_parquets/starcoder/cpp/*.parquet",
      "data/raw_parquets/starcoder/go/*.parquet",
      "data/raw_parquets/starcoder/rust/*.parquet",
      "data/raw_parquets/starcoder/shell/*.parquet"
    ]
  },
  "min_tokens": 128,
  "max_doc_tokens": 32768
}