{
  "recipe": "web40_edu20_chinese10_science10_logic10_math5_code5",
  "target_mix": {
    "english_web": 0.4,
    "english_edu": 0.2,
    "chinese_clean": 0.1,
    "science": 0.1,
    "logic": 0.1,
    "math": 0.05,
    "code": 0.05
  },
  "target_token_quotas": {
    "english_web": 400000,
    "english_edu": 200000,
    "chinese_clean": 100000,
    "science": 100000,
    "logic": 100000,
    "math": 50000,
    "code": 50000
  },
  "effective_token_quotas": {
    "english_web": 400000,
    "english_edu": 200000,
    "chinese_clean": 100000,
    "science": 100000,
    "logic": 100000,
    "math": 50000,
    "code": 50000
  },
  "actual_fraction": {
    "english_web": 0.5018046381616852,
    "english_edu": 0.249618690426187,
    "chinese_clean": 0.0,
    "science": 0.12395194074492588,
    "logic": 0.0,
    "math": 0.06253353108153148,
    "code": 0.06209119958567044
  },
  "quota_fill": {
    "english_web": 1.0125,
    "english_edu": 1.00732,
    "chinese_clean": 0.0,
    "science": 1.0004,
    "logic": 0.0,
    "math": 1.0094,
    "code": 1.00226
  },
  "effective_quota_fill": {
    "english_web": 1.0125,
    "english_edu": 1.00732,
    "chinese_clean": 0.0,
    "science": 1.0004,
    "logic": 0.0,
    "math": 1.0094,
    "code": 1.00226
  },
  "shortfall_policy": {
    "enabled": true,
    "source": "science",
    "target": "english_edu",
    "science_target_tokens": 100000,
    "science_actual_tokens_capped": 100000,
    "science_shortfall_tokens": 0,
    "english_edu_base_target_tokens": 200000,
    "english_edu_effective_target_tokens": 200000
  },
  "stats": {
    "files_seen": 5,
    "docs_seen": 5000,
    "docs_written": 1614,
    "docs_dropped_missing_text": 0,
    "docs_dropped_quota_full": 3386,
    "malformed_json": 0,
    "estimated_tokens_seen": 4102388,
    "estimated_tokens_written": 807087,
    "shards_written": 1,
    "category_docs": {
      "science": 932,
      "english_web": 390,
      "english_edu": 140,
      "code": 104,
      "math": 48
    },
    "category_tokens": {
      "english_web": 405000,
      "english_edu": 201464,
      "science": 100040,
      "math": 50470,
      "code": 50113
    },
    "family_docs": {
      "science": 932,
      "english_web": 390,
      "english_edu": 140,
      "code": 104,
      "auto_text": 48
    },
    "family_tokens": {
      "english_web": 405000,
      "english_edu": 201464,
      "science": 100040,
      "auto_text": 50470,
      "code": 50113
    },
    "language_docs": {
      "unknown": 1362,
      "en": 188,
      "Python": 64
    },
    "source_docs": {
      "unknown": 1072,
      "FineWeb": 390,
      "stack-edu": 64,
      "finemath-4plus": 48,
      "python_clean": 40
    }
  },
  "inputs": [
    {
      "family": "english_web",
      "file_count": 133,
      "preview": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0001-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0002-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0003-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0004-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0005-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0006-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0007-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0008-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0009-of-2048.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/data/ultrafineweb_en/ultrafineweb-en-part-0010-of-2048.parquet"
      ]
    },
    {
      "family": "science",
      "file_count": 22,
      "preview": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_medmcqa/data/test-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_medmcqa/data/train-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_medmcqa/data/validation-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_openbookqa/additional/test-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_openbookqa/additional/train-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_openbookqa/additional/validation-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_openbookqa/main/test-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_openbookqa/main/train-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_openbookqa/main/validation-00000-of-00001.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_proofwriter/data/test-00000-of-00001-3e27b013c60e12d8.parquet"
      ]
    },
    {
      "family": "english_edu",
      "file_count": 96,
      "preview": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00000-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00001-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00002-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00003-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00004-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00005-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00006-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00007-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00008-of-00011.parquet",
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/data/CC-MAIN-2015-40/train-00009-of-00011.parquet"
      ]
    },
    {
      "family": "auto_text",
      "file_count": 315,
      "preview": [
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000000.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000001.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000002.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000003.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000004.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000005.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000006.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000007.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000008.jsonl",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_00/shard_000009.jsonl"
      ]
    },
    {
      "family": "code",
      "file_count": 1122,
      "preview": [
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000000.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000001.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000002.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000003.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000004.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000005.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000006.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000007.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000008.parquet",
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/part_000009.parquet"
      ]
    }
  ],
  "args": {
    "english_web_input": [
      "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/**/*.parquet"
    ],
    "english_edu_input": [
      "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/**/*.parquet"
    ],
    "chinese_clean_input": [],
    "science_input": [
      "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_*/*/*.parquet"
    ],
    "logic_input": [],
    "math_input": [],
    "code_input": [
      "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/*.parquet"
    ],
    "jiayi_scored_input": [
      "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_*/*.jsonl"
    ],
    "output_dir": "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/build_smoke/rebalanced_auto_text_1m",
    "target_tokens": 1000000,
    "mix": [],
    "seed": 20260630,
    "output_shard_rows": 100000,
    "compression": "zstd",
    "quality_keep": "high,mid,medium,unknown",
    "allow_science_shortfall_to_english_edu": true,
    "max_files_per_family": 1,
    "max_docs_per_file": 1000,
    "progress_every_files": 25,
    "overwrite": true,
    "dry_run": false
  }
}
