{
  "name": "pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_v1_20260630",
  "target_tokens": 200000000000,
  "target_mix": {
    "english_web": 0.4,
    "english_edu": 0.2,
    "chinese_clean": 0.1,
    "science": 0.1,
    "logic": 0.1,
    "math": 0.05,
    "code": 0.05
  },
  "source_plan": {
    "english_web": {
      "datasets": [
        "OpenBMB/Ultra-FineWeb ultrafineweb_en from ModelScope",
        "HuggingFaceFW/fineweb if available"
      ],
      "default_paths": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_en/**/*.parquet"
      ],
      "expected_format": "parquet",
      "notes": "Use the ModelScope Ultra-FineWeb English mirror as the current practical source; FineWeb parquet remains acceptable if linked."
    },
    "english_edu": {
      "datasets": [
        "AI-ModelScope/fineweb-edu",
        "local cjy scored FineWeb-Edu-derived pool as auto_text fallback"
      ],
      "default_paths": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_fineweb_edu/**/*.parquet"
      ],
      "notes": "Full build should use the downloaded ModelScope FineWeb-Edu parquet. The cjy scored pool is mixed and must go through auto_text classification."
    },
    "chinese_clean": {
      "datasets": [
        "OpenBMB/Ultra-FineWeb ultrafineweb_zh from ModelScope",
        "CCI3-HQ and SkyPile-150B if later linked"
      ],
      "default_paths": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/ms_ultra_fineweb_zh/**/*.parquet"
      ],
      "notes": "Current build uses Ultra-FineWeb Chinese as the reachable clean Chinese source."
    },
    "science": {
      "datasets": [
        "medmcqa",
        "proofwriter",
        "scienceqa",
        "sciq",
        "qasc",
        "openbookqa"
      ],
      "default_paths": [
        "/mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/science_*/*/*.parquet"
      ],
      "notes": "Builder supports either text/content rows or QA-style question/choices/answer rows."
    },
    "logic": {
      "datasets": [
        "explicit logic datasets if provided",
        "small logic/proof QA traces from science sources"
      ],
      "notes": "No clean standalone jiayi logic pool was found. Do not relabel the whole cjy scored math/world pool as logic."
    },
    "math": {
      "datasets": [
        "existing cjy scored math pool: proof-pile-2 and finemath-derived rows"
      ],
      "default_paths": [
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/scored/edu_fineweb_math_knowledge_cross_near_h1_v1_20260620/rank_*/*.jsonl"
      ],
      "notes": "Use as auto_text input so only proof-pile/finemath/math rows flow into math; do not pass this mixed pool as --math-input."
    },
    "code": {
      "datasets": [
        "existing cjy cleaned CodeParrot/multilang code pool"
      ],
      "default_paths": [
        "/mnt/beegfs/cjy/cjy-training/laoyao_model/data/dedup/pretrain_python_clean_code_multilang_cross_near_h1_min256_v1_20260623/*.parquet"
      ],
      "notes": "Sample 5% from jiayi's deduplicated multilang code pool instead of downloading new code."
    }
  },
  "quality_keep": [
    "high",
    "mid",
    "medium",
    "unknown"
  ],
  "shortfall_policy": {
    "source": "science",
    "target": "english_edu",
    "enabled_by_default": true,
    "description": "Download all available science QA data first. If science has fewer tokens than its 10% target, add the shortfall to english_edu."
  },
  "notes": [
    "This recipe intentionally reduces code to 5% for stage-1 pretraining.",
    "The previous code40/text60 mixture is useful as cleaned input inventory, not as the target mixture.",
    "Science shortfall is explicitly reassigned to english_edu and recorded in report.json. Other missing quotas should remain visible.",
    "Run dataset/pretrain/scripts/start_rebalanced_pretrain_dataset.sh to materialize parquet shards and report.json."
  ]
}
