experiment:
  name: laoyao_2b_moe_rebalanced_200b
  seed: 42
  output_dir: /mnt/beegfs/yi/laoyao_2b_moe/runs/laoyao_2b_moe_rebalanced_200b

paths:
  model_config: /mnt/beegfs/yi/laoyao_2b_moe/model/nemo_megatron/laoyao_2b_moe_nemo_megatron.yaml
  train_data: /mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701
  validation_messages: /mnt/beegfs/yi/laoyao_2b_moe/dataset/val/data/heldout_2p8k.jsonl
  validation_sft: /mnt/beegfs/yi/laoyao_2b_moe/dataset/val/data/heldout_2p8k_sft_prompt_completion.jsonl
  validation_packed_text: /mnt/beegfs/yi/laoyao_2b_moe/dataset/val/data/heldout_2p8k_packed_text.jsonl

training:
  target_tokens: 200000000000
  global_batch_size: 1024
  micro_batch_size: 1
  seq_length: 16384
  optimizer: adamw
  learning_rate: 3.0e-4
  min_learning_rate: 5.0e-6
  weight_decay: 0.1
  warmup_ratio: 0.01
  lr_schedule: cosine
  grad_clip: 1.0
  log_every: 10
  eval_every: 1000
  save_every: 1000
  eval_metrics: [nll, perplexity, accuracy]

notes:
  - Validation accuracy requires a task adapter over the 2.8K messages file.
  - Validation nll/perplexity can consume prompt/completion or packed_text forms.
  - Keep generated checkpoints and the 200B corpus out of git.
