yi_lu/laoyao_2b_moe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add proof-pile logic inputs to pretrain builder

Browse Source
This commit is contained in:
路易
2026-07-01 22:42:18 +08:00
parent cdd8e2f85b
commit f0f72767f5
11 changed files with 122 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
training/nemo_megatron/pretrain_2b_moe_200b.yaml
View File

@@ -5,7 +5,7 @@ experiment:
paths:
model_config: /mnt/beegfs/yi/laoyao_2b_moe/model/nemo_megatron/laoyao_2b_moe_nemo_megatron.yaml
train_data: /mnt/beegfs/yi/laoyao_2b_moe/dataset/pretrain/data/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701
train_data: /mnt/beegfs/yi/laoyao_2b_moe_pretraining_dataset/train/pretrain_rebalanced_web40_edu20_chinese10_science10_logic10_math5_code5_200b_v1_20260701
validation_messages: /mnt/beegfs/yi/laoyao_2b_moe/dataset/val/data/heldout_2p8k.jsonl
validation_sft: /mnt/beegfs/yi/laoyao_2b_moe/dataset/val/data/heldout_2p8k_sft_prompt_completion.jsonl
validation_packed_text: /mnt/beegfs/yi/laoyao_2b_moe/dataset/val/data/heldout_2p8k_packed_text.jsonl