Revert "Remove drop tokens and presoftmax for ds-v3 and remove useless test dataloader …"

Author: jerryli1981

examples/deepseek_v3/README.md

@@ -156,8 +156,8 @@ false \
 sel   \
 1.0 \
 100000  \
-/mnt/deepseek-datasets/mmap_deepseekv3_datasets_text_document   \
-/mnt/deepseek-datasets/mmap_deepseekv3_datasets_text_document   \
+/mnt/deepseek-datasets/mmap_deepseekv2_datasets_text_document   \
+/mnt/deepseek-datasets/mmap_deepseekv2_datasets_text_document   \
 /mnt/deepseek-ckpts/DeepSeek-V3-to-mcore-tp8-pp8-ep16  \
 1000000000  \
 10000   \
@@ -192,8 +192,8 @@ true \
 sel   \
 1.0 \
 100000  \
-/mnt/deepseek-datasets/mmap_deepseekv3_datasets_text_document   \
-/mnt/deepseek-datasets/mmap_deepseekv3_datasets_text_document   \
+/mnt/deepseek-datasets/mmap_deepseekv2_datasets_text_document   \
+/mnt/deepseek-datasets/mmap_deepseekv2_datasets_text_document   \
 /mnt/deepseek-ckpts/DeepSeek-V3-to-mcore-tp8-pp8-ep16  \
 10000  \
 100   \

examples/deepseek_v3/run_mcore_deepseek.sh

@@ -76,7 +76,7 @@ NUM_ATTENTION_HEADS=128
 NUM_LAYERS=61
 INTERMEDIATE_SIZE=18432
 MOE_INTERMEDIATE_SIZE=2048
-MAX_POSITION_EMBEDDINGS=163840
+MAX_POSITION_EMBEDDINGS=${SEQ_LEN}
 EXTRA_VOCAB_SIZE=467
 Q_LORA_RANK=1536
 KV_LORA_RANK=512
@@ -106,6 +106,8 @@ moe_options=" \
     --moe-router-enable-expert-bias \
     --mscale 1.0 \
     --mscale-all-dim 1.0 \
+    --moe-token-drop-policy probs \
+    --moe-router-pre-softmax \
     --moe-router-score-function sigmoid \
     --moe-router-bias-update-rate 0.001 \
     --moe-aux-loss-coeff 0.001 \

examples/qwen2_vl/pretrain_qwen.py

@@ -541,9 +541,9 @@ def train_valid_test_dataloaders_provider(train_val_test_num_samples):
         EnergonDataloader(get_loader(valid_ds, worker_config=worker_config))
         for valid_ds in valid_ds1
     ]
-    test_dataloader = None # NOTE: no test
+    test_dataloader = None
 
-    return EnergonDataloader(train_dataloader), valid_dataloader, None
+    return EnergonDataloader(train_dataloader), valid_dataloader, EnergonDataloader(test_dataloader)
 
 
 class EnergonDataloader: