[AutoParallel] Open dynamic sharding CI test (#10793)

waliwali777 · web-flow · commit 15d49bbe957e · 2025-07-10T21:21:26.000+08:00
* open sharding CI test and remove flag

* update loss_base
diff --git a/scripts/distribute/ci_case_auto.sh b/scripts/distribute/ci_case_auto.sh
@@ -166,7 +166,7 @@ function llm_gpt_case_list_auto() {
         llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2
         llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2-PP2
         llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2
-        # llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2_intermediate
+        llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2_intermediate
         llm_gpt_pir_auto_bs4_TP2
         llm_gpt_pir_auto_bs4_TP2_PP2
         llm_gpt_pir_auto_bs8_DP2_TP2_PP2
@@ -225,7 +225,7 @@ function llama_dygraph_auto_bs4_bf16_SD2() {
         
         export CUDA_DEVICE_MAX_CONNECTIONS=1
 
-        flags=("" "FLAGS_fuse_allreduce_in_opt" "FLAGS_fuse_reducescatter_in_opt" "FLAGS_enable_tensor_fusion FLAGS_enable_sharding_overlap")
+        flags=("" "FLAGS_enable_tensor_fusion FLAGS_enable_sharding_overlap")
         for i in "${!flags[@]}"; do
             flag="${flags[$i]}"
 
@@ -292,7 +292,7 @@ function llama_dygraph_auto_bs4_bf16_SD2() {
                     --tensor_parallel_degree 1 \
                     --sharding "stage1" \
                     --data_parallel_config "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate" \
-                    --sharding_parallel_config "" \
+                    --sharding_parallel_config "enable_tensor_fusion enable_overlap" \
                     --to_static 0 \
                     --amp_custom_black_list "reduce_sum" "c_softmax_with_cross_entropy" \
                     --amp_custom_white_list "lookup_table" "lookup_table_v2" \
@@ -304,10 +304,6 @@ function llama_dygraph_auto_bs4_bf16_SD2() {
                 echo "result: loss=$loss ips=$ips mem=$mem"
                 echo "flag=$flag acc_step=$acc_step"
                 if [ -z "$flag" ]; then
-                    loss_base=9.23504791
-                elif [ "$flag" = "FLAGS_fuse_allreduce_in_opt" ]; then
-                    loss_base=9.23502579
-                elif [ "$flag" = "FLAGS_fuse_reducescatter_in_opt" ]; then
                     loss_base=9.23504105
                 elif [ "$flag" = "FLAGS_enable_tensor_fusion FLAGS_enable_sharding_overlap" ]; then
                     if [ $acc_step -eq 1 ]; then
@@ -2798,11 +2794,11 @@ function llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2_intermediate() {
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem loss_md5=$loss_md5"
     # loss_base=10.58456802     # note: need to debug
-    loss_base=10.56716251
+    loss_base=10.56668091
     ips_base=-1
     mem_base=-1
     if [ $IS_A100 -ne 0 ];then
-        loss_base=10.56166935 # after add dropout spmd
+        loss_base=10.56199837 # after add dropout spmd
     fi
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
     echo "=========== $FUNCNAME run  end ==========="
@@ -3989,7 +3985,6 @@ function llama_baichuan_dygraph_auto_sp_async_reduce_scatter_bs8_bf16_DP4-MP2-SP
         export NVIDIA_TF32_OVERRIDE=0
 
         export CUDA_DEVICE_MAX_CONNECTIONS=1
-        export FLAGS_fuse_reducescatter_in_opt=1
         export FLAGS_enable_inplace_master_grad=1
         export FLAGS_auto_parallel_align_mode=1
         export FLAGS_max_inplace_grad_add=65536
diff --git a/tests/test_tipc/static/auto_parallel/baichuan2/N4C32/baichuan-inc-baichuan-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP4_PP1_Sharding8_Stage1.sh b/tests/test_tipc/static/auto_parallel/baichuan2/N4C32/baichuan-inc-baichuan-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP4_PP1_Sharding8_Stage1.sh
@@ -20,7 +20,6 @@ param+="nnodes=4 "
 param+="model_type=baichuan2_13b "
 param+='dynamic_auto=_dynamic_auto '
 
-export FLAGS_fuse_reducescatter_in_opt=1
 export FLAGS_enable_sharding_overlap=1
 export FLAGS_enable_tensor_fusion=1
 
diff --git a/tests/test_tipc/static/auto_parallel/llama2/N4C32/meta-llama-Llama-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP1_PP4_VPP5_Sharding8_Stage1.sh b/tests/test_tipc/static/auto_parallel/llama2/N4C32/meta-llama-Llama-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP1_PP4_VPP5_Sharding8_Stage1.sh
@@ -20,9 +20,6 @@ param+="nnodes=4 "
 param+="model_type=llama2_13b "
 param+='dynamic_auto=_dynamic_auto '
 
-# This optimization currently only runs in the dynamic automatic parallelism of Llama7B.
-export FLAGS_fuse_reducescatter_in_opt=1
-
 # Enable tensor fusion and sharding overlap optimization
 export FLAGS_enable_tensor_fusion=1
 export FLAGS_enable_sharding_overlap=1
diff --git a/tests/test_tipc/static/auto_parallel/llama2/N4C32/meta-llama-Llama-2-7b_pretrain_dynamic_auto_bs32_bf16_Sharding32_Stage2.sh b/tests/test_tipc/static/auto_parallel/llama2/N4C32/meta-llama-Llama-2-7b_pretrain_dynamic_auto_bs32_bf16_Sharding32_Stage2.sh
@@ -20,9 +20,6 @@ param+="nnodes=4 "
 param+="model_type=llama2_7b "
 param+='dynamic_auto=_dynamic_auto '
 
-# This optimization currently only runs in the dynamic automatic parallelism of Llama7B.
-export FLAGS_fuse_reducescatter_in_opt=1
-
 # Enable tensor fusion and sharding overlap optimization
 export FLAGS_enable_tensor_fusion=1
 export FLAGS_enable_sharding_overlap=1
diff --git a/tests/test_tipc/static/auto_parallel/qwen/N4C32/qwen-14b_pretrain_dynamic_auto_bs32_bf16_DP1_MP2_Sharding16_Stage1.sh b/tests/test_tipc/static/auto_parallel/qwen/N4C32/qwen-14b_pretrain_dynamic_auto_bs32_bf16_DP1_MP2_Sharding16_Stage1.sh
@@ -20,7 +20,6 @@ param+="nnodes=4 "
 param+="model_type=qwen_14b "
 param+='dynamic_auto=_dynamic_auto '
 
-export FLAGS_fuse_reducescatter_in_opt=1
 export FLAGS_enable_tensor_fusion=1
 export FLAGS_enable_sharding_overlap=1
 
