Fix ckpt convert bug1 (#9522)

* refine log

* refine

* fix

* fix

Author: zhangbo9674

paddlenlp/trainer/utils/ckpt_converter.py

@@ -269,7 +269,7 @@ def gen_metadata_and_prepare_source_state_dict(self):
 
             malloc_size = 0
             for opt_state_name, opt_state_value in optimizer_state_dict.items():
-                malloc_size += opt_state_value.numel() * opt_state_value.element_size()
+                malloc_size += opt_state_value.numel().numpy() * opt_state_value.element_size()
             malloc_size = malloc_size / 2**20
             logger.debug(f"{malloc_size} MB of GPU memory were allocated.")
 
@@ -529,6 +529,7 @@ def load_state_dict_and_rename(self):
             rank_access_files[self.cur_rank] = self.cur_rank_optimizer_state_file_names
 
         global_rank_access_files = self.gather_global_object(rank_access_files)
+        logger.info(f"The file(s) to be loaded for the global rank are: {global_rank_access_files}")
         need_read_files = get_rank_to_read_files(global_rank_access_files, global_rank_access_files)
         logger.info(f"The file(s) to be loaded for the current rank are: {need_read_files}")
         self.cur_rank_loaded_state_dict = {}
@@ -553,8 +554,7 @@ def load_state_dict_and_rename(self):
         memory_size = 0
         for file, state_dict in self.cur_rank_loaded_state_dict.items():
             for k, v in state_dict.items():
-                memory_size += v.numel() * v.element_size()
-
+                memory_size += v.numel().numpy() * v.element_size()
         memory_size = memory_size / 2**20
         logger.debug(
             f"The current rank has finished loading the checkpoint file and has allocated {memory_size} MB of GPU memory."

scripts/distribute/ci_case_auto.sh

@@ -95,6 +95,7 @@ function llama_case_list_auto() {
         llama_dy2st_auto_bs4_bf16_DP1-MP1-PP4-SD2
         llama_align_dygraph_dy2st_auto_bs2_bf16_DP2-MP1-PP1
         llama_pir_auto_fuse_ffn_attention_qkv_MP2
+        llama_convert_hybrid_ckpt_to_auto_parallel_bs2_fp32_DP2-MP1-PP1
         llama_align_dygraph_dy2st_pir_auto_bs2_bf16_DP2-MP2-PP1-SP
         llama_align_dygraph_dy2st_pir_auto_bs2_bf16_DP2-MP2-PP2-SP
         llama_align_dygraph_dy2st_pir_auto_grad_merge_bs2_fp32_DP1-MP1-PP1
@@ -1420,6 +1421,7 @@ function llama_convert_hybrid_ckpt_to_auto_parallel_bs2_fp32_DP2-MP1-PP1() {
         --sharding "" \
         --to_static 0 \
         --num_hidden_layers 2 \
+        --unified_checkpoint false \
         >>${log_path}/$FUNCNAME 2>&1
     dy_loss=`cat $dy_case_log_dir/workerlog.0 | grep 'global_step: 4' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
     dy_ips=-1