[Bug Fix] Fix benchmark caused by recently dir and requirements change (#2228)

* fix benchmark

* more steps for bigru crf

Author: FrostML

examples/machine_translation/transformer/deploy/python/inference.py

@@ -281,11 +281,11 @@ def do_inference(args):
 
     if args.profile:
         import importlib
-        import util.recorder as recorder
+        import tls.recorder as recorder
         try:
             mod = importlib.import_module("auto_log")
         except ImportError:
-            mod = importlib.import_module("util.recorder")
+            mod = importlib.import_module("tls.recorder")
         args.mod = mod
 
     do_inference(args)

examples/machine_translation/transformer/deploy/python/util/recorder.py

@@ -17,7 +17,7 @@
 import paddle
 import psutil
 
-from util.benchmark_utils import PaddleInferBenchmark
+from tls.benchmark_utils import PaddleInferBenchmark
 
 import paddle
 if paddle.is_compiled_with_cuda():

examples/machine_translation/transformer/static/train.py

@@ -19,7 +19,7 @@
 sys.path.append(
     os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
 import reader
-from util.record import AverageStatistical
+from tls.record import AverageStatistical
 
 FORMAT = '%(asctime)s-%(levelname)s: %(message)s'
 logging.basicConfig(level=logging.INFO, format=FORMAT)

examples/machine_translation/transformer/util/distributed_utils.py renamed to examples/machine_translation/transformer/tls/distributed_utils.py

@@ -16,8 +16,8 @@
 from paddlenlp.utils.log import logger
 from paddlenlp.utils import profiler
 
-from util.record import AverageStatistical
-from util.to_static import apply_to_static
+from tls.record import AverageStatistical
+from tls.to_static import apply_to_static
 
 
 def parse_args():

examples/machine_translation/transformer/util/record.py renamed to examples/machine_translation/transformer/tls/record.py

@@ -52,6 +52,6 @@ null:null
 ===========================train_benchmark_params==========================
 batch_size:32
 fp_items:fp32
-epoch:1
+epoch:3
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile  
 flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096

examples/machine_translation/transformer/util/to_static.py renamed to examples/machine_translation/transformer/tls/to_static.py

@@ -272,7 +272,7 @@ elif [ ${MODE} = "whole_infer" ];then
 elif [ ${MODE} = "benchmark_train" ];then
     if [ ${model_name} == "bigru_crf" ]; then
         rm -rf ./data/lexical_analysis_dataset_tiny ./data/lexical_analysis_dataset_tiny.tar.gz
-        wget -nc -P ./data/ https://bj.bcebos.com/paddlenlp/datasets/lexical_analysis_dataset_tiny.tar.gz --no-check-certificate
+        python ${BENCHMARK_ROOT}/paddlecloud/file_upload_download.py --remote-path frame_benchmark/paddle/PaddleNLP/lexical_analysis_dataset_tiny/ --local-path ./data/ --mode download
         cd ./data/ && tar xfz lexical_analysis_dataset_tiny.tar.gz && cd .. 
     fi
 
@@ -288,6 +288,8 @@ elif [ ${MODE} = "benchmark_train" ];then
         ln -s hdf5_lower_case_1_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5/wikicorpus_en_seqlen512/ wikicorpus_en_seqlen512
 
         cd ..
+
+        python -m pip install h5py -i https://mirror.baidu.com/pypi/simple
     fi
 
     if [[ ${model_name} =~ gpt* ]]; then

examples/machine_translation/transformer/train.py

@@ -15,7 +15,7 @@ python -m pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple
 python -m pip install setuptools_scm 
 python -m pip install Cython 
 python -m pip install -r ../requirements.txt  -i https://pypi.tuna.tsinghua.edu.cn/simple
-python -m pip install pybind11 regex sentencepiece tqdm visualdl attrdict pyyaml -i https://mirror.baidu.com/pypi/simple
+python -m pip install pybind11 regex sentencepiece tqdm visualdl attrdict pyyaml h5py -i https://mirror.baidu.com/pypi/simple
 
 python -m pip install -e ../
 # python -m pip install paddlenlp    # PDC 镜像中安装失败