fix tipc bugs from qa

Author: wangzhen38

models/rank/dlrm/config.yaml

@@ -37,6 +37,10 @@ runner:
   split_file_list: False
   thread_num: 1
 
+   # use inference save model
+  inference: True  # 静态图训练时保存为inference model
+  save_inference_feed_varnames: ["C1","C2","C3","C4","C5","C6","C7","C8","C9","C10","C11","C12","C13","C14","C15","C16","C17","C18","C19","C20","C21","C22","C23","C24","C25","C26","dense_input"] # inference model 的feed参数的名字
+  save_inference_fetch_varnames: ["sigmoid_0.tmp_0"] # inference model 的fetch参数的名字
 
 # hyper parameters of user-defined network
 hyper_parameters:
@@ -51,5 +55,5 @@ hyper_parameters:
   bot_layer_sizes: [512, 256, 64, 16]
   sparse_feature_number: 1000001
   sparse_feature_dim: 16
-  top_layer_sizes: [512, 256, 2]
+  top_layer_sizes: [512, 256, 1]
   num_field: 26

models/rank/dlrm/criteo_reader.py

@@ -22,6 +22,25 @@ class RecDataset(IterableDataset):
     def __init__(self, file_list, config):
         super(RecDataset, self).__init__()
         self.file_list = file_list
+        if config:
+            use_fleet = config.get("runner.use_fleet", False)
+            self.inference = config.get("runner.inference", False)
+        else:
+            use_fleet = False
+        if use_fleet:
+            worker_id = paddle.distributed.get_rank()
+            worker_num = paddle.distributed.get_world_size()
+            file_num = len(file_list)
+            if file_num < worker_num:
+                raise ValueError(
+                    "The number of data files is less than the number of workers"
+                )
+            blocksize = int(file_num / worker_num)
+            self.file_list = file_list[worker_id * blocksize:(worker_id + 1) *
+                                       blocksize]
+            remainder = file_num - (blocksize * worker_num)
+            if worker_id < remainder:
+                self.file_list.append(file_list[-(worker_id + 1)])
         self.init()
 
     def init(self):
@@ -78,4 +97,8 @@ def __iter__(self):
                     output_list.append(
                         np.array(output[-1][1]).astype("float32"))
                     # list
-                    yield output_list
+                    #yield output_list
+                    if self.inference:
+                        yield output_list[1:]
+                    else:
+                        yield output_list

models/recall/tisas/config_test.yaml

@@ -0,0 +1,42 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+runner:
+  train_data_dir: "data/sample_data"
+  train_reader_path: "movielens_reader"  # importlib format
+  train_batch_size: 2
+  model_save_path: "output_model_tisas"
+  mode: "test"
+  use_gpu: False 
+  epochs: 3
+  print_interval: 10
+  
+  test_data_dir: "data/sample_data"
+  infer_reader_path: "movielens_reader"  # importlib format
+  infer_batch_size: 1
+  infer_load_path: "output_model_tisas"
+  infer_start_epoch: 2
+  infer_end_epoch: 3
+
+hyper_parameters:
+  optimizer: 
+    class: adam
+    learning_rate: 0.001
+  num_users: 6040
+  num_items: 3416
+  num_blocks: 2
+  num_heads: 1
+  maxlen: 50
+  hidden_units: 50
+  time_span: 256

test_tipc/configs/tisas/paddle_infer.py

@@ -23,7 +23,7 @@
 from importlib import import_module
 
 __dir__ = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
+sys.path.append(os.path.abspath(os.path.join(__dir__, '../../../tools')))
 from utils.utils_single import load_yaml, load_dy_model_class, get_abs_model
 from utils.save_load import save_model, load_model
 from paddle.io import DistributedBatchSampler, DataLoader

test_tipc/configs/tisas/to_static.py

@@ -22,7 +22,7 @@
 
 __dir__ = os.path.dirname(os.path.abspath(__file__))
 # sys.path.append(__dir__)
-sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
+sys.path.append(os.path.abspath(os.path.join(__dir__, '../../../tools')))
 
 from utils.utils_single import load_yaml, load_dy_model_class, get_abs_model, create_data_loader
 from utils.save_load import load_model, save_model, save_jit_model

test_tipc/configs/tisas/train_infer_python.txt

@@ -9,8 +9,8 @@ runner.model_save_path
 runner.train_batch_size:lite_train_lite_infer=2|whole_train_whole_infer=128|whole_infer=1|lite_train_whole_infer=2
 runner.infer_load_path:null
 train_model_name:lite_train_lite_infer=0|whole_train_whole_infer=101|whole_infer=101|lite_train_whole_infer=0
-runner.test_data_dir:test_tipc/data
-runner.train_data_dir:../../../test_tipc/data
+runner.test_data_dir:test_tipc/data/infer
+runner.train_data_dir:../../../test_tipc/data/train
 ##
 trainer:norm_train
 norm_train:-u tools/trainer.py -m ./models/recall/tisas/config.yaml -o runner.print_interval=2
@@ -27,7 +27,7 @@ null:null
 ===========================infer_params===========================
 runner.model_save_path:
 runner.model_init_path:
-norm_export:-u ./to_static.py -m ./models/recall/tisas/config.yaml -o runner.CE=true
+norm_export:-u test_tipc/configs/tisas/to_static.py -m ./models/recall/tisas/config.yaml -o runner.CE=true
 quant_export:null
 fpgm_export:null
 distill_export:null
@@ -37,15 +37,15 @@ null:null
 infer_model:test_tipc/save_tisas_model
 infer_export:null
 infer_quant:False
-inference:-u ./paddle_infer.py --model_name=tisas --reader_file=models/recall/tisas/movielens_reader.py
+inference:-u test_tipc/configs/tisas/paddle_infer.py --model_name=tisas --reader_file=models/recall/tisas/movielens_reader.py
 --use_gpu:True|False
 --enable_mkldnn:True|False
 --cpu_threads:1|6
 --batchsize:1
 --enable_tensorRT:False
 --precision:fp32
 --model_dir:
---data_dir:test_tipc/data
+--data_dir:test_tipc/data/infer
 --save_log_path:./test_tipc/output/
 --benchmark:True
 null:null

test_tipc/test_train_inference_python.sh

@@ -359,7 +359,7 @@ else
                     #run inference
                     eval $env
                     save_infer_path="${save_log}"
-                    if [ ${inference_dir} != "null" ] && [ ${inference_dir} != '##' ]; then
+                    if [ "${inference_dir}" != "null" ] && [ "${inference_dir}" != '##' ]; then
                         infer_model_dir="${save_infer_path}/${inference_dir}"
                     else
                         infer_model_dir=${save_infer_path}