Merge branch 'master' into fat_fix

wangzhen38 · web-flow · commit ed786043d6d3 · 2022-04-01T11:08:24.000+08:00
diff --git a/datasets/Avazu_flen/data_config.yaml b/datasets/Avazu_flen/data_config.yaml
@@ -14,7 +14,7 @@
 
 
 runner:
-  raw_file_dir: "path"  # raw_data dir
+  raw_file_dir: "raw_file/train"  # raw_data dir
   raw_filled_file_dir: "./raw_data" # raw_data_filled dir
   train_data_dir: "./train_data_full"  # train datasets
   test_data_dir:  "./test_data_full"  # test datasets
diff --git a/datasets/Avazu_flen/preprocess.py b/datasets/Avazu_flen/preprocess.py
@@ -59,7 +59,7 @@ def __init__(self, config):
         self.min_threshold = self.config.get("runner.min_threshold")
         self.feature_map_cache = self.config.get("runner.feature_map_cache")
 
-        # self.filled_raw()
+        self.filled_raw()
 
         self.init()
 
diff --git a/datasets/Avazu_flen/readme.md b/datasets/Avazu_flen/readme.md
@@ -2,11 +2,11 @@
 #### 1.Get raw datasets:
 you can go to：[https://www.kaggle.com/c/avazu-ctr-prediction/data](https://www.kaggle.com/c/avazu-ctr-prediction)
 
-将下载的原始数据目录配置在data_config.yaml中，执行命令获取全量数据
+将下载的数据解压后，只保留训练集即可，且命名为`train``
 
 | 名称 | 说明 | 
 | -------- | -------- | 
-| raw_file_dir | 原始数据集目录 |
+| raw_file | 原始数据集目录 |
 | raw_filled_file_dir     | 原始数据缺失值处理后的目录     |
 |   train_data_dir   | 训练集存放目录     | 
 |   test_data_dir   | 测试集存放目录     | 
@@ -15,9 +15,9 @@ you can go to：[https://www.kaggle.com/c/avazu-ctr-prediction/data](https://www
 | feature_map_cache     | 特征缓存数据     | 
 
 
-
+然后执行脚本
 ```bash
-sh data_process.sh
+sh run.sh
 ```
 #### 2.Get preprocessd datasets:
 you can also go to: [AiStudio数据集](https://aistudio.baidu.com/aistudio/datasetdetail/125200)
diff --git a/datasets/Avazu_flen/run.sh b/datasets/Avazu_flen/run.sh
@@ -1 +1,7 @@
+mkdir train_data_full
+mkdir test_data_full
+mkdir raw_file
+mkdir raw_filled_file_dir
+mv train ./raw_file
+
 python preprocess.py -m data_config.yaml
diff --git a/models/rank/autofis/readme.md b/models/rank/autofis/readme.md
@@ -5,17 +5,19 @@
 ```
 ├── data #样例数据
     ├── sample_data #样例数据
-        ├── train
-            ├── train_x.npy #训练数据样例
-            ├── train_y.npy #训练数据样例
+        ├── sample_train_x.txt #训练数据样例
+        ├── sample_train_y.txt #训练数据样例
 ├── __init__.py
-├── README.md #文档
 ├── config.yaml # sample数据配置
 ├── config_bigdata.yaml # 全量数据配置
-├── net.py # 模型核心组网
 ├── criteo_reader.py #数据读取程序
 ├── dygraph_model.py # 构建动态图
+├── metrics.py # 自定义评估指标
+├── net.py # 模型核心组网
+├── optimizer.py # 自定义优化器
 ├── trainer.py # 训练脚本
+├── README.md #文档
+├── trainer.py #训练入口
 ```
 
 注：在阅读该示例前，建议您先了解以下内容：
@@ -54,7 +56,7 @@ os : windows/linux/macos
 # cd models/rank/autofis # 在任意目录均可运行
 # 动态图训练
 python trainer.py -m config.yaml # stage0：自动搜索最佳特征组合 全量数据运行config_bigdata.yaml
-python trainer.py -m config.yaml -o stage=1 # stage1：训练最终模 全量数据运行config_bigdata.yaml型
+python trainer.py -m config.yaml -o stage=1 # stage1：训练最终模型 全量数据运行config_bigdata.yaml
 
 # 动态图预测
 python -u ../../../tools/infer.py -m config.yaml -o stage=1 # 全量数据运行config_bigdata.yaml
diff --git a/models/rank/dcn/net.py b/models/rank/dcn/net.py
@@ -88,6 +88,7 @@ def __init__(self, sparse_feature_number, sparse_feature_dim,
             if acts[i] == 'relu':
                 act = paddle.nn.ReLU()
                 self.add_sublayer('act_%d' % i, act)
+                self._mlp_layers.append(act)
 
         self.fc = paddle.nn.Linear(
             in_features=self.layer_sizes[-1] + self.sparse_num_field *
diff --git a/models/rank/dcn_v2/README.md b/models/rank/dcn_v2/README.md
@@ -4,9 +4,7 @@
 
 ```
 ├── data # 样例数据
-    ├── sample_data # 样例数据
-        ├── train
-            ├── sample_train.txt # 训练数据样例
+    ├── sample_train.txt # 训练数据样例
 ├── __init__.py
 ├── README.md # 文档
 ├── config.yaml # sample数据配置
diff --git a/models/rank/dcn_v2/net.py b/models/rank/dcn_v2/net.py
@@ -169,6 +169,7 @@ def __init__(self,
             if acts[i] == 'relu':
                 act = paddle.nn.ReLU()
                 self.add_sublayer('act_%d' % i, act)
+                self._mlp_layers.append(act)
 
     def forward(self, feat_embeddings):
         # y_dnn = paddle.reshape(feat_embeddings,[feat_embeddings.shape[0], -1])
diff --git a/models/rank/deepfefm/net.py b/models/rank/deepfefm/net.py
@@ -224,6 +224,7 @@ def __init__(self,
             if acts[i] == 'relu':
                 act = paddle.nn.ReLU()
                 self.add_sublayer('act_%d' % i, act)
+                self._mlp_layers.append(act)
 
     def forward(self, feat_embeddings):
         y_dnn = paddle.reshape(feat_embeddings, [-1, self.input_size])
diff --git a/models/rank/deepfm/net.py b/models/rank/deepfm/net.py
@@ -164,6 +164,7 @@ def __init__(self, sparse_feature_number, sparse_feature_dim,
             if acts[i] == 'relu':
                 act = paddle.nn.ReLU()
                 self.add_sublayer('act_%d' % i, act)
+                self._mlp_layers.append(act)
 
     def forward(self, feat_embeddings):
         y_dnn = paddle.reshape(feat_embeddings,
diff --git a/models/rank/flen/README.md b/models/rank/flen/README.md
@@ -63,8 +63,14 @@ os : windows/linux/macos
 
 ## 快速开始
 
-
-本文提供了[FLEN-Paddle AiStudio项目](https://aistudio.baidu.com/aistudio/projectdetail/3247609)可以供您快速体验，进入项目快速开始。
+本文提供了样例数据可以供您快速体验，在任意目录下均可执行。在FLEN模型目录的快速执行命令如下： 
+```bash
+# 进入模型目录
+# cd models/rank/flen # 在任意目录均可运行
+# 动态图训练
+python -u ../../../tools/trainer.py -m config.yaml # 全量数据运行config_bigdata.yaml 
+# 动态图预测
+python -u ../../../tools/infer.py -m config.yaml # 全量数据运行config_bigdata.yaml
 
 
 ## 模型组网
diff --git a/models/rank/flen/config.yaml b/models/rank/flen/config.yaml
@@ -14,7 +14,7 @@
 
 
 runner:
-  train_data_dir: "./data/sample_data/dataset"
+  train_data_dir: "./data/sample_data/train"
   train_reader_path: "avazu_reader" # importlib format
   use_gpu: False
   use_auc: True
@@ -25,7 +25,7 @@ runner:
 
   #model_init_path: "output_model/0" # init model
   model_save_path: "output_model_flen"
-  test_data_dir:  "./data/sample_data/dataset" #"../../../../data/test"
+  test_data_dir:  "./data/sample_data/train" #"../../../../data/test"
   infer_reader_path: "avazu_reader" # importlib format
   infer_batch_size: 3 #512
   infer_load_path: "output_model_flen"
@@ -41,7 +41,7 @@ hyper_parameters:
     learning_rate: 0.04
     strategy: async
   # user-defined <key, value> pairs
-  sparse_inputs_slots: 23
+  sparse_inputs_slots: 22
   sparse_feature_number: 20 #1544488
   sparse_num_field: 3
   sparse_feature_dim: 32
diff --git a/models/rank/slot_dnn/config_online.yaml b/models/rank/slot_dnn/config_online.yaml
@@ -58,7 +58,7 @@ table_parameters:
     table_class: "MemorySparseTable"
     shard_num: 10
     accessor:
-      accessor_class: "CtrCommonAccessor"
+      accessor_class: "SparseAccessor"
       fea_dim: 11
       embedx_dim: 8
       embedx_threshold: 10
diff --git a/models/recall/mhcn/README.md b/models/recall/mhcn/README.md
@@ -6,10 +6,6 @@
 
 ```
 ├── data # 样例数据
-    ├── train
-        ├── train.txt
-    ├── test
-        ├── test.txt
     ├── ratings.txt
     ├── trusts.txt
 ├── __init__.py
diff --git a/tools/static_ps_online_trainer.py b/tools/static_ps_online_trainer.py
@@ -254,7 +254,7 @@ def run_online_worker(self):
             logger.info("training a new day {}, end_day = {}".format(
                 day, self.end_day))
             if last_day != -1 and int(day) < last_day:
-                day = int(get_next_day(day))
+                day = get_next_day(day)
                 continue
             # base_model_saved = False
             for pass_id in range(1, 1 + len(self.online_intervals)):
@@ -409,18 +409,24 @@ def run_online_worker(self):
                                 monitor_data=metric_str)
                 fleet.barrier_worker()
 
+            logger.info("shrink table")
+            begin = time.time()
+            fleet.shrink()
+            end = time.time()
+            logger.info("shrink table done, cost %s min" % (
+                (end - begin) / 60.0))
+
             if fleet.is_first_worker():
                 last_base_day, last_base_path, last_base_key = get_last_save_xbox_base(
                     self.save_model_path, self.hadoop_client)
                 logger.info(
                     "one epoch finishes, get_last_save_xbox, last_base_day = {}, last_base_path = {}, last_base_key = {}".
                     format(last_base_day, last_base_path, last_base_key))
-                next_day = int(get_next_day(day))
-                if next_day <= last_base_day:
+                next_day = get_next_day(day)
+                if int(next_day) <= last_base_day:
                     logger.info("batch model/base xbox model exists")
                 else:
                     xbox_base_key = int(time.time())
-                    fleet.shrink()
                     save_xbox_model(self.save_model_path, next_day, -1,
                                     self.exe, self.inference_feed_vars,
                                     self.inference_target_var,
diff --git a/tools/utils/static_ps/flow_helper.py b/tools/utils/static_ps/flow_helper.py
@@ -118,7 +118,7 @@ def load_model(model_path, mode, client):
         if os.path.exists(local_path):
             shutil.rmtree(local_path)
         os.mkdir(local_path)
-        client.download(model_path + "/dnn_plugin", local_path)
+        client.download(model_path + "/dnn_plugin/", local_path)
     fleet.load_model(model_path, mode)
 
 
@@ -354,7 +354,8 @@ def get_last_save_xbox(output_path, client):
         logger.info("get_last_save_xbox donefile_path {} is file".format(
             donefile_path))
         pre_content = client.cat(donefile_path)
-        logger.info("get_last_save_xbox get a pre_content = ", pre_content)
+        logger.info("get_last_save_xbox get a pre_content = {}".format(
+            pre_content))
         last_dict = json.loads(pre_content.split("\n")[-1])
         last_day = int(last_dict["input"].split("/")[-3])
         last_pass = int(last_dict["input"].split("/")[-2].split("-")[-1])
