Merge pull request #365 from seemingwang/master

use auc for naml

Author: fuyinno4

models/rank/naml/NAMLDataReader.py

@@ -63,6 +63,7 @@ def init(self):
         #line [0]id cate_id sub_cate_id [3]title content
         for file in self.article_file_list:
             with open(file, "r") as rf:
+
                 for l in rf:
                     line_x = [x.strip() for x in l.split('\t')]
                     id = line_x[0]

models/rank/naml/README.md

@@ -91,7 +91,7 @@ python3 -u ../../../tools/infer.py -m config.yaml
 #### Loss及Acc计算
 - 预测的结果为一个softmax向量，表示实际浏览文章和负采样文章同时出现的情况下被用户浏览的概率
 - 样本的损失函数值由交叉熵给出
-- 我们同时还会计算预测的acc，即top1的准确率
+- 我们同时还会计算预测的auc
 
 ## 效果复现
 为了方便使用者能够快速的跑通每一个模型，我们在每个模型下都提供了样例数据。
@@ -111,12 +111,11 @@ python3 -u ../../../tools/trainer.py -m config_bigdata.yaml
 以下为训练2个epoch的结果
 | 模型 | top1 acc | batch_size | epoch_num| Time of each epoch| 
 | :------| :------ | :------ | :------| :------ | 
-| naml | 0.43 | 512 | 4 | 约0.5小时 | 
+| naml | 0.72 | 50 | 3 | 约4小时 | 
 
 预测
 ```
 python3 -u ../../../tools/infer.py -m config_bigdata.yaml
 ```
 
-期待运行结果如下
-INFO - epoch: 1 done, acc: 0.427140, epoch time: 126.27 s
+期待预测auc为0.65

models/rank/naml/config_bigdata.yaml

@@ -17,16 +17,16 @@ runner:
   train_reader_path: "NAMLDataReader" # importlib format
   use_gpu: False
   train_batch_size: 50
-  epochs: 10
+  epochs: 3
   print_interval: 10
   #model_init_path: "output_model/0" # init model
   model_save_path: "output_model_all"
   infer_batch_size: 10
   infer_reader_path: "NAMLDataReader" # importlib format
   test_data_dir:  "../../../datasets/MIND/data/test"
   infer_load_path: "output_model_all"
-  infer_start_epoch: 0
-  infer_end_epoch: 10
+  infer_start_epoch: 3
+  infer_end_epoch: 4
 
 # hyper parameters of user-defined network
 hyper_parameters:

models/rank/naml/dygraph_model.py

@@ -16,11 +16,23 @@
 import paddle.nn as nn
 import paddle.nn.functional as F
 import math
+import numpy as np
 
 import net
 
 
 class DygraphModel():
+    def __init__(self):
+        self.bucket = 100000
+        self.absolute_limt = 200.0
+
+    def rescale(self, number):
+        if number > self.absolute_limt:
+            number = self.absolute_limt
+        elif number < -self.absolute_limt:
+            number = -self.absolute_limt
+        return (number + self.absolute_limt) / (self.absolute_limt * 2 + 1e-8)
+
     # define model
     def create_model(self, config):
         article_content_size = config.get(
@@ -63,8 +75,10 @@ def create_optimizer(self, dy_model, config):
     # define metrics such as auc/acc
     # multi-task need to define multi metric
     def create_metrics(self):
-        metrics_list_name = ["acc"]
-        auc_metric = paddle.metric.Accuracy()
+        # metrics_list_name = ["acc"]
+        # auc_metric = paddle.metric.Accuracy()
+        metrics_list_name = ["auc"]
+        auc_metric = paddle.metric.Auc(num_thresholds=self.bucket)
         metrics_list = [auc_metric]
         return metrics_list, metrics_list_name
 
@@ -77,18 +91,37 @@ def train_forward(self, dy_model, metrics_list, batch_data, config):
 
         loss = paddle.nn.functional.cross_entropy(
             input=raw, label=paddle.cast(labels, "float32"), soft_label=True)
-        correct = metrics_list[0].compute(raw, labels)
-        metrics_list[0].update(correct)
+
+        scaled = raw.numpy()
+        scaled_pre = []
+        [rows, cols] = scaled.shape
+        for i in range(rows):
+            for j in range(cols):
+                scaled_pre.append(1.0 - self.rescale(scaled[i, j]))
+                scaled_pre.append(self.rescale(scaled[i, j]))
+        scaled_np_predict = np.array(scaled_pre).reshape([-1, 2])
+        metrics_list[0].update(scaled_np_predict,
+                               paddle.reshape(labels, [-1, 1]))
+
         loss = paddle.mean(loss)
         print_dict = None
         return loss, metrics_list, print_dict
 
     def infer_forward(self, dy_model, metrics_list, batch_data, config):
-        label, sparse_tensor, dense_tensor = self.create_feeds(batch_data,
-                                                               config)
+        labels, sparse_tensor, dense_tensor = self.create_feeds(batch_data,
+                                                                config)
         raw = dy_model(sparse_tensor, None)
-        raw = paddle.nn.functional.softmax(raw)
-        correct = metrics_list[0].compute(raw, label)
-        metrics_list[0].update(correct)
+        #predict_raw = paddle.nn.functional.softmax(raw)
+
+        scaled = raw.numpy()
+        scaled_pre = []
+        [rows, cols] = scaled.shape
+        for i in range(rows):
+            for j in range(cols):
+                scaled_pre.append(1.0 - self.rescale(scaled[i, j]))
+                scaled_pre.append(self.rescale(scaled[i, j]))
+        scaled_np_predict = np.array(scaled_pre).reshape([-1, 2])
+        metrics_list[0].update(scaled_np_predict,
+                               paddle.reshape(labels, [-1, 1]))
 
         return metrics_list, None

models/rank/naml/net.py

@@ -84,7 +84,7 @@ def __init__(self, article_content_size, article_title_size, browse_size,
         self.sub_category_size = sub_category_size
         self.cate_dimension = cate_dimension
         self.word_dict_size = word_dict_size
-        self.conv_out_channel_size = 100
+        self.conv_out_channel_size = 400
         self.attention_projection_size = 100
         self.load_word_embedding()
         self.attention_vec = []