change to bigdata_set

Author: seemingwang

models/rank/naml/NAMLDataReader.py

@@ -64,12 +64,18 @@ def init(self):
         for file in self.article_file_list:
             with open(file, "r") as rf:
                 for l in rf:
-                    line = l.strip().split('\t')
-                    id = line[0]
+                    line_x = [x.strip() for x in l.split('\t')]
+                    id = line_x[0]
                     #line 0 cate   1:subcate,  2:title, 3 content;
-                    line = [[int(line[1])], [int(line[2])],
-                            [int(t) for t in line[3].split(" ")],
-                            [int(t) for t in line[4].split(" ")]]
+                    line = [[int(line_x[1])], [int(line_x[2])]]
+                    if len(line_x[3]) == 0:
+                        line.append([])
+                    else:
+                        line.append([int(t) for t in line_x[3].split(" ")])
+                    if len(line_x[4]) == 0:
+                        line.append([])
+                    else:
+                        line.append([int(t) for t in line_x[4].split(" ")])
                     line[2] += [self.word_dict_size] * (
                         self.article_title_size - len(line[2]))
                     line[3] += [self.word_dict_size] * (

models/rank/naml/config_bigdata.yaml

@@ -15,16 +15,16 @@
 runner:
   train_data_dir: "../../../datasets/MIND/data/train"
   train_reader_path: "NAMLDataReader" # importlib format
-  use_gpu: True
+  use_gpu: False
   train_batch_size: 50
   epochs: 2
-  print_interval: 2
+  print_interval: 10
   #model_init_path: "output_model/0" # init model
-  model_save_path: "output_model"
+  model_save_path: "output_model_all"
   infer_batch_size: 10
   infer_reader_path: "NAMLDataReader" # importlib format
   test_data_dir:  "../../../datasets/MIND/data/test"
-  infer_load_path: "output_model"
+  infer_load_path: "output_model_all"
   infer_start_epoch: 0
   infer_end_epoch: 2
 
@@ -42,6 +42,6 @@ hyper_parameters:
   neg_condidate_sample_size: 4
   word_dimension: 60
   category_size: 18
-  sub_category_size: 260
+  sub_category_size: 285
   category_dimension: 32
-  word_dict_size: 19200
+  word_dict_size: 27756