Merge pull request #360 from yinhaofeng/fix_reader

frankwhzhang · web-flow · commit 76579dc9d48c · 2021-01-29T15:02:21.000+08:00
fix_reader
diff --git a/models/contentunderstanding/tagspace/agnews_reader.py b/models/contentunderstanding/tagspace/agnews_reader.py
@@ -52,8 +52,8 @@ def __iter__(self):
                             neg_index = rand_i
                             neg_tag.append(neg_index)
 
-                    output_list.append(np.array(text))
-                    output_list.append(np.array(pos_tag))
-                    output_list.append(np.array(neg_tag))
+                    output_list.append(np.array(text).astype('int64'))
+                    output_list.append(np.array(pos_tag).astype('int64'))
+                    output_list.append(np.array(neg_tag).astype('int64'))
 
                     yield output_list
diff --git a/models/contentunderstanding/textcnn/senti_clas_reader.py b/models/contentunderstanding/textcnn/senti_clas_reader.py
@@ -34,6 +34,8 @@ def __iter__(self):
                     label = l[101:]
                     if data is None:
                         continue
-                    data_tensor = np.array([int(i) for i in data])
-                    label_tensor = np.array([int(i) for i in label])
+                    data_tensor = np.array(
+                        [int(i) for i in data]).astype('int64')
+                    label_tensor = np.array(
+                        [int(i) for i in label]).astype('int64')
                     yield data_tensor, label_tensor
diff --git a/models/match/match-pyramid/letor_reader.py b/models/match/match-pyramid/letor_reader.py
@@ -32,6 +32,6 @@ def __iter__(self):
                     features = l.strip('\n').split('\t')
                     doc1 = [int(word_id) for word_id in features[0].split(",")]
                     doc2 = [int(word_id) for word_id in features[1].split(",")]
-                    output_list.append(np.array(doc1))
-                    output_list.append(np.array(doc2))
+                    output_list.append(np.array(doc1).astype('int64'))
+                    output_list.append(np.array(doc2).astype('int64'))
                     yield output_list
diff --git a/models/match/multiview-simnet/bq_reader_infer.py b/models/match/multiview-simnet/bq_reader_infer.py
@@ -38,7 +38,7 @@ def __iter__(self):
                             slot_0.append(int(i.strip().split(":")[1]))
                         if i.strip().split(":")[0] == "1":
                             slot_1.append(int(i.strip().split(":")[1]))
-                    output_list.append(np.array(slot_0))
-                    output_list.append(np.array(slot_1))
+                    output_list.append(np.array(slot_0).astype('int64'))
+                    output_list.append(np.array(slot_1).astype('int64'))
 
                     yield output_list
diff --git a/models/match/multiview-simnet/bq_reader_train.py b/models/match/multiview-simnet/bq_reader_train.py
@@ -41,8 +41,8 @@ def __iter__(self):
                             slot_1.append(int(i.strip().split(":")[1]))
                         if i.strip().split(":")[0] == "2":
                             slot_2.append(int(i.strip().split(":")[1]))
-                    output_list.append(np.array(slot_0))
-                    output_list.append(np.array(slot_1))
-                    output_list.append(np.array(slot_2))
+                    output_list.append(np.array(slot_0).astype('int64'))
+                    output_list.append(np.array(slot_1).astype('int64'))
+                    output_list.append(np.array(slot_2).astype('int64'))
 
                     yield output_list
diff --git a/models/multitask/esmm/esmm_reader.py b/models/multitask/esmm/esmm_reader.py
@@ -64,12 +64,14 @@ def __iter__(self):
                         self.all_field_id_dict[field_id][0] = False
                         if len(output[index][1]) > self.max_len:
                             output_list.append(
-                                np.array(output[index][1][:self.max_len]))
+                                np.array(output[index][1][:self.max_len])
+                                .astype('int64'))
                         else:
                             for ii in range(self.max_len - len(output[index][
                                     1])):
                                 output[index][1].append(self.padding)
-                            output_list.append(np.array(output[index][1]))
-                    output_list.append(np.array([ctr]))
-                    output_list.append(np.array([ctcvr]))
+                            output_list.append(
+                                np.array(output[index][1]).astype('int64'))
+                    output_list.append(np.array([ctr]).astype('int64'))
+                    output_list.append(np.array([ctcvr]).astype('int64'))
                     yield output_list
diff --git a/models/multitask/mmoe/census_reader.py b/models/multitask/mmoe/census_reader.py
@@ -45,6 +45,6 @@ def __iter__(self):
                         label_marital = [1]
                     output_list = []
                     output_list.append(np.array(data).astype('float32'))
-                    output_list.append(np.array(label_income))
-                    output_list.append(np.array(label_marital))
+                    output_list.append(np.array(label_income).astype('int64'))
+                    output_list.append(np.array(label_marital).astype('int64'))
                     yield output_list
diff --git a/models/rank/deepfm/criteo_reader.py b/models/rank/deepfm/criteo_reader.py
@@ -73,7 +73,7 @@ def __iter__(self):
                     # sparse
                     output_list = []
                     for key, value in output[:-1]:
-                        output_list.append(np.array(value))
+                        output_list.append(np.array(value).astype('int64'))
                     # dense
                     output_list.append(
                         np.array(output[-1][1]).astype("float32"))
diff --git a/models/rank/dnn/criteo_reader.py b/models/rank/dnn/criteo_reader.py
@@ -73,7 +73,7 @@ def __iter__(self):
                     # sparse
                     output_list = []
                     for key, value in output[:-1]:
-                        output_list.append(np.array(value))
+                        output_list.append(np.array(value).astype('int64'))
                     # dense
                     output_list.append(
                         np.array(output[-1][1]).astype("float32"))
diff --git a/models/rank/fm/criteo_reader.py b/models/rank/fm/criteo_reader.py
@@ -73,7 +73,7 @@ def __iter__(self):
                     # sparse
                     output_list = []
                     for key, value in output[:-1]:
-                        output_list.append(np.array(value))
+                        output_list.append(np.array(value).astype('int64'))
                     # dense
                     output_list.append(
                         np.array(output[-1][1]).astype("float32"))
diff --git a/models/rank/gateDnn/criteo_reader.py b/models/rank/gateDnn/criteo_reader.py
@@ -73,7 +73,7 @@ def __iter__(self):
                     # sparse
                     output_list = []
                     for key, value in output[:-1]:
-                        output_list.append(np.array(value))
+                        output_list.append(np.array(value).astype('int64'))
                     # dense
                     output_list.append(
                         np.array(output[-1][1]).astype("float32"))
diff --git a/models/rank/logistic_regression/criteo_lr_reader.py b/models/rank/logistic_regression/criteo_lr_reader.py
@@ -70,5 +70,6 @@ def __iter__(self):
                         else:
                             self.visit[slot] = False
                     # label, feat_idx, feat_value
-                    yield np.array(output[0][1]), np.array(output[1][
-                        1]), np.array(output[2][1]).astype('float32')
+                    yield np.array(output[0][1]).astype('int64'), np.array(
+                        output[1][1]).astype('int64'), np.array(output[2][
+                            1]).astype('float32')
diff --git a/models/rank/logistic_regression/dygraph_model.py b/models/rank/logistic_regression/dygraph_model.py
@@ -35,7 +35,7 @@ def create_model(self, config):
     # define feeds which convert numpy of batch data to paddle.tensor 
     def create_feeds(self, batch_data, config):
         num_field = config.get('hyper_parameters.num_field', None)
-        label = paddle.to_tensor(batch_data[0].numpy().astype('int32').reshape(
+        label = paddle.to_tensor(batch_data[0].numpy().astype('int64').reshape(
             -1, 1))
         feat_idx = paddle.to_tensor(batch_data[1].numpy().astype('int64')
                                     .reshape(-1, 1))
diff --git a/models/rank/wide_deep/criteo_reader.py b/models/rank/wide_deep/criteo_reader.py
@@ -72,7 +72,7 @@ def __iter__(self):
                     # sparse
                     output_list = []
                     for key, value in output[:-1]:
-                        output_list.append(np.array(value))
+                        output_list.append(np.array(value).astype('int64'))
                     # dense
                     output_list.append(
                         np.array(output[-1][1]).astype("float32"))
diff --git a/models/recall/word2vec/word2vec_infer_reader.py b/models/recall/word2vec/word2vec_infer_reader.py
@@ -90,9 +90,10 @@ def __iter__(self):
                     output_list = []
                     for i in range(4):
                         output_list.append(
-                            np.array([self.word_to_id[features[i]]]))
+                            np.array([self.word_to_id[features[i]]]).astype(
+                                'int64'))
                     inputs_words = [
                         self.word_to_id[features[i]] for i in range(3)
                     ]
-                    output_list.append(np.array(inputs_words))
+                    output_list.append(np.array(inputs_words).astype('int64'))
                     yield output_list
diff --git a/models/recall/word2vec/word2vec_reader.py b/models/recall/word2vec/word2vec_reader.py
@@ -176,9 +176,10 @@ def __iter__(self):
                     output_list = []
                     for i in range(4):
                         output_list.append(
-                            np.array([self.word_to_id[features[i]]]))
+                            np.array([self.word_to_id[features[i]]]).astype(
+                                'int64'))
                     inputs_words = [
                         self.word_to_id[features[i]] for i in range(3)
                     ]
-                    output_list.append(np.array(inputs_words))
+                    output_list.append(np.array(inputs_words).astype('int64'))
                     yield output_list
