Fix datas to data (#2410)

* fix datas to data

* update code

* change all_data to texts in triton deploy

Author: yeliang2258

applications/neural_search/recall/domain_adaptive_pretraining/data_tools/dataset_utils.py

@@ -52,10 +52,10 @@ def __init__(self, path, skip_warmup=False):
 
         self._token_ids = np.load(
             path + "_ids.npy", mmap_mode="r", allow_pickle=True)
-        process_datas = np.load(path + "_idx.npz")
-        self._sizes = process_datas["lens"]
-        self._pointers = process_datas["sents"]
-        self._doc_idx = process_datas["docs"]
+        process_data = np.load(path + "_idx.npz")
+        self._sizes = process_data["lens"]
+        self._pointers = process_data["sents"]
+        self._doc_idx = process_data["docs"]
 
     def __getstate__(self):
         return self._path

examples/information_extraction/DuEE/duee_1_postprocess.py

@@ -24,16 +24,16 @@
 def predict_data_process(trigger_file, role_file, schema_file, save_path):
     """predict_data_process"""
     pred_ret = []
-    trigger_datas = read_by_lines(trigger_file)
+    trigger_data = read_by_lines(trigger_file)
     role_data = read_by_lines(role_file)
-    schema_datas = read_by_lines(schema_file)
+    schema_data = read_by_lines(schema_file)
     print("trigger predict {} load from {}".format(
-        len(trigger_datas), trigger_file))
+        len(trigger_data), trigger_file))
     print("role predict {} load from {}".format(len(role_data), role_file))
-    print("schema {} load from {}".format(len(schema_datas), schema_file))
+    print("schema {} load from {}".format(len(schema_data), schema_file))
 
     schema = {}
-    for s in schema_datas:
+    for s in schema_data:
         d_json = json.loads(s)
         schema[d_json["event_type"]] = [r["role"] for r in d_json["role_list"]]
 
@@ -50,7 +50,7 @@ def predict_data_process(trigger_file, role_file, schema_file, save_path):
             role_ret[role_type].append("".join(r["text"]))
         sent_role_mapping[d_json["id"]] = role_ret
 
-    for d in trigger_datas:
+    for d in trigger_data:
         d_json = json.loads(d)
         t_ret = extract_result(d_json["text"], d_json["pred"]["labels"])
         pred_event_types = list(set([t["type"] for t in t_ret]))

examples/language_model/chinesebert/utils.py

@@ -210,7 +210,7 @@ def load_ds(datafiles):
         MapDataset
     '''
 
-    datas = []
+    data = []
 
     def read(ds_file):
         with open(ds_file, 'r', encoding='utf-8') as fp:
@@ -229,7 +229,7 @@ def read(ds_file):
 
 
 def load_ds_xnli(datafiles):
-    datas = []
+    data = []
 
     def read(ds_file):
         with open(ds_file, 'r', encoding='utf-8') as fp:

examples/language_model/elmo/elmo.py

@@ -338,10 +338,10 @@ def encode(self, sentences: List[List[str]]):
         Each sentence is a list of tokens without <s> or </s>, e.g.
         [['The', 'first', 'sentence', '.'], ['Second', '.']]
         """
-        batch_datas = create_batches(sentences, self._batch_size, self._vocab,
-                                     self._max_seq_len)
+        batch_data = create_batches(sentences, self._batch_size, self._vocab,
+                                    self._max_seq_len)
         embeddings = []
-        for data in batch_datas:
+        for data in batch_data:
             ids, ids_reverse, seq_lens = data
             # [batch_size, num_lstm_layers + 1, max_seq_len, projection_dim * 2]
             outputs = self._elmo([ids, ids_reverse])

examples/language_model/gpt-3/dygraph/dataset.py

@@ -301,10 +301,10 @@ def create_pretrained_dataset(args,
         logger.warning(
             "You are using compatible dataset, please make new dataset as the readme!"
         )
-        process_datas = np.load(
+        process_data = np.load(
             input_prefix + "_ids.npz", mmap_mode="r+", allow_pickle=True)
-        sample_ids = process_datas["ids"]
-        sample_lens = process_datas["lens"].astype("int32")
+        sample_ids = process_data["ids"]
+        sample_lens = process_data["lens"].astype("int32")
     else:
         for suffix in ["_ids.npy", "_idx.npz"]:
             if not os.path.isfile(input_prefix + suffix):
@@ -314,10 +314,10 @@ def create_pretrained_dataset(args,
             input_prefix + "_ids.npy", mmap_mode="r", allow_pickle=True)
         # All documment ids, extend as 1-D array.
 
-        process_datas = np.load(input_prefix + "_idx.npz")
+        process_data = np.load(input_prefix + "_idx.npz")
         # The len(sample_lens) num of docs
         # The sum(sample_lens) should equal len(sample_ids)
-        sample_lens = process_datas["lens"]
+        sample_lens = process_data["lens"]
 
     splits = get_train_valid_test_split_(args.split, len(sample_lens))
     assert len(sample_lens) >= splits[

examples/language_model/gpt-3/static/dataset.py

@@ -302,10 +302,10 @@ def create_pretrained_dataset(args,
         logger.warning(
             "You are using compatible dataset, please make new dataset as the readme!"
         )
-        process_datas = np.load(
+        process_data = np.load(
             input_prefix + "_ids.npz", mmap_mode="r+", allow_pickle=True)
-        sample_ids = process_datas["ids"]
-        sample_lens = process_datas["lens"].astype("int32")
+        sample_ids = process_data["ids"]
+        sample_lens = process_data["lens"].astype("int32")
     else:
         for suffix in ["_ids.npy", "_idx.npz"]:
             if not os.path.isfile(input_prefix + suffix):
@@ -315,10 +315,10 @@ def create_pretrained_dataset(args,
             input_prefix + "_ids.npy", mmap_mode="r", allow_pickle=True)
         # All documment ids, extend as 1-D array.
 
-        process_datas = np.load(input_prefix + "_idx.npz")
+        process_data = np.load(input_prefix + "_idx.npz")
         # The len(sample_lens) num of docs
         # The sum(sample_lens) should equal len(sample_ids)
-        sample_lens = process_datas["lens"]
+        sample_lens = process_data["lens"]
 
     splits = get_train_valid_test_split_(args.split, len(sample_lens))
     assert len(sample_lens) >= splits[

examples/language_model/moe/dygraph/dataset.py

@@ -248,12 +248,12 @@ def create_pretrained_dataset(args,
         "The distributed run, total device num:{}, distinct dataflow num:{}.".
         format(device_world_size, data_world_size))
 
-    process_datas = np.load(input_path, mmap_mode="r+", allow_pickle=True)
+    process_data = np.load(input_path, mmap_mode="r+", allow_pickle=True)
     # All documment ids, extend as 1-D array.
-    sample_ids = process_datas["ids"]
+    sample_ids = process_data["ids"]
     # The len(sample_lens) num of docs
     # The sum(sample_lens) should equal len(sample_ids)
-    sample_lens = process_datas["lens"]
+    sample_lens = process_data["lens"]
 
     splits = get_train_valid_test_split_(args.split, len(sample_lens))
     assert len(sample_lens) >= splits[

examples/text_graph/erniesage/data/graph_reader.py

@@ -54,7 +54,7 @@ def construct(tensors):
             """
             graph_num = 1
             start_len = 0
-            datas = []
+            data = []
             graph_list = []
             for graph in range(graph_num):
                 graph_list.append(
@@ -64,7 +64,7 @@ def construct(tensors):
                 start_len += 2
 
             for i in range(start_len, len(tensors)):
-                datas.append(tensors[i])
-            return graph_list, datas
+                data.append(tensors[i])
+            return graph_list, data
 
         return construct

examples/text_graph/erniesage/models/model.py

@@ -46,17 +46,17 @@ def __init__(self, ernie, config):
         self.encoder = Encoder.factory(self.config_file, self.ernie)
         self.loss_func = LossFactory(self.config_file)
 
-    def forward(self, graphs, datas):
+    def forward(self, graphs, data):
         """Forward function of link prediction task.
 
         Args:
             graphs (Graph List): the Graph list.
-            datas (Tensor List): other input of the model.
+            data (Tensor List): other input of the model.
 
         Returns:
             Tensor: loss and output tensors.
         """
-        term_ids, user_index, pos_item_index, neg_item_index, user_real_index, pos_item_real_index = datas
+        term_ids, user_index, pos_item_index, neg_item_index, user_real_index, pos_item_real_index = data
         # encoder model
         outputs = self.encoder(graphs, term_ids,
                                [user_index, pos_item_index, neg_item_index])

model_zoo/electra/deploy/python/predict.py

@@ -157,7 +157,7 @@ def predict(args, sentences=[], paths=[]):
     predictor = create_paddle_predictor(config)
 
     start_time = time.time()
-    output_datas = []
+    output_data = []
     count = 0
     for i, sen in enumerate(predicted_input):
         sen = np.array(sen).astype("int64")
@@ -176,9 +176,9 @@ def predict(args, sentences=[], paths=[]):
         output_names = predictor.get_output_names()
         # get output pointer and copy data(nd.array)
         output_tensor = predictor.get_output_tensor(output_names[0])
-        output_data = output_tensor.copy_to_cpu()
-        output_res = np.argmax(output_data, axis=1).tolist()
-        output_datas.append(output_res)
+        predict_data = output_tensor.copy_to_cpu()
+        output_res = np.argmax(predict_data, axis=1).tolist()
+        output_data.append(output_res)
 
         print("===== batch {} =====".format(i))
         for j in range(len(predicted_sens[i])):