[BUGFIX] Cnn_dailymail and xnli raise error when downloading in multi-gpus mode (#1587)

* fix: multi-gpus count file_num

* fix: update xnli

Co-authored-by: Jiaqi Liu <[email protected]>

Author: gongenlei

docs/data_prepare/dataset_list.md

@@ -50,7 +50,7 @@ PaddleNLP提供了以下数据集的快速读取API，实际使用时请根据
 |  [CLUEWSCF](https://github.com/CLUEbenchmark/FewCLUE/tree/main/datasets)  | FewCLUE 评测中的 WSC Winograd 模式挑战中文版，代词消歧任务，二分类任务 | `paddlenlp.datasets.load_dataset('fewclue', 'cluewsc')`|
 | [THUCNews](https://github.com/gaussic/text-classification-cnn-rnn#%E6%95%B0%E6%8D%AE%E9%9B%86) |  THUCNews中文新闻类别分类 | `paddlenlp.datasets.load_dataset('thucnews')` |
 | [HYP](https://pan.webis.de/semeval19/semeval19-web/) | 英文政治新闻情感分类语料  | `paddlenlp.datasets.load_dataset('hyp')` |
-|  [XNLI](https://github.com/facebookresearch/XNLI) | 15种语言自然语言推理数据集，三分类任务. | `paddlenlp.datasets.load_dataset('xnli')`|
+|  [XNLI](https://github.com/facebookresearch/XNLI) | 15种语言自然语言推理数据集，三分类任务. | `paddlenlp.datasets.load_dataset('xnli', 'ar')`|
 |  [XNLI_CN](https://github.com/facebookresearch/XNLI) | 中文自然语言推理数据集（XNLI的子集），三分类任务. | `paddlenlp.datasets.load_dataset('xnli_cn')`|
 
 ## 文本匹配

examples/language_model/ernie-m/run_classifier.py

@@ -189,7 +189,7 @@ def convert_example(example, tokenizer, max_seq_length=256):
 
 
 def get_test_dataloader(args, language, batchify_fn, trans_func):
-    test_ds = load_dataset("xnli", splits="test", language=language)
+    test_ds = load_dataset("xnli", language, splits="test")
     test_ds = test_ds.map(trans_func, lazy=True)
     test_batch_sampler = BatchSampler(
         test_ds, batch_size=args.batch_size, shuffle=False)
@@ -240,12 +240,12 @@ def do_train(args):
         tokenizer=tokenizer,
         max_seq_length=args.max_seq_length)
     if args.task_type == "cross-lingual-transfer":
-        train_ds = load_dataset("xnli", splits="train", language="en")
+        train_ds = load_dataset("xnli", "en", splits="train")
         train_ds = train_ds.map(trans_func, lazy=True)
     elif args.task_type == "translate-train-all":
         all_train_ds = []
         for language in all_languages:
-            train_ds = load_dataset("xnli", splits="train", language=language)
+            train_ds = load_dataset("xnli", language, splits="train")
             all_train_ds.append(train_ds.map(trans_func, lazy=True))
         train_ds = XnliDataset(all_train_ds)
     train_batch_sampler = DistributedBatchSampler(

paddlenlp/datasets/cnn_dailymail.py

@@ -16,9 +16,11 @@
 import collections
 import os
 import hashlib
+import shutil
 
 from paddle.dataset.common import md5file
-from paddlenlp.utils.downloader import get_path_from_url, _decompress
+from paddle.utils.download import get_path_from_url, _decompress, _get_unique_endpoints
+from paddle.distributed import ParallelEnv
 from paddlenlp.utils.env import DATA_HOME
 from paddlenlp.utils.log import logger
 from . import DatasetBuilder
@@ -190,13 +192,17 @@ def _get_data(self, mode):
             dir_path = os.path.join(default_root, k)
             if not os.path.exists(dir_path):
                 get_path_from_url(v["url"], default_root, v["md5"])
-            file_num = len(os.listdir(os.path.join(dir_path, "stories")))
-            if file_num != v["file_num"]:
-                logger.warning(
-                    "Number of %s stories is %d != %d, decompress again." %
-                    (k, file_num, v["file_num"]))
-                _decompress(
-                    os.path.join(default_root, os.path.basename(v["url"])))
+            unique_endpoints = _get_unique_endpoints(ParallelEnv()
+                                                     .trainer_endpoints[:])
+            if ParallelEnv().current_endpoint in unique_endpoints:
+                file_num = len(os.listdir(os.path.join(dir_path, "stories")))
+                if file_num != v["file_num"]:
+                    logger.warning(
+                        "Number of %s stories is %d != %d, decompress again." %
+                        (k, file_num, v["file_num"]))
+                    shutil.rmtree(os.path.join(dir_path, "stories"))
+                    _decompress(
+                        os.path.join(default_root, os.path.basename(v["url"])))
             dl_paths[k] = dir_path
         filename, url, data_hash = self.SPLITS[mode]
         fullname = os.path.join(default_root, filename)

paddlenlp/datasets/xnli.py

@@ -16,9 +16,11 @@
 import os
 import csv
 from contextlib import ExitStack
+import shutil
 
 from paddle.dataset.common import md5file
-from paddle.utils.download import get_path_from_url, _decompress
+from paddle.utils.download import get_path_from_url, _decompress, _get_unique_endpoints
+from paddle.distributed import ParallelEnv
 from paddlenlp.utils.env import DATA_HOME
 from paddlenlp.utils.log import logger
 from . import DatasetBuilder
@@ -64,12 +66,17 @@ def _get_data(self, mode, **kwargs):
         if mode == 'train':
             if not os.path.exists(fullname):
                 get_path_from_url(url, default_root, zipfile_hash)
-            file_num = len(os.listdir(fullname))
-            if file_num != 15:
-                logger.warning(
-                    "Number of train files is %d != %d, decompress again." %
-                    (file_num, 15))
-                _decompress(os.path.join(default_root, os.path.basename(url)))
+            unique_endpoints = _get_unique_endpoints(ParallelEnv()
+                                                     .trainer_endpoints[:])
+            if ParallelEnv().current_endpoint in unique_endpoints:
+                file_num = len(os.listdir(fullname))
+                if file_num != len(ALL_LANGUAGES):
+                    logger.warning(
+                        "Number of train files is %d != %d, decompress again." %
+                        (file_num, len(ALL_LANGUAGES)))
+                    shutil.rmtree(fullname)
+                    _decompress(
+                        os.path.join(default_root, os.path.basename(url)))
         else:
             if not os.path.exists(fullname) or (
                     data_hash and not md5file(fullname) == data_hash):
@@ -79,7 +86,7 @@ def _get_data(self, mode, **kwargs):
 
     def _read(self, filename, split):
         """Reads data."""
-        language = self.config.get("language", "all_languages")
+        language = self.name
         if language == "all_languages":
             languages = ALL_LANGUAGES
         else: