Change load_dataset design for loading local datasets (#565)

* Fix dataset doc

* fix dataset doc

* Change load_dataset design for loading local dataset

* Update softmax_with_cross_entropy to cross_entropy

Author: smallv0221

docs/data_prepare/dataset_load.rst

@@ -55,9 +55,13 @@
 
 .. note::
 
-    对于某些数据集，不同的split的读取方式不同。对于这种情况则需要在 :attr:`data_files` 参数中以字典的形式传入split信息。以 **COLA** 数据集为例：
+    对于某些数据集，不同的split的读取方式不同。对于这种情况则需要在 :attr:`splits` 参数中以传入与 :attr:`data_files` **一一对应** 的split信息。
+    
+    此时 :attr:`splits` 不再代表选取的内置数据集，而代表以何种格式读取本地数据集。
+    
+    下面以 **COLA** 数据集为例：
 
     .. code-block::
 
         >>> from paddlenlp.datasets import load_dataset
-        >>> train_ds, test_ds = load_dataset("glue", "cola", data_files={"train": "my_train_file.csv", "test": "my_test_file.csv"})
+        >>> train_ds, test_ds = load_dataset("glue", "cola", splits=["train", "test"], data_files=["my_train_file.csv", "my_test_file.csv"])

examples/language_model/rnnlm/model.py

@@ -77,8 +77,8 @@ def __init__(self):
 
     def forward(self, y, label):
         label = paddle.unsqueeze(label, axis=2)
-        loss = paddle.nn.functional.softmax_with_cross_entropy(
-            logits=y, label=label, soft_label=False)
+        loss = paddle.nn.functional.cross_entropy(
+            input=y, label=label, reduction='none')
         loss = paddle.squeeze(loss, axis=[2])
         loss = paddle.mean(loss, axis=[0])
         loss = paddle.sum(loss)

examples/machine_reading_comprehension/DuReader-robust/run_du.py

@@ -101,13 +101,10 @@ def forward(self, y, label):
         start_position, end_position = label
         start_position = paddle.unsqueeze(start_position, axis=-1)
         end_position = paddle.unsqueeze(end_position, axis=-1)
-        start_loss = paddle.nn.functional.softmax_with_cross_entropy(
-            logits=start_logits, label=start_position, soft_label=False)
-        start_loss = paddle.mean(start_loss)
-        end_loss = paddle.nn.functional.softmax_with_cross_entropy(
-            logits=end_logits, label=end_position, soft_label=False)
-        end_loss = paddle.mean(end_loss)
-
+        start_loss = paddle.nn.functional.cross_entropy(
+            input=start_logits, label=start_position)
+        end_loss = paddle.nn.functional.cross_entropy(
+            input=end_logits, label=end_position)
         loss = (start_loss + end_loss) / 2
         return loss
 

examples/machine_reading_comprehension/SQuAD/run_squad.py

@@ -204,13 +204,10 @@ def forward(self, y, label):
         start_position, end_position = label
         start_position = paddle.unsqueeze(start_position, axis=-1)
         end_position = paddle.unsqueeze(end_position, axis=-1)
-        start_loss = paddle.nn.functional.softmax_with_cross_entropy(
-            logits=start_logits, label=start_position, soft_label=False)
-        start_loss = paddle.mean(start_loss)
-        end_loss = paddle.nn.functional.softmax_with_cross_entropy(
-            logits=end_logits, label=end_position, soft_label=False)
-        end_loss = paddle.mean(end_loss)
-
+        start_loss = paddle.nn.functional.cross_entropy(
+            input=start_logits, label=start_position)
+        end_loss = paddle.nn.functional.cross_entropy(
+            input=end_logits, label=end_position)
         loss = (start_loss + end_loss) / 2
         return loss
 

paddlenlp/datasets/dataset.py

@@ -472,30 +472,6 @@ def __init__(self, lazy=None, name=None, **config):
     def read_datasets(self, splits=None, data_files=None):
         datasets = []
         assert splits or data_files, "`data_files` and `splits` can not both be None."
-        assert splits is None or data_files is None , "Only one of `data_files` and `splits` can be set."
-
-        if data_files:
-            assert isinstance(data_files, str) or isinstance(
-                data_files, dict
-            ) or isinstance(data_files, tuple) or isinstance(
-                data_files, list
-            ), "`data_files` should be a string or tuple or list or a dictionary whose key is split name and value is the path of data file."
-            if isinstance(data_files, str):
-                split = 'train'
-                datasets.append(self.read(filename=data_files, split=split))
-            elif isinstance(data_files, tuple) or isinstance(data_files, list):
-                split = 'train'
-                datasets += [
-                    self.read(
-                        filename=filename, split=split)
-                    for filename in data_files
-                ]
-            else:
-                datasets += [
-                    self.read(
-                        filename=filename, split=split)
-                    for split, filename in data_files.items()
-                ]
 
         def remove_if_exit(filepath):
             if isinstance(filepath, (list, tuple)):
@@ -510,7 +486,7 @@ def remove_if_exit(filepath):
                 except OSError:
                     pass
 
-        if splits:
+        if splits and data_files is None:
             assert isinstance(splits, str) or (
                 isinstance(splits, list) and isinstance(splits[0], str)
             ) or (
@@ -551,6 +527,33 @@ def remove_if_exit(filepath):
                         time.sleep(1)
                 datasets.append(self.read(filename=filename, split=split))
 
+        if data_files:
+            assert isinstance(data_files, str) or isinstance(
+                data_files, tuple) or isinstance(
+                    data_files, list
+                ), "`data_files` should be a string or tuple or list of strings."
+
+            if isinstance(data_files, str):
+                data_files = [data_files]
+            default_split = 'train'
+            if splits:
+                if isinstance(splits, str):
+                    splits = [splits]
+                assert len(splits) == len(
+                    data_files
+                ), "Number of `splits` and number of `data_files` should be the same if you want to specify the split of loacl data file."
+                datasets += [
+                    self.read(
+                        filename=data_files[i], split=splits[i])
+                    for i in range(len(data_files))
+                ]
+            else:
+                datasets += [
+                    self.read(
+                        filename=data_files[i], split=default_split)
+                    for i in range(len(data_files))
+                ]
+
         return datasets if len(datasets) > 1 else datasets[0]
 
     def read(self, filename, split='train'):