Fix community provided datasets and fix cross_entropy for elmo and dgu (#576)

* Fix dataset doc

* fix dataset doc

* Change load_dataset design for loading local dataset

* Update softmax_with_cross_entropy to cross_entropy

* Fix community provided datasets and fix cross_entropy for elmo and dgu

Author: smallv0221

examples/dialogue/dgu/main.py

@@ -65,20 +65,18 @@ def get_loss_fn(self):
         if self.task_name in [
                 'udc', 'atis_slot', 'atis_intent', 'mrda', 'swda'
         ]:
-            return F.softmax_with_cross_entropy
+            return F.cross_entropy
         elif self.task_name == 'dstc2':
             return nn.BCEWithLogitsLoss(reduction='sum')
 
     def forward(self, logits, labels):
         if self.task_name in ['udc', 'atis_intent', 'mrda', 'swda']:
             loss = self.loss_fn(logits, labels)
-            loss = paddle.mean(loss)
         elif self.task_name == 'dstc2':
             loss = self.loss_fn(logits, paddle.cast(labels, dtype=logits.dtype))
         elif self.task_name == 'atis_slot':
             labels = paddle.unsqueeze(labels, axis=-1)
             loss = self.loss_fn(logits, labels)
-            loss = paddle.mean(loss)
         return loss
 
 

examples/language_model/elmo/elmo.py

@@ -302,10 +302,10 @@ def forward(self, x, y):
         bw_label = paddle.unsqueeze(bw_label, axis=2)
 
         # [batch_size, seq_len, 1]
-        fw_loss = F.softmax_with_cross_entropy(logits=fw_logits, label=fw_label)
-        bw_loss = F.softmax_with_cross_entropy(logits=bw_logits, label=bw_label)
+        fw_loss = F.cross_entropy(input=fw_logits, label=fw_label)
+        bw_loss = F.cross_entropy(input=bw_logits, label=bw_label)
 
-        avg_loss = 0.5 * (paddle.mean(fw_loss) + paddle.mean(bw_loss))
+        avg_loss = 0.5 * (fw_loss + bw_loss)
         return avg_loss
 
 

paddlenlp/datasets/bq_corpus.py

@@ -12,9 +12,11 @@
 
 class BQCorpus(DatasetBuilder):
     """
-    BQCorpus: the largest dataset available for for the banking and finance sector
+    BQCorpus: A Large-scale Domain-specific Chinese Corpus For Sentence 
+    Semantic Equivalence Identification. More information please refer 
+    to `https://www.aclweb.org/anthology/D18-1536.pdf`
 
-    by frozenfish123@Wuhan University
+    Contributed by frozenfish123@Wuhan University
 
     """
     lazy = False
@@ -23,13 +25,13 @@ class BQCorpus(DatasetBuilder):
     META_INFO = collections.namedtuple('META_INFO', ('file', 'md5'))
     SPLITS = {
         'train': META_INFO(
-            os.path.join('BQCorpus', 'train.tsv'),
+            os.path.join('bq_corpus', 'bq_corpus', 'train.tsv'),
             'd37683e9ee778ee2f4326033b654adb9'),
         'dev': META_INFO(
-            os.path.join('BQCorpus', 'dev.tsv'),
+            os.path.join('bq_corpus', 'bq_corpus', 'dev.tsv'),
             '8a71f2a69453646921e9ee1aa457d1e4'),
         'test': META_INFO(
-            os.path.join('BQCorpus', 'test.tsv'),
+            os.path.join('bq_corpus', 'bq_corpus', 'test.tsv'),
             'c797995baa248b144ceaa4018b191e52'),
     }
 
@@ -47,18 +49,18 @@ def _get_data(self, mode, **kwargs):
     def _read(self, filename):
         """Reads data."""
         with open(filename, 'r', encoding='utf-8') as f:
-            head = None
             for line in f:
                 data = line.strip().split("\t")
-                if not head:
-                    head = data
-                else:
+                if len(data) == 3:
                     sentence1, sentence2, label = data
-                    yield {
-                        "sentence1": sentence1,
-                        "sentence2": sentence2,
-                        "label": label
-                    }
+                elif len(data) == 2:
+                    sentence1, sentence2 = data
+                    label = ''
+                yield {
+                    "sentence1": sentence1,
+                    "sentence2": sentence2,
+                    "label": label
+                }
 
     def get_labels(self):
         """

paddlenlp/datasets/paws-x.py

@@ -21,9 +21,10 @@
 from paddlenlp.utils.env import DATA_HOME
 from . import DatasetBuilder
 
-__all__ = ['PAWS']
+__all__ = ['PAWSX']
 
-class PAWS(DatasetBuilder):
+
+class PAWSX(DatasetBuilder):
     """
     PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification
     More information please refer to `https://arxiv.org/abs/1908.11828`
@@ -60,11 +61,19 @@ def _read(self, filename):
             for line in f:
                 data = line.strip().split("\t")
                 if len(data) == 3:
-                    sentence1, sentence2, label = data                    
-                    yield {"sentence1": sentence1, "sentence2": sentence2, "label": label}
+                    sentence1, sentence2, label = data
+                    yield {
+                        "sentence1": sentence1,
+                        "sentence2": sentence2,
+                        "label": label
+                    }
                 elif len(data) == 2:
-                    sentence1, sentence2 = data                    
-                    yield {"sentence1": sentence1, "sentence2": sentence2, "label":''}
+                    sentence1, sentence2 = data
+                    yield {
+                        "sentence1": sentence1,
+                        "sentence2": sentence2,
+                        "label": ''
+                    }
                 else:
                     continue