Refactor GLUE data loaders (#138)

Refactor GLUE data loaders and misc utils.

Author: Peter Izsak

nlp_architect/data/glue_tasks.py

@@ -16,8 +16,11 @@
 import logging
 import os
 
+from sklearn.metrics import matthews_corrcoef
+
 from nlp_architect.data.sequence_classification import SequenceClsInputExample
 from nlp_architect.data.utils import DataProcessor, Task, read_tsv
+from nlp_architect.utils.metrics import acc_and_f1, pearson_and_spearman, simple_accuracy
 
 logger = logging.getLogger(__name__)
 
@@ -539,6 +542,31 @@ def _truncate_seq_pair(tokens_a, tokens_b, max_length):
 }
 
 
+# GLUE task metrics
+def get_metric_fn(task_name):
+    if task_name == "cola":
+        return lambda p, l: {"mcc": matthews_corrcoef(p, l)}
+    if task_name == "sst-2":
+        return lambda p, l: {"acc": simple_accuracy(p, l)}
+    if task_name == "mrpc":
+        return acc_and_f1
+    if task_name == "sts-b":
+        return pearson_and_spearman
+    if task_name == "qqp":
+        return acc_and_f1
+    if task_name == "mnli":
+        return lambda p, l: {"acc": simple_accuracy(p, l)}
+    if task_name == "mnli-mm":
+        return lambda p, l: {"acc": simple_accuracy(p, l)}
+    if task_name == "qnli":
+        return lambda p, l: {"acc": simple_accuracy(p, l)}
+    if task_name == "rte":
+        return lambda p, l: {"acc": simple_accuracy(p, l)}
+    if task_name == "wnli":
+        return lambda p, l: {"acc": simple_accuracy(p, l)}
+    raise KeyError(task_name)
+
+
 def get_glue_task(task_name: str, data_dir: str = None):
     """Return a GLUE task object
     Args:
@@ -551,6 +579,9 @@ def get_glue_task(task_name: str, data_dir: str = None):
         raise ValueError("Task not found: {}".format(task_name))
     task_processor = processors[task_name]()
     if data_dir is None:
-        data_dir = os.path.join(os.environ["GLUE_DIR"], DEFAULT_FOLDER_NAMES[task_name])
+        try:
+            data_dir = os.path.join(os.environ["GLUE_DIR"], DEFAULT_FOLDER_NAMES[task_name])
+        except Exception:
+            data_dir = None
     task_type = output_modes[task_name]
     return Task(task_name, task_processor, data_dir, task_type)

nlp_architect/data/utils.py

@@ -30,6 +30,7 @@ class InputExample(ABC):
     def __init__(self, guid: str, text, label=None):
         self.guid = guid
         self.text = text
+        self.text_a = text  # for compatibility with trasformer library
         self.label = label
 
 
@@ -181,3 +182,30 @@ def split_column_dataset(
     second_data = selected_lines[first_count:]
     write_column_tagged_file(out_folder + os.sep + first_filename, first_data)
     write_column_tagged_file(out_folder + os.sep + second_filename, second_data)
+
+
+def get_cached_filepath(data_dir, model_name, seq_length, task_name, set_type="train"):
+    """get cached file name
+
+    Arguments:
+        data_dir {str} -- data directory string
+        model_name {str} -- model name
+        seq_length {int} -- max sequence length
+        task_name {str} -- name of task
+
+    Keyword Arguments:
+        set_type {str} -- set type (choose from train/dev/test) (default: {"train"})
+
+    Returns:
+        str -- cached filename
+    """
+    cached_features_file = os.path.join(
+        data_dir,
+        "cached_{}_{}_{}_{}".format(
+            set_type,
+            list(filter(None, model_name.split("/"))).pop(),
+            str(seq_length),
+            str(task_name),
+        ),
+    )
+    return cached_features_file

nlp_architect/models/__init__.py

@@ -26,23 +26,29 @@ class TrainableModel(ABC):
     def convert_to_tensors(self, *args, **kwargs):
         """convert any chosen input to valid model format of tensors
         """
+        raise NotImplementedError
 
     def get_logits(self, *args, **kwargs):
         """get model logits from given input
         """
+        raise NotImplementedError
 
     def train(self, *args, **kwargs):
         """train the model
         """
+        raise NotImplementedError
 
     def inference(self, *args, **kwargs):
         """run inference
         """
+        raise NotImplementedError
 
     def save_model(self, *args, **kwargs):
         """save the model
         """
+        ...
 
     def load_model(self, *args, **kwargs):
         """load a model
         """
+        ...

nlp_architect/procedures/transformers/glue.py

@@ -18,17 +18,15 @@
 import logging
 import os
 
-from sklearn.metrics import matthews_corrcoef
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 
-from nlp_architect.data.glue_tasks import get_glue_task, processors
+from nlp_architect.data.glue_tasks import get_glue_task, get_metric_fn, processors
 from nlp_architect.models.transformers import TransformerSequenceClassifier
-from nlp_architect.nn.torch import setup_backend, set_seed
+from nlp_architect.nn.torch import set_seed, setup_backend
 from nlp_architect.procedures.procedure import Procedure
 from nlp_architect.procedures.registry import register_inference_cmd, register_train_cmd
 from nlp_architect.procedures.transformers.base import create_base_args, inference_args, train_args
 from nlp_architect.utils.io import prepare_output_path
-from nlp_architect.utils.metrics import acc_and_f1, pearson_and_spearman, simple_accuracy
 
 logger = logging.getLogger(__name__)
 
@@ -168,28 +166,3 @@ def do_inference(args):
     with io.open(os.path.join(args.output_dir, "output.txt"), "w", encoding="utf-8") as fw:
         for p in preds:
             fw.write("{}\n".format(p))
-
-
-# GLUE task metrics
-def get_metric_fn(task_name):
-    if task_name == "cola":
-        return lambda p, l: {"mcc": matthews_corrcoef(p, l)}
-    if task_name == "sst-2":
-        return lambda p, l: {"acc": simple_accuracy(p, l)}
-    if task_name == "mrpc":
-        return acc_and_f1
-    if task_name == "sts-b":
-        return pearson_and_spearman
-    if task_name == "qqp":
-        return acc_and_f1
-    if task_name == "mnli":
-        return lambda p, l: {"acc": simple_accuracy(p, l)}
-    if task_name == "mnli-mm":
-        return lambda p, l: {"acc": simple_accuracy(p, l)}
-    if task_name == "qnli":
-        return lambda p, l: {"acc": simple_accuracy(p, l)}
-    if task_name == "rte":
-        return lambda p, l: {"acc": simple_accuracy(p, l)}
-    if task_name == "wnli":
-        return lambda p, l: {"acc": simple_accuracy(p, l)}
-    raise KeyError(task_name)