Merge pull request #114 from snipsco/release/0.14.2

Release 0.14.2

Author: adrienball

CHANGELOG.md

@@ -1,6 +1,15 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.14.2] - 2019-03-21
+### Added
+- Number of exact parsings
+- Possibility to provide out-of-domain utterances
+- Logging
+
+### Fixed
+- Hanging issue when using multiple workers, when one job returns a non-zero exit code
+
 ## [0.14.1] - 2019-01-07
 ### Added
 - Support for new NLU output format
@@ -35,6 +44,7 @@ All notable changes to this project will be documented in this file.
 - Samples
 
 
+[0.14.2]: https://github.com/snipsco/snips-nlu-metrics/compare/0.14.1...0.14.2
 [0.14.1]: https://github.com/snipsco/snips-nlu-metrics/compare/0.14.0...0.14.1
 [0.14.0]: https://github.com/snipsco/snips-nlu-metrics/compare/0.13.0...0.14.0
 [0.13.0]: https://github.com/snipsco/snips-nlu-metrics/compare/0.12.0...0.13.0

setup.py

@@ -22,7 +22,7 @@
     "numpy>=1.7,<2.0",
     "scipy>=1.0,<2.0",
     "scikit-learn>=0.19,<0.20",
-    "pathos~=0.2"
+    "joblib>=0.13,<0.14"
 ]
 
 extras_require = {

snips_nlu_metrics/__version__

@@ -1 +1 @@
-0.14.1
+0.14.2

snips_nlu_metrics/metrics.py

@@ -2,11 +2,12 @@
 
 import io
 import json
+import logging
 from builtins import map
 
 from future.utils import iteritems
+from joblib import Parallel, delayed
 from past.builtins import basestring
-from pathos.multiprocessing import Pool
 
 from snips_nlu_metrics.utils.constants import (
     AVERAGE_METRICS, CONFUSION_MATRIX, INTENTS, INTENT_UTTERANCES, METRICS,
@@ -17,12 +18,14 @@
     compute_engine_metrics, compute_precision_recall_f1, compute_split_metrics,
     create_shuffle_stratified_splits)
 
+logger = logging.getLogger(__name__)
+
 
 def compute_cross_val_metrics(
         dataset, engine_class, nb_folds=5, train_size_ratio=1.0,
         drop_entities=False, include_slot_metrics=True,
         slot_matching_lambda=None, progression_handler=None, num_workers=1,
-        seed=None):
+        seed=None, out_of_domain_utterances=None):
     """Compute end-to-end metrics on the dataset using cross validation
 
     Args:
@@ -49,13 +52,17 @@ class must inherit from `Engine`
         num_workers (int, optional): number of workers to use. Each worker
             is assigned a certain number of splits (default=1)
         seed (int, optional): seed for the split creation
+        out_of_domain_utterances (list, optional): If defined, list of 
+            out-of-domain utterances to be added to the pool of test utterances 
+            in each split
 
     Returns:
         dict: Metrics results containing the following data
-
+    
             - "metrics": the computed metrics
             - "parsing_errors": the list of parsing errors
-
+            - "confusion_matrix": the computed confusion matrix
+            - "average_metrics": the metrics averaged over all intents    
     """
 
     if isinstance(dataset, basestring):
@@ -64,9 +71,11 @@ class must inherit from `Engine`
 
     try:
         splits = create_shuffle_stratified_splits(
-            dataset, nb_folds, train_size_ratio, drop_entities, seed)
+            dataset, nb_folds, train_size_ratio, drop_entities,
+            seed, out_of_domain_utterances)
     except NotEnoughDataError as e:
-        print("Skipping metrics computation because of: %s" % e.message)
+        logger.warning("Skipping metrics computation because of: %s"
+                       % e.message)
         return {
             AVERAGE_METRICS: None,
             CONFUSION_MATRIX: None,
@@ -80,33 +89,38 @@ class must inherit from `Engine`
     global_errors = []
     total_splits = len(splits)
 
-    if num_workers > 1:
-        effective_num_workers = min(num_workers, len(splits))
-        pool = Pool(effective_num_workers)
-        runner = pool.imap_unordered
-    else:
-        runner = map
+    def compute_metrics(split_):
+        logger.info("Computing metrics for dataset split ...")
+        return compute_split_metrics(
+            engine_class, split_, intent_list, include_slot_metrics,
+            slot_matching_lambda)
 
-    results = runner(
-        lambda split:
-        compute_split_metrics(engine_class, split, intent_list,
-                              include_slot_metrics, slot_matching_lambda),
-        splits)
+    effective_num_workers = min(num_workers, len(splits))
+    if effective_num_workers > 1:
+        parallel = Parallel(n_jobs=effective_num_workers)
+        results = parallel(delayed(compute_metrics)(split) for split in splits)
+    else:
+        results = map(compute_metrics, splits)
 
-    for split_index, (split_metrics, errors, confusion_matrix) in \
-            enumerate(results):
+    for result in enumerate(results):
+        split_index, (split_metrics, errors, confusion_matrix) = result
         global_metrics = aggregate_metrics(
             global_metrics, split_metrics, include_slot_metrics)
         global_confusion_matrix = aggregate_matrices(
             global_confusion_matrix, confusion_matrix)
         global_errors += errors
+        logger.info("Done computing %d/%d splits"
+                    % (split_index + 1, total_splits))
 
         if progression_handler is not None:
             progression_handler(
                 float(split_index + 1) / float(total_splits))
 
     global_metrics = compute_precision_recall_f1(global_metrics)
-    average_metrics = compute_average_metrics(global_metrics)
+
+    average_metrics = compute_average_metrics(
+        global_metrics,
+        ignore_none_intent=True if out_of_domain_utterances is None else False)
 
     nb_utterances = {intent: len(data[UTTERANCES])
                      for intent, data in iteritems(dataset[INTENTS])}
@@ -147,6 +161,8 @@ class must inherit from `Engine`
 
             - "metrics": the computed metrics
             - "parsing_errors": the list of parsing errors
+            - "confusion_matrix": the computed confusion matrix
+            - "average_metrics": the metrics averaged over all intents
     """
 
     if isinstance(train_dataset, basestring):
@@ -161,13 +177,16 @@ class must inherit from `Engine`
     intent_list.update(test_dataset["intents"])
     intent_list = sorted(intent_list)
 
+    logger.info("Training engine...")
     engine = engine_class()
     engine.fit(train_dataset)
     test_utterances = [
         (intent_name, utterance)
         for intent_name, intent_data in iteritems(test_dataset[INTENTS])
         for utterance in intent_data[UTTERANCES]
     ]
+
+    logger.info("Computing metrics...")
     metrics, errors, confusion_matrix = compute_engine_metrics(
         engine, test_utterances, intent_list, include_slot_metrics,
         slot_matching_lambda)

snips_nlu_metrics/tests/mock_engine.py

@@ -23,3 +23,15 @@ def fit(self, dataset):
 
     def parse(self, text):
         return dummy_parsing_result(text)
+
+
+class MockEngineSegfault(Engine):
+    def __init__(self):
+        self.fitted = False
+
+    def fit(self, dataset):
+        self.fitted = True
+
+    def parse(self, text):
+        # Simulate a segmentation fault
+        exit(139)