Alternative to 1115 (#1124)

* Alternative to 1115

* fix meta-data generation

Author: mfeurer

autosklearn/smbo.py

@@ -69,61 +69,73 @@
 }
 
 
+def get_send_warnings_to_logger(logger):
+    def _send_warnings_to_log(message, category, filename, lineno, file, line):
+        logger.debug('%s:%s: %s:%s', filename, lineno, category.__name__, message)
+    return _send_warnings_to_log
+
+
 # metalearning helpers
 def _calculate_metafeatures(data_feat_type, data_info_task, basename,
-                            x_train, y_train, watcher, logger):
-    # == Calculate metafeatures
-    task_name = 'CalculateMetafeatures'
-    watcher.start_task(task_name)
-    categorical = [True if feat_type.lower() in ['categorical'] else False
-                   for feat_type in data_feat_type]
+                            x_train, y_train, watcher, logger_):
+    with warnings.catch_warnings():
+        warnings.showwarning = get_send_warnings_to_logger(logger_)
+
+        # == Calculate metafeatures
+        task_name = 'CalculateMetafeatures'
+        watcher.start_task(task_name)
+        categorical = [True if feat_type.lower() in ['categorical'] else False
+                       for feat_type in data_feat_type]
+
+        EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
+            if data_info_task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION
+
+        if data_info_task in [MULTICLASS_CLASSIFICATION, BINARY_CLASSIFICATION,
+                              MULTILABEL_CLASSIFICATION, REGRESSION,
+                              MULTIOUTPUT_REGRESSION]:
+            logger_.info('Start calculating metafeatures for %s', basename)
+            result = calculate_all_metafeatures_with_labels(
+                x_train, y_train, categorical=categorical,
+                dataset_name=basename,
+                dont_calculate=EXCLUDE_META_FEATURES, logger=logger_)
+            for key in list(result.metafeature_values.keys()):
+                if result.metafeature_values[key].type_ != 'METAFEATURE':
+                    del result.metafeature_values[key]
 
-    EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
-        if data_info_task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION
+        else:
+            result = None
+            logger_.info('Metafeatures not calculated')
+        watcher.stop_task(task_name)
+        logger_.info(
+            'Calculating Metafeatures (categorical attributes) took %5.2f',
+            watcher.wall_elapsed(task_name))
+        return result
 
-    if data_info_task in [MULTICLASS_CLASSIFICATION, BINARY_CLASSIFICATION,
-                          MULTILABEL_CLASSIFICATION, REGRESSION,
-                          MULTIOUTPUT_REGRESSION]:
-        logger.info('Start calculating metafeatures for %s', basename)
-        result = calculate_all_metafeatures_with_labels(
-            x_train, y_train, categorical=categorical,
-            dataset_name=basename,
-            dont_calculate=EXCLUDE_META_FEATURES, logger=logger)
-        for key in list(result.metafeature_values.keys()):
-            if result.metafeature_values[key].type_ != 'METAFEATURE':
-                del result.metafeature_values[key]
 
-    else:
-        result = None
-        logger.info('Metafeatures not calculated')
-    watcher.stop_task(task_name)
-    logger.info(
-        'Calculating Metafeatures (categorical attributes) took %5.2f',
-        watcher.wall_elapsed(task_name))
-    return result
+def _calculate_metafeatures_encoded(data_feat_type, basename, x_train, y_train, watcher,
+                                    task, logger_):
+    with warnings.catch_warnings():
+        warnings.showwarning = get_send_warnings_to_logger(logger_)
 
+        EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
+            if task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION
 
-def _calculate_metafeatures_encoded(data_feat_type, basename, x_train, y_train, watcher,
-                                    task, logger):
-    EXCLUDE_META_FEATURES = EXCLUDE_META_FEATURES_CLASSIFICATION \
-        if task in CLASSIFICATION_TASKS else EXCLUDE_META_FEATURES_REGRESSION
+        task_name = 'CalculateMetafeaturesEncoded'
+        watcher.start_task(task_name)
+        categorical = [True if feat_type.lower() in ['categorical'] else False
+                       for feat_type in data_feat_type]
 
-    task_name = 'CalculateMetafeaturesEncoded'
-    watcher.start_task(task_name)
-    categorical = [True if feat_type.lower() in ['categorical'] else False
-                   for feat_type in data_feat_type]
-
-    result = calculate_all_metafeatures_encoded_labels(
-        x_train, y_train, categorical=categorical,
-        dataset_name=basename, dont_calculate=EXCLUDE_META_FEATURES, logger=logger)
-    for key in list(result.metafeature_values.keys()):
-        if result.metafeature_values[key].type_ != 'METAFEATURE':
-            del result.metafeature_values[key]
-    watcher.stop_task(task_name)
-    logger.info(
-        'Calculating Metafeatures (encoded attributes) took %5.2fsec',
-        watcher.wall_elapsed(task_name))
-    return result
+        result = calculate_all_metafeatures_encoded_labels(
+            x_train, y_train, categorical=categorical,
+            dataset_name=basename, dont_calculate=EXCLUDE_META_FEATURES, logger=logger_)
+        for key in list(result.metafeature_values.keys()):
+            if result.metafeature_values[key].type_ != 'METAFEATURE':
+                del result.metafeature_values[key]
+        watcher.stop_task(task_name)
+        logger_.info(
+            'Calculating Metafeatures (encoded attributes) took %5.2fsec',
+            watcher.wall_elapsed(task_name))
+        return result
 
 
 def _get_metalearning_configurations(meta_base, basename, metric,
@@ -286,11 +298,6 @@ def __init__(self, config_space, dataset_name,
                 port=self.port,
             )
 
-    def _send_warnings_to_log(self, message, category, filename, lineno,
-                              file=None, line=None):
-        self.logger.debug('%s:%s: %s:%s', filename, lineno, category.__name__,
-                          message)
-
     def reset_data_manager(self, max_mem=None):
         if max_mem is None:
             max_mem = self.data_memory_limit
@@ -323,20 +330,6 @@ def collect_metalearning_suggestions(self, meta_base):
 
         return metalearning_configurations
 
-    def _calculate_metafeatures(self):
-        with warnings.catch_warnings():
-            warnings.showwarning = self._send_warnings_to_log
-
-            meta_features = _calculate_metafeatures(
-                data_feat_type=self.datamanager.feat_type,
-                data_info_task=self.datamanager.info['task'],
-                x_train=self.datamanager.data['X_train'],
-                y_train=self.datamanager.data['Y_train'],
-                basename=self.dataset_name,
-                watcher=self.watcher,
-                logger=self.logger)
-            return meta_features
-
     def _calculate_metafeatures_with_limits(self, time_limit):
         res = None
         time_limit = max(time_limit, 1)
@@ -348,37 +341,42 @@ def _calculate_metafeatures_with_limits(self, time_limit):
                                               grace_period_in_s=30,
                                               context=context,
                                               logger=self.logger)(
-                self._calculate_metafeatures)
-            res = safe_mf()
+                _calculate_metafeatures)
+            res = safe_mf(
+                data_feat_type=self.datamanager.feat_type,
+                data_info_task=self.datamanager.info['task'],
+                x_train=self.datamanager.data['X_train'],
+                y_train=self.datamanager.data['Y_train'],
+                basename=self.dataset_name,
+                watcher=self.watcher,
+                logger_=self.logger
+            )
         except Exception as e:
             self.logger.error('Error getting metafeatures: %s', str(e))
 
         return res
 
-    def _calculate_metafeatures_encoded(self):
-        with warnings.catch_warnings():
-            warnings.showwarning = self._send_warnings_to_log
-
-            meta_features_encoded = _calculate_metafeatures_encoded(
-                self.datamanager.feat_type,
-                self.dataset_name,
-                self.datamanager.data['X_train'],
-                self.datamanager.data['Y_train'],
-                self.watcher,
-                self.datamanager.info['task'],
-                self.logger)
-            return meta_features_encoded
-
     def _calculate_metafeatures_encoded_with_limits(self, time_limit):
         res = None
         time_limit = max(time_limit, 1)
         try:
+            context = multiprocessing.get_context(self.pynisher_context)
+            preload_modules(context)
             safe_mf = pynisher.enforce_limits(mem_in_mb=self.memory_limit,
                                               wall_time_in_s=int(time_limit),
                                               grace_period_in_s=30,
+                                              context=context,
                                               logger=self.logger)(
-                self._calculate_metafeatures_encoded)
-            res = safe_mf()
+                _calculate_metafeatures_encoded)
+            res = safe_mf(
+                data_feat_type=self.datamanager.feat_type,
+                task=self.datamanager.info['task'],
+                x_train=self.datamanager.data['X_train'],
+                y_train=self.datamanager.data['Y_train'],
+                basename=self.dataset_name,
+                watcher=self.watcher,
+                logger_=self.logger
+            )
         except Exception as e:
             self.logger.error('Error getting metafeatures (encoded) : %s',
                               str(e))
@@ -612,7 +610,7 @@ def get_metalearning_suggestions(self):
                     meta_features_encoded = None
                 else:
                     with warnings.catch_warnings():
-                        warnings.showwarning = self._send_warnings_to_log
+                        warnings.showwarning = get_send_warnings_to_logger(self.logger)
                     meta_features_encoded = \
                         self._calculate_metafeatures_encoded_with_limits(
                             metafeature_calculation_time_limit)
@@ -636,7 +634,7 @@ def get_metalearning_suggestions(self):
                                             inplace=True)
 
                     with warnings.catch_warnings():
-                        warnings.showwarning = self._send_warnings_to_log
+                        warnings.showwarning = get_send_warnings_to_logger(self.logger)
                         metalearning_configurations = self.collect_metalearning_suggestions(
                             meta_base)
                     if metalearning_configurations is None:

scripts/03_calculate_metafeatures.py

@@ -38,13 +38,13 @@ def calculate_metafeatures(task_id):
 
     _metafeatures_labels = _calculate_metafeatures(
         x_train=X_train, y_train=y_train, data_feat_type=cat,
-        data_info_task=task_type, basename=dataset_name, logger=logger,
+        data_info_task=task_type, basename=dataset_name, logger_=logger,
         watcher=watch,
     )
 
     _metafeatures_encoded_labels = _calculate_metafeatures_encoded(
         x_train=X_train, y_train=y_train, data_feat_type=cat,
-        task=task_type, basename=dataset_name, logger=logger,
+        task=task_type, basename=dataset_name, logger_=logger,
         watcher=watch,
     )