MAINT zip was horribly slow, dump it again

Author: mfeurer

autosklearn/ensemble_builder.py

@@ -2,7 +2,6 @@
 
 import multiprocessing
 import glob
-import gzip
 import os
 import re
 import sys
@@ -100,15 +99,15 @@ def main(self):
 
             if self.shared_mode is False:
                 dir_ensemble_list = sorted(glob.glob(os.path.join(
-                    dir_ensemble, 'predictions_ensemble_%s_*.npy.gz' % self.seed)))
+                    dir_ensemble, 'predictions_ensemble_%s_*.npy' % self.seed)))
                 if exists[1]:
                     dir_valid_list = sorted(glob.glob(os.path.join(
-                        dir_valid, 'predictions_valid_%s_*.npy.gz' % self.seed)))
+                        dir_valid, 'predictions_valid_%s_*.npy' % self.seed)))
                 else:
                     dir_valid_list = []
                 if exists[2]:
                     dir_test_list = sorted(glob.glob(os.path.join(
-                        dir_test, 'predictions_test_%s_*.npy.gz' % self.seed)))
+                        dir_test, 'predictions_test_%s_*.npy' % self.seed)))
                 else:
                     dir_test_list = []
             else:
@@ -127,8 +126,8 @@ def main(self):
             for dir_ensemble_file in dir_ensemble_list:
                 if dir_ensemble_file.endswith("/"):
                     dir_ensemble_file = dir_ensemble_file[:-1]
-                if not dir_ensemble_file.endswith(".npy.gz"):
-                    self.logger.info('Error loading file (not .npy.gz): %s', dir_ensemble_file)
+                if not dir_ensemble_file.endswith(".npy"):
+                    self.logger.info('Error loading file (not .npy): %s', dir_ensemble_file)
                     continue
 
                 dir_ensemble_model_files.append(dir_ensemble_file)
@@ -161,7 +160,7 @@ def main(self):
             #  later
             include_num_runs = []
             backup_num_runs = []
-            model_and_automl_re = re.compile(r'_([0-9]*)_([0-9]*)\.npy\.gz$')
+            model_and_automl_re = re.compile(r'_([0-9]*)_([0-9]*)\.npy')
             if self.ensemble_nbest is not None:
                 # Keeps track of the single scores of each model in our ensemble
                 scores_nbest = []
@@ -179,7 +178,7 @@ def main(self):
                 basename = os.path.basename(model_name)
 
                 try:
-                    with gzip.open(os.path.join(dir_ensemble, basename)) as fh:
+                    with open(os.path.join(dir_ensemble, basename), 'rb') as fh:
                         if self.precision is "16":
                             predictions = np.load(fh).astype(dtype=np.float16)
                         elif self.precision is "32":
@@ -455,7 +454,7 @@ def get_predictions(self, dir_path, dir_path_list, include_num_runs,
             basename = os.path.basename(model_name)
 
             if (automl_seed, num_run) in include_num_runs:
-                with gzip.open(os.path.join(dir_path, basename)) as fh:
+                with open(os.path.join(dir_path, basename), 'rb') as fh:
                     if precision == "16":
                         predictions = np.load(fh).astype(
                             dtype=np.float16)

autosklearn/util/backend.py

@@ -1,5 +1,4 @@
 import glob
-import gzip
 import os
 import tempfile
 import time
@@ -170,7 +169,7 @@ def load_start_time(self, seed):
 
     def _get_targets_ensemble_filename(self):
         return os.path.join(self.internals_directory,
-                            "true_targets_ensemble.npy.gz")
+                            "true_targets_ensemble.npy")
 
     def save_targets_ensemble(self, targets):
         self._make_internals_directory()
@@ -196,7 +195,7 @@ def save_targets_ensemble(self, targets):
         lock_path = filepath + '.lock'
         with lockfile.LockFile(lock_path):
             if os.path.exists(filepath):
-                with gzip.open(filepath) as fh:
+                with open(filepath) as fh:
                     existing_targets = np.load(fh)
                     if existing_targets.shape[0] > targets.shape[0] or \
                             (existing_targets.shape == targets.shape and
@@ -205,9 +204,7 @@ def save_targets_ensemble(self, targets):
 
             with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
                     filepath), delete=False) as fh:
-                zipfile = gzip.GzipFile(fileobj=fh)
-                np.save(zipfile, targets.astype(np.float32))
-                zipfile.close()
+                np.save(fh, targets.astype(np.float32))
                 tempname = fh.name
 
             os.rename(tempname, filepath)
@@ -219,13 +216,13 @@ def load_targets_ensemble(self):
 
         lock_path = filepath + '.lock'
         with lockfile.LockFile(lock_path):
-            with gzip.open(filepath) as fh:
+            with open(filepath, 'rb') as fh:
                 targets = np.load(fh)
 
         return targets
 
     def _get_datamanager_pickle_filename(self):
-        return os.path.join(self.internals_directory, 'datamanager.pkl.gz')
+        return os.path.join(self.internals_directory, 'datamanager.pkl')
 
     def save_datamanager(self, datamanager):
         self._make_internals_directory()
@@ -236,9 +233,7 @@ def save_datamanager(self, datamanager):
             if not os.path.exists(filepath):
                 with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
                         filepath), delete=False) as fh:
-                    zipfile = gzip.GzipFile(fileobj=fh)
-                    pickle.dump(datamanager, zipfile, -1)
-                    zipfile.close()
+                    pickle.dump(datamanager, fh, -1)
                     tempname = fh.name
                 os.rename(tempname, filepath)
 
@@ -248,35 +243,21 @@ def load_datamanager(self):
         filepath = self._get_datamanager_pickle_filename()
         lock_path = filepath + '.lock'
         with lockfile.LockFile(lock_path):
-            with gzip.open(filepath, 'rb') as fh:
+            with open(filepath, 'rb') as fh:
                 return pickle.load(fh)
 
     def get_model_dir(self):
         return os.path.join(self.internals_directory, 'models')
 
     def save_model(self, model, idx, seed):
         # This should fail if no models directory exists
-        try:
-            filepath = os.path.join(self.get_model_dir(),
-                                    '%s.%s.model.gz' % (seed, idx))
+        filepath = os.path.join(self.get_model_dir(),
+                                '%s.%s.model' % (seed, idx))
 
-            with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
-                    filepath), delete=False) as fh:
-                zipfile = gzip.GzipFile(fileobj=fh)
-                pickle.dump(model, zipfile, -1)
-                zipfile.close()
+        with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
+                filepath), delete=False) as fh:
+            pickle.dump(model, fh, -1)
             tempname = fh.name
-        # Actually I would like to catch a RecursionError here, but it turns out
-        # that it was added in python3.5 and cannot be used in python3.4. But
-        # since it is a subclass of RuntimeError this works fine as well
-        except RuntimeError:
-            filepath = os.path.join(self.get_model_dir(),
-                                    '%s.%s.model' % (seed, idx))
-
-            with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
-                    filepath), delete=False) as fh:
-                pickle.dump(model, fh, -1)
-                tempname = fh.name
 
         os.rename(tempname, filepath)
 
@@ -285,9 +266,7 @@ def load_all_models(self, seed):
 
         if seed >= 0:
             model_files = glob.glob(os.path.join(model_directory,
-                                                 '%s.*.model.gz' % seed))
-            model_files.extend(glob.glob(os.path.join(model_directory,
-                                                      '%s.*.model' % seed)))
+                                                 '%s.*.model' % seed))
         else:
             model_files = os.listdir(model_directory)
             model_files = [os.path.join(model_directory, mf) for mf in model_files]
@@ -303,7 +282,7 @@ def load_models_by_file_names(self, model_file_names):
             # File names are like: {seed}.{index}.model
             if model_file.endswith('/'):
                 model_file = model_file[:-1]
-            if not model_file.endswith('.model.gz') and \
+            if not model_file.endswith('.model') and \
                     not model_file.endswith('.model'):
                 continue
 
@@ -329,14 +308,10 @@ def load_models_by_identifiers(self, identifiers):
     def load_model_by_seed_and_id(self, seed, idx):
         model_directory = self.get_model_dir()
 
-        model_file_name = '%s.%s.model.gz' % (seed, idx)
+        model_file_name = '%s.%s.model' % (seed, idx)
         model_file_path = os.path.join(model_directory, model_file_name)
-        if os.path.exists(model_file_path):
-            with gzip.open(model_file_path, 'rb') as fh:
-                return pickle.load(fh)
-        else:
-            with open(model_file_path[:-3], 'rb') as fh:
-                return pickle.load(fh)
+        with open(model_file_path, 'rb') as fh:
+            return pickle.load(fh)
 
     def get_ensemble_dir(self):
         return os.path.join(self.internals_directory, 'ensembles')
@@ -387,14 +362,12 @@ def save_predictions_as_npy(self, predictions, subset, automl_seed, idx):
         if not os.path.exists(output_dir):
             os.makedirs(output_dir)
 
-        filepath = os.path.join(output_dir, 'predictions_%s_%s_%s.npy.gz' %
+        filepath = os.path.join(output_dir, 'predictions_%s_%s_%s.npy' %
                                             (subset, automl_seed, str(idx)))
 
         with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
                 filepath), delete=False) as fh:
-            zipfile = gzip.GzipFile(fileobj=fh)
-            pickle.dump(predictions.astype(np.float32), zipfile, -1)
-            zipfile.close()
+            pickle.dump(predictions.astype(np.float32), fh, -1)
             tempname = fh.name
         os.rename(tempname, filepath)
 

test/test_automl/test_automl.py

@@ -1,5 +1,4 @@
 # -*- encoding: utf-8 -*-
-import gzip
 import os
 import pickle
 import sys
@@ -167,7 +166,7 @@ def test_automl_outputs(self):
         name = '31_bac'
         dataset = os.path.join(self.test_dir, '..', '.data', name)
         data_manager_file = os.path.join(output, '.auto-sklearn',
-                                         'datamanager.pkl.gz')
+                                         'datamanager.pkl')
 
         backend_api = backend.create(output, output)
         auto = autosklearn.automl.AutoML(
@@ -177,14 +176,14 @@ def test_automl_outputs(self):
         auto.fit_automl_dataset(dataset)
 
         # pickled data manager (without one hot encoding!)
-        with gzip.open(data_manager_file, 'rb') as fh:
+        with open(data_manager_file, 'rb') as fh:
             D = pickle.load(fh)
             self.assertTrue(np.allclose(D.data['X_train'][0, :3],
                                         [1., 12., 2.]))
 
         # Check that all directories are there
-        fixture = ['predictions_valid', 'true_targets_ensemble.npy.gz',
-                   'start_time_100', 'datamanager.pkl.gz',
+        fixture = ['predictions_valid', 'true_targets_ensemble.npy',
+                   'start_time_100', 'datamanager.pkl',
                    'predictions_ensemble',
                    'ensembles', 'predictions_test', 'models']
         self.assertEqual(sorted(os.listdir(os.path.join(output,
@@ -195,11 +194,11 @@ def test_automl_outputs(self):
         # model and one ensemble
         fixture = os.listdir(os.path.join(output, '.auto-sklearn',
                                           'predictions_ensemble'))
-        self.assertIn('predictions_ensemble_100_00001.npy.gz', fixture)
+        self.assertIn('predictions_ensemble_100_00001.npy', fixture)
 
         fixture = os.listdir(os.path.join(output, '.auto-sklearn',
                                           'models'))
-        self.assertIn('100.1.model.gz', fixture)
+        self.assertIn('100.1.model', fixture)
 
         fixture = os.listdir(os.path.join(output, '.auto-sklearn',
                                           'ensembles'))
@@ -240,7 +239,7 @@ def test_do_dummy_prediction(self):
                                                          '.auto-sklearn')))
             self.assertTrue(os.path.exists(os.path.join(
                 output, '.auto-sklearn', 'predictions_ensemble',
-                'predictions_ensemble_1_00001.npy.gz')))
+                'predictions_ensemble_1_00001.npy')))
 
             del auto
             self._tearDown(output)

test/test_automl/test_estimators.py

@@ -109,17 +109,17 @@ def test_fit_pSMAC(self):
         # more than 99%; it should be in the final ensemble if the ensemble
         # building of the second AutoSklearn classifier works correct
         true_targets_ensemble_path = os.path.join(output, '.auto-sklearn',
-                                                  'true_targets_ensemble.npy.gz')
-        with gzip.open(true_targets_ensemble_path) as fh:
+                                                  'true_targets_ensemble.npy')
+        with open(true_targets_ensemble_path, 'rb') as fh:
             true_targets_ensemble = np.load(fh)
         true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
         probas = np.zeros((len(true_targets_ensemble), 3), dtype=float)
         for i, value in enumerate(true_targets_ensemble):
             probas[i, value] = 1.0
         dummy_predictions_path = os.path.join(output, '.auto-sklearn',
                                               'predictions_ensemble',
-                                              'predictions_ensemble_1_00030.npy.gz')
-        with gzip.open(dummy_predictions_path, 'wb') as fh:
+                                              'predictions_ensemble_1_00030.npy')
+        with open(dummy_predictions_path, 'wb') as fh:
             np.save(fh, probas)
 
         probas_test = np.zeros((len(Y_test), 3), dtype=float)

test/test_util/test_backend.py

@@ -1,4 +1,5 @@
 # -*- encoding: utf-8 -*-
+import builtins
 import sys
 import unittest
 import unittest.mock
@@ -23,28 +24,12 @@ def setUp(self):
     def test_load_models_by_file_names(self):
         self.backend.load_model_by_seed_and_id = unittest.mock.Mock()
         self.backend.load_model_by_seed_and_id.side_effect = lambda *args: args
-        rval = self.backend.load_models_by_file_names(['1.2.model.gz',
+        rval = self.backend.load_models_by_file_names(['1.2.model',
                                                        '1.3.model',
                                                        '1.4.models'])
         self.assertEqual(rval, {(1, 2): (1, 2),
                                 (1, 3): (1, 3)})
 
-    @unittest.mock.patch('pickle.load')
-    @unittest.mock.patch('gzip.open')
-    @unittest.mock.patch('os.path.exists')
-    def test_load_model_by_seed_and_id_gz(self, exists_mock, openMock,
-                                          pickleLoadMock):
-        exists_mock.return_value = True
-        seed = 13
-        idx = 17
-        expected_model = self._setup_load_model_mocks(openMock,
-                                                      pickleLoadMock,
-                                                      seed, idx)
-
-        actual_model = self.backend.load_model_by_seed_and_id(seed, idx)
-
-        self.assertEqual(expected_model, actual_model)
-
     @unittest.mock.patch('pickle.load')
     @unittest.mock.patch('os.path.exists')
     def test_load_model_by_seed_and_id(self, exists_mock, pickleLoadMock):
@@ -55,14 +40,14 @@ def test_load_model_by_seed_and_id(self, exists_mock, pickleLoadMock):
             idx = 17
             expected_model = self._setup_load_model_mocks(open_mock,
                                                           pickleLoadMock,
-                                                          seed, idx, zip=False)
+                                                          seed, idx)
 
             actual_model = self.backend.load_model_by_seed_and_id(seed, idx)
 
             self.assertEqual(expected_model, actual_model)
 
     @unittest.mock.patch('pickle.load')
-    @unittest.mock.patch('gzip.open')
+    @unittest.mock.patch.object(builtins, 'open')
     @unittest.mock.patch('os.path.exists')
     def test_loads_models_by_identifiers(self, exists_mock, openMock, pickleLoadMock):
         exists_mock.return_value = True
@@ -76,11 +61,8 @@ def test_loads_models_by_identifiers(self, exists_mock, openMock, pickleLoadMock
         self.assertIsInstance(actual_dict, dict)
         self.assertDictEqual(expected_dict, actual_dict)
 
-    def _setup_load_model_mocks(self, openMock, pickleLoadMock, seed, idx,
-                                zip=True):
+    def _setup_load_model_mocks(self, openMock, pickleLoadMock, seed, idx):
         model_path = '/model_directory/%s.%s.model' % (seed, idx)
-        if zip:
-            model_path += '.gz'
         file_handler = 'file_handler'
         expected_model = 'model'
 
@@ -91,23 +73,3 @@ def _setup_load_model_mocks(self, openMock, pickleLoadMock, seed, idx,
         pickleLoadMock.side_effect = lambda fh: expected_model if fh == file_handler else None
 
         return expected_model
-
-    @unittest.mock.patch('pickle.dump')
-    def test_save_model_recursion_depth_error(self, dump_mock):
-        class SideEffect(object):
-            def __init__(self):
-                self.num_calls = 0
-
-            def side_effect(self, *args):
-                self.num_calls += 1
-                if self.num_calls == 1:
-                    if sys.version_info >= (3, 5):
-                        raise RecursionError
-                    else:
-                        raise RuntimeError
-
-        dump_mock.side_effect = SideEffect().side_effect
-        model = sklearn.tree.DecisionTreeClassifier()
-        self.backend.get_model_dir = lambda: '/tmp/'
-        self.backend.save_model(model, 1, 1)
-        self.assertEqual(dump_mock.call_count, 2)