fix issue with np 1.16.3 which forbids pickle loading

Author: mfeurer

autosklearn/ensemble_builder.py

@@ -636,11 +636,11 @@ def predict(self, set_: str,
 
     def _read_np_fn(self, fp):
         if self.precision is "16":
-            predictions = np.load(fp).astype(dtype=np.float16)
+            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float16)
         elif self.precision is "32":
-            predictions = np.load(fp).astype(dtype=np.float32)
+            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float32)
         elif self.precision is "64":
-            predictions = np.load(fp).astype(dtype=np.float64)
+            predictions = np.load(fp, allow_pickle=True).astype(dtype=np.float64)
         else:
-            predictions = np.load(fp)
+            predictions = np.load(fp, allow_pickle=True)
         return predictions

autosklearn/util/backend.py

@@ -265,7 +265,7 @@ def save_targets_ensemble(self, targets):
         # number of times where we erronously keep a lock on the ensemble
         # targets file although the process already was killed
         try:
-            existing_targets = np.load(filepath)
+            existing_targets = np.load(filepath, allow_pickle=True)
             if existing_targets.shape[0] > targets.shape[0] or \
                     (existing_targets.shape == targets.shape and
                          np.allclose(existing_targets, targets)):
@@ -278,7 +278,7 @@ def save_targets_ensemble(self, targets):
         with lockfile.LockFile(lock_path):
             if os.path.exists(filepath):
                 with open(filepath, 'rb') as fh:
-                    existing_targets = np.load(fh)
+                    existing_targets = np.load(fh, allow_pickle=True)
                     if existing_targets.shape[0] > targets.shape[0] or \
                             (existing_targets.shape == targets.shape and
                              np.allclose(existing_targets, targets)):
@@ -299,7 +299,7 @@ def load_targets_ensemble(self):
         lock_path = filepath + '.lock'
         with lockfile.LockFile(lock_path):
             with open(filepath, 'rb') as fh:
-                targets = np.load(fh)
+                targets = np.load(fh, allow_pickle=True)
 
         return targets
 

test/test_automl/test_estimators.py

@@ -271,7 +271,7 @@ def test_fit_pSMAC(self):
         true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn',
                                                   'true_targets_ensemble.npy')
         with open(true_targets_ensemble_path, 'rb') as fh:
-            true_targets_ensemble = np.load(fh)
+            true_targets_ensemble = np.load(fh, allow_pickle=True)
         true_targets_ensemble[-1] = 1 if true_targets_ensemble[-1] != 1 else 0
         true_targets_ensemble = true_targets_ensemble.astype(int)
         probas = np.zeros((len(true_targets_ensemble), 2), dtype=float)

test/test_ensemble_builder/test_ensemble.py

@@ -16,27 +16,27 @@
 
 
 class BackendMock(object):
-    
+
     def __init__(self):
         this_directory = os.path.abspath(
             os.path.dirname(__file__)
         )
         self.temporary_directory = os.path.join(
             this_directory, 'data',
         )
-    
+
     def load_targets_ensemble(self):
         with open(os.path.join(
             self.temporary_directory,
             ".auto-sklearn",
             "predictions_ensemble",
             "predictions_ensemble_true.npy"
         ),"rb") as fp:
-            y = np.load(fp)
+            y = np.load(fp, allow_pickle=True)
         return y
-    
+
 class EnsembleBuilderMemMock(EnsembleBuilder):
-    
+
     def fit_ensemble(self,selected_keys):
         np.ones([10000000,1000000])
 
@@ -49,7 +49,7 @@ def tearDown(self):
         pass
 
     def testRead(self):
-        
+
         ensbuilder = EnsembleBuilder(
             backend=self.backend,
             dataset_name="TEST",
@@ -74,9 +74,9 @@ def testRead(self):
             ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
         )
         self.assertEqual(ensbuilder.read_preds[filename]["ens_score"], 1.0)
-                    
+
     def testNBest(self):
-        
+
         ensbuilder = EnsembleBuilder(
             backend=self.backend,
             dataset_name="TEST",
@@ -86,7 +86,7 @@ def testNBest(self):
             seed=0, # important to find the test files
             ensemble_nbest=1,
         )
-        
+
         ensbuilder.read_ensemble_preds()
         sel_keys = ensbuilder.get_n_best_preds()
 
@@ -97,18 +97,18 @@ def testNBest(self):
             ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
         )
         self.assertEquals(sel_keys[0], fixture)
-        
+
     def testFallBackNBest(self):
-        
-        ensbuilder = EnsembleBuilder(backend=self.backend, 
+
+        ensbuilder = EnsembleBuilder(backend=self.backend,
                                     dataset_name="TEST",
                                     task_type=1,  #Binary Classification
                                     metric=roc_auc,
                                     limit=-1, # not used,
                                     seed=0, # important to find the test files
                                     ensemble_nbest=1
                                     )
-        
+
         ensbuilder.read_ensemble_preds()
 
         filename = os.path.join(
@@ -122,28 +122,28 @@ def testFallBackNBest(self):
             ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
         )
         ensbuilder.read_preds[filename]["ens_score"] = -1
-        
+
         sel_keys = ensbuilder.get_n_best_preds()
 
         fixture = os.path.join(
             self.backend.temporary_directory,
             ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
         )
         self.assertEquals(sel_keys[0], fixture)
-        
+
     def testGetValidTestPreds(self):
-        
-        ensbuilder = EnsembleBuilder(backend=self.backend, 
+
+        ensbuilder = EnsembleBuilder(backend=self.backend,
                                     dataset_name="TEST",
                                     task_type=1,  #Binary Classification
                                     metric=roc_auc,
                                     limit=-1, # not used,
                                     seed=0, # important to find the test files
                                     ensemble_nbest=1
                                     )
-        
+
         ensbuilder.read_ensemble_preds()
-        
+
         d2 = os.path.join(
             self.backend.temporary_directory,
             ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_2.npy"
@@ -152,21 +152,21 @@ def testGetValidTestPreds(self):
             self.backend.temporary_directory,
             ".auto-sklearn/predictions_ensemble/predictions_ensemble_0_1.npy"
         )
-        
+
         sel_keys = ensbuilder.get_n_best_preds()
-        
+
         ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
-        
+
         # selected --> read valid and test predictions
         self.assertIsNotNone(ensbuilder.read_preds[d2][Y_VALID])
         self.assertIsNotNone(ensbuilder.read_preds[d2][Y_TEST])
-        
+
         # not selected --> should still be None
         self.assertIsNone(ensbuilder.read_preds[d1][Y_VALID])
         self.assertIsNone(ensbuilder.read_preds[d1][Y_TEST])
-        
+
     def testEntireEnsembleBuilder(self):
-        
+
         ensbuilder = EnsembleBuilder(
             backend=self.backend,
             dataset_name="TEST",
@@ -177,7 +177,7 @@ def testEntireEnsembleBuilder(self):
             ensemble_nbest=2,
         )
         ensbuilder.SAVE2DISC = False
-        
+
         ensbuilder.read_ensemble_preds()
 
         d2 = os.path.join(
@@ -187,12 +187,12 @@ def testEntireEnsembleBuilder(self):
 
         sel_keys = ensbuilder.get_n_best_preds()
         self.assertGreater(len(sel_keys), 0)
-        
+
         ensemble = ensbuilder.fit_ensemble(selected_keys=sel_keys)
         print(ensemble, sel_keys)
-        
+
         n_sel_valid, n_sel_test = ensbuilder.get_valid_test_preds(selected_keys=sel_keys)
-        
+
         # both valid and test prediction files are available
         self.assertGreater(len(n_sel_valid), 0)
         self.assertEqual(n_sel_valid, n_sel_test)
@@ -221,10 +221,10 @@ def testEntireEnsembleBuilder(self):
         # so that y_valid should be exactly y_valid_d2
         y_valid_d2 = ensbuilder.read_preds[d2][Y_VALID][:, 1]
         np.testing.assert_array_almost_equal(y_valid, y_valid_d2)
-        
+
     def testMain(self):
-        
-        ensbuilder = EnsembleBuilder(backend=self.backend, 
+
+        ensbuilder = EnsembleBuilder(backend=self.backend,
                                     dataset_name="TEST",
                                     task_type=1,  #Binary Classification
                                     metric=roc_auc,
@@ -234,17 +234,17 @@ def testMain(self):
                                     max_iterations=1 # prevents infinite loop
                                     )
         ensbuilder.SAVE2DISC = False
-        
+
         ensbuilder.main()
-        
+
         self.assertEqual(len(ensbuilder.read_preds), 2)
         self.assertIsNotNone(ensbuilder.last_hash)
         self.assertIsNotNone(ensbuilder.y_true_ensemble)
-        
+
     def testLimit(self):
-        
-                
-        ensbuilder = EnsembleBuilderMemMock(backend=self.backend, 
+
+
+        ensbuilder = EnsembleBuilderMemMock(backend=self.backend,
                                             dataset_name="TEST",
                                             task_type=1,  #Binary Classification
                                             metric=roc_auc,
@@ -255,9 +255,9 @@ def testLimit(self):
                                             memory_limit=10 # small memory limit to trigger MemoryException
                                             )
         ensbuilder.SAVE2DISC = False
-        
+
         ensbuilder.run()
-        
+
         # it should try to reduce ensemble_nbest until it also failed at 2
         self.assertEqual(ensbuilder.ensemble_nbest,1)