Drop lockfile requirement

The lockfile requirement is no longer necessary since the drop of
pSMAC for parallel SMAC/Auto-sklearn as all parallelism is now
controlled by a central Auto-sklearn instance which saves the
datamanager and the ensemble targets to disk.

As a positive side-effect this removes a potential deadlock in the
ensemble builder.

Author: mfeurer

autosklearn/ensemble_builder.py

@@ -148,7 +148,7 @@ def __call__(
         result: RunValue,
         time_left: float,
     ):
-        if result.status in (StatusType.STOP, StatusType.ABORT):
+        if result.status in (StatusType.STOP, StatusType.ABORT) or smbo._stop:
             return
         self.build_ensemble(smbo.tae_runner.client)
 
@@ -598,12 +598,13 @@ def run(
                 else:
                     time_left = end_at - current_time
 
-            if time_left - time_buffer < 1:
+            wall_time_in_s = int(time_left - time_buffer)
+            if wall_time_in_s < 1:
                 break
             context = multiprocessing.get_context(pynisher_context)
 
             safe_ensemble_script = pynisher.enforce_limits(
-                wall_time_in_s=int(time_left - time_buffer),
+                wall_time_in_s=wall_time_in_s,
                 mem_in_mb=self.memory_limit,
                 logger=self.logger,
                 context=context,

autosklearn/util/backend.py

@@ -7,8 +7,6 @@
 import uuid
 from typing import Dict, List, Optional, Tuple, Union
 
-import lockfile
-
 import numpy as np
 
 from sklearn.pipeline import Pipeline
@@ -258,30 +256,20 @@ def save_targets_ensemble(self, targets: np.ndarray) -> str:
         except Exception:
             pass
 
-        with lockfile.LockFile(filepath):
-            if os.path.exists(filepath):
-                with open(filepath, 'rb') as fh:
-                    existing_targets = np.load(fh, allow_pickle=True)
-                    if existing_targets.shape[0] > targets.shape[0] or \
-                            (existing_targets.shape == targets.shape and
-                             np.allclose(existing_targets, targets)):
-                        return filepath
-
-            with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
-                    filepath), delete=False) as fh_w:
-                np.save(fh_w, targets.astype(np.float32))
-                tempname = fh_w.name
+        with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
+                filepath), delete=False) as fh_w:
+            np.save(fh_w, targets.astype(np.float32))
+            tempname = fh_w.name
 
-            os.rename(tempname, filepath)
+        os.rename(tempname, filepath)
 
         return filepath
 
     def load_targets_ensemble(self) -> np.ndarray:
         filepath = self._get_targets_ensemble_filename()
 
-        with lockfile.LockFile(filepath):
-            with open(filepath, 'rb') as fh:
-                targets = np.load(fh, allow_pickle=True)
+        with open(filepath, 'rb') as fh:
+            targets = np.load(fh, allow_pickle=True)
 
         return targets
 
@@ -292,21 +280,18 @@ def save_datamanager(self, datamanager: AbstractDataManager) -> str:
         self._make_internals_directory()
         filepath = self._get_datamanager_pickle_filename()
 
-        with lockfile.LockFile(filepath):
-            if not os.path.exists(filepath):
-                with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
-                        filepath), delete=False) as fh:
-                    pickle.dump(datamanager, fh, -1)
-                    tempname = fh.name
-                os.rename(tempname, filepath)
+        with tempfile.NamedTemporaryFile('wb', dir=os.path.dirname(
+                filepath), delete=False) as fh:
+            pickle.dump(datamanager, fh, -1)
+            tempname = fh.name
+        os.rename(tempname, filepath)
 
         return filepath
 
     def load_datamanager(self) -> AbstractDataManager:
         filepath = self._get_datamanager_pickle_filename()
-        with lockfile.LockFile(filepath):
-            with open(filepath, 'rb') as fh:
-                return pickle.load(fh)
+        with open(filepath, 'rb') as fh:
+            return pickle.load(fh)
 
     def get_runs_directory(self) -> str:
         return os.path.join(self.internals_directory, 'runs')
@@ -484,10 +469,9 @@ def save_predictions_as_txt(self,
         os.rename(tempname, filepath)
 
     def write_txt_file(self, filepath: str, data: str, name: str) -> None:
-        with lockfile.LockFile(filepath):
-            with tempfile.NamedTemporaryFile('w', dir=os.path.dirname(
-                    filepath), delete=False) as fh:
-                fh.write(data)
-                tempname = fh.name
-            os.rename(tempname, filepath)
-            self.logger.debug('Created %s file %s' % (name, filepath))
+        with tempfile.NamedTemporaryFile('w', dir=os.path.dirname(
+                filepath), delete=False) as fh:
+            fh.write(data)
+            tempname = fh.name
+        os.rename(tempname, filepath)
+        self.logger.debug('Created %s file %s' % (name, filepath))

requirements.txt

@@ -8,7 +8,6 @@ scikit-learn>=0.23.0,<0.24
 
 dask
 distributed>=2.2.0
-lockfile
 pyyaml
 pandas>=1.0
 liac-arff