Merge pull request #3152 from Flamefire/empty_folders

Only remove folders that contain no files recursively

Author: bartoldeman

easybuild/framework/easyblock.py

@@ -75,7 +75,7 @@
 from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file, dir_contains_files
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
@@ -1305,15 +1305,14 @@ def make_module_req(self):
                     if path and not self.dry_run:
                         paths = sorted(glob.glob(path))
                         if paths and key in keys_requiring_files:
-                            self.log.info("Only retaining paths for %s that include at least one file: %s", key, paths)
-                            # only retain paths that include at least one file
-                            retained_paths = []
-                            for path in paths:
-                                full_path = os.path.join(self.installdir, path)
-                                if os.path.isdir(full_path):
-                                    if any(os.path.isfile(os.path.join(full_path, x)) for x in os.listdir(full_path)):
-                                        retained_paths.append(path)
-                            self.log.info("Retained paths for %s: %s", key, retained_paths)
+                            # only retain paths that contain at least one file
+                            retained_paths = [
+                                path for path in paths
+                                if os.path.isdir(os.path.join(self.installdir, path))
+                                and dir_contains_files(os.path.join(self.installdir, path))
+                            ]
+                            self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s",
+                                          key, paths, retained_paths)
                             paths = retained_paths
                     else:
                         # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME)

easybuild/tools/filetools.py

@@ -681,6 +681,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
     return var_defs, hits
 
 
+def dir_contains_files(path):
+    """Return True if the given directory does contain any file in itself or any subdirectory"""
+    return any(files for _root, _dirs, files in os.walk(path))
+
+
 def find_eb_script(script_name):
     """Find EasyBuild script with given name (in easybuild/scripts subdirectory)."""
     filetools, eb_dir = __file__, None
@@ -746,7 +751,7 @@ def calc_block_checksum(path, algorithm):
     try:
         # in hashlib, blocksize is a class parameter
         blocksize = algorithm.blocksize * 262144  # 2^18
-    except AttributeError as err:
+    except AttributeError:
         blocksize = 16777216  # 2^24
     _log.debug("Using blocksize %s for calculating the checksum" % blocksize)
 

test/framework/filetools.py

@@ -1724,6 +1724,30 @@ def test_search_file(self):
         for pattern in ['*foo', '(foo', ')foo', 'foo)', 'foo(']:
             self.assertErrorRegex(EasyBuildError, "Invalid search query", ft.search_file, [test_ecs], pattern)
 
+    def test_dir_contains_files(self):
+        def makedirs_in_test(*paths):
+            """Make dir specified by paths and return top-level folder"""
+            os.makedirs(os.path.join(self.test_prefix, *paths))
+            return os.path.join(self.test_prefix, paths[0])
+
+        empty_dir = makedirs_in_test('empty_dir')
+        self.assertFalse(ft.dir_contains_files(empty_dir))
+
+        dir_w_subdir = makedirs_in_test('dir_w_subdir', 'sub_dir')
+        self.assertFalse(ft.dir_contains_files(dir_w_subdir))
+
+        dir_subdir_file = makedirs_in_test('dir_subdir_file', 'sub_dir_w_file')
+        ft.write_file(os.path.join(dir_subdir_file, 'sub_dir_w_file', 'file.h'), '')
+        self.assertTrue(ft.dir_contains_files(dir_subdir_file))
+
+        dir_w_file = makedirs_in_test('dir_w_file')
+        ft.write_file(os.path.join(dir_w_file, 'file.h'), '')
+        self.assertTrue(ft.dir_contains_files(dir_w_file))
+
+        dir_w_dir_and_file = makedirs_in_test('dir_w_dir_and_file', 'sub_dir')
+        ft.write_file(os.path.join(dir_w_dir_and_file, 'file.h'), '')
+        self.assertTrue(ft.dir_contains_files(dir_w_dir_and_file))
+
     def test_find_eb_script(self):
         """Test find_eb_script function."""
         self.assertTrue(os.path.exists(ft.find_eb_script('rpath_args.py')))