Merge pull request #3551 from Flamefire/create_unused_dir

add create_unused_dir function to create a directory which does not yet exist

Author: boegel

easybuild/tools/filetools.py

@@ -1667,6 +1667,25 @@ def patch_perl_script_autoflush(path):
         write_file(path, newtxt)
 
 
+def set_gid_sticky_bits(path, set_gid=None, sticky=None, recursive=False):
+    """Set GID/sticky bits on specified path."""
+    if set_gid is None:
+        set_gid = build_option('set_gid_bit')
+    if sticky is None:
+        sticky = build_option('sticky_bit')
+
+    bits = 0
+    if set_gid:
+        bits |= stat.S_ISGID
+    if sticky:
+        bits |= stat.S_ISVTX
+    if bits:
+        try:
+            adjust_permissions(path, bits, add=True, relative=True, recursive=recursive, onlydirs=True)
+        except OSError as err:
+            raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err)
+
+
 def mkdir(path, parents=False, set_gid=None, sticky=None):
     """
     Create a directory
@@ -1702,18 +1721,9 @@ def mkdir(path, parents=False, set_gid=None, sticky=None):
             raise EasyBuildError("Failed to create directory %s: %s", path, err)
 
         # set group ID and sticky bits, if desired
-        bits = 0
-        if set_gid:
-            bits |= stat.S_ISGID
-        if sticky:
-            bits |= stat.S_ISVTX
-        if bits:
-            try:
-                new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep)
-                new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0])
-                adjust_permissions(new_path, bits, add=True, relative=True, recursive=True, onlydirs=True)
-            except OSError as err:
-                raise EasyBuildError("Failed to set groud ID/sticky bit: %s", err)
+        new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep)
+        new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0])
+        set_gid_sticky_bits(new_path, set_gid, sticky, recursive=True)
     else:
         _log.debug("Not creating existing path %s" % path)
 
@@ -2588,3 +2598,32 @@ def copy_framework_files(paths, target_dir):
             raise EasyBuildError("Couldn't find parent folder of updated file: %s", path)
 
     return file_info
+
+
+def create_unused_dir(parent_folder, name):
+    """
+    Create a new folder in parent_folder using name as the name.
+    When a folder of that name already exists, '_0' is appended which is retried for increasing numbers until
+    an unused name was found
+    """
+    if not os.path.isabs(parent_folder):
+        parent_folder = os.path.abspath(parent_folder)
+
+    start_path = os.path.join(parent_folder, name)
+    for number in range(-1, 10000):  # Start with no suffix and limit the number of attempts
+        if number < 0:
+            path = start_path
+        else:
+            path = start_path + '_' + str(number)
+        try:
+            os.mkdir(path)
+            break
+        except OSError as err:
+            # Distinguish between error due to existing folder and anything else
+            if not os.path.exists(path):
+                raise EasyBuildError("Failed to create directory %s: %s", path, err)
+
+    # set group ID and sticky bits, if desired
+    set_gid_sticky_bits(path, recursive=True)
+
+    return path

test/framework/filetools.py

@@ -2947,6 +2947,110 @@ def test_locate_files(self):
         error_pattern = r"One or more files not found: 2\.txt \(search paths: \)"
         self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], [])
 
+    def test_set_gid_sticky_bits(self):
+        """Test for set_gid_sticky_bits function."""
+        test_dir = os.path.join(self.test_prefix, 'test_dir')
+        test_subdir = os.path.join(test_dir, 'subdir')
+
+        ft.mkdir(test_subdir, parents=True)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, 0)
+        self.assertEqual(dir_perms & stat.S_ISVTX, 0)
+        dir_perms = os.lstat(test_subdir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, 0)
+        self.assertEqual(dir_perms & stat.S_ISVTX, 0)
+
+        # by default, GID & sticky bits are not set
+        ft.set_gid_sticky_bits(test_dir)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, 0)
+        self.assertEqual(dir_perms & stat.S_ISVTX, 0)
+
+        ft.set_gid_sticky_bits(test_dir, set_gid=True)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+        self.assertEqual(dir_perms & stat.S_ISVTX, 0)
+        ft.remove_dir(test_dir)
+        ft.mkdir(test_subdir, parents=True)
+
+        ft.set_gid_sticky_bits(test_dir, sticky=True)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, 0)
+        self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+        ft.remove_dir(test_dir)
+        ft.mkdir(test_subdir, parents=True)
+
+        ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+        self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+        # no recursion by default
+        dir_perms = os.lstat(test_subdir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, 0)
+        self.assertEqual(dir_perms & stat.S_ISVTX, 0)
+
+        ft.remove_dir(test_dir)
+        ft.mkdir(test_subdir, parents=True)
+
+        ft.set_gid_sticky_bits(test_dir, set_gid=True, sticky=True, recursive=True)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+        self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+        dir_perms = os.lstat(test_subdir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+        self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+
+        ft.remove_dir(test_dir)
+        ft.mkdir(test_subdir, parents=True)
+
+        # set_gid_sticky_bits honors relevant build options
+        init_config(build_options={'set_gid_bit': True, 'sticky_bit': True})
+        ft.set_gid_sticky_bits(test_dir, recursive=True)
+        dir_perms = os.lstat(test_dir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+        self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+        dir_perms = os.lstat(test_subdir)[stat.ST_MODE]
+        self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID)
+        self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX)
+
+    def test_create_unused_dir(self):
+        """Test create_unused_dir function."""
+        path = ft.create_unused_dir(self.test_prefix, 'folder')
+        self.assertEqual(path, os.path.join(self.test_prefix, 'folder'))
+        self.assertTrue(os.path.exists(path))
+
+        # Repeat with existing folder(s) should create new ones
+        for i in range(10):
+            path = ft.create_unused_dir(self.test_prefix, 'folder')
+            self.assertEqual(path, os.path.join(self.test_prefix, 'folder_%s' % i))
+            self.assertTrue(os.path.exists(path))
+
+        # Not influenced by similar folder
+        path = ft.create_unused_dir(self.test_prefix, 'folder2')
+        self.assertEqual(path, os.path.join(self.test_prefix, 'folder2'))
+        self.assertTrue(os.path.exists(path))
+        for i in range(10):
+            path = ft.create_unused_dir(self.test_prefix, 'folder2')
+            self.assertEqual(path, os.path.join(self.test_prefix, 'folder2_%s' % i))
+            self.assertTrue(os.path.exists(path))
+
+        # Fail cleanly if passed a readonly folder
+        readonly_dir = os.path.join(self.test_prefix, 'ro_folder')
+        ft.mkdir(readonly_dir)
+        old_perms = os.lstat(readonly_dir)[stat.ST_MODE]
+        ft.adjust_permissions(readonly_dir, stat.S_IREAD | stat.S_IEXEC, relative=False)
+        try:
+            self.assertErrorRegex(EasyBuildError, 'Failed to create directory',
+                                  ft.create_unused_dir, readonly_dir, 'new_folder')
+        finally:
+            ft.adjust_permissions(readonly_dir, old_perms, relative=False)
+
+        # Ignore files same as folders. So first just create a file with no contents
+        ft.write_file(os.path.join(self.test_prefix, 'file'), '')
+        path = ft.create_unused_dir(self.test_prefix, 'file')
+        self.assertEqual(path, os.path.join(self.test_prefix, 'file_0'))
+        self.assertTrue(os.path.exists(path))
+
 
 def suite():
     """ returns all the testcases in this module """