Add option to disable cleaning of the test stage directory

To better support this, the `os_ext.copytree()` was rewritten to comply with the
`dirs_exist_ok` argument added in Python 3.8

Author: Vasileios Karakasis

reframe/core/pipeline.py

@@ -1069,9 +1069,11 @@ def _copy_to_stagedir(self, path):
                           (path, self._stagedir))
         self.logger.debug('symlinking files: %s' % self.readonly_files)
         try:
-            os_ext.copytree_virtual(path, self._stagedir, self.readonly_files)
+            os_ext.copytree_virtual(
+                path, self._stagedir, self.readonly_files, dirs_exist_ok=True
+            )
         except (OSError, ValueError, TypeError) as e:
-            raise PipelineError('virtual copying of files failed') from e
+            raise PipelineError('copying of files failed') from e
 
     def _clone_to_stagedir(self, url):
         self.logger.debug('cloning URL %s to stage directory (%s)' %

reframe/core/runtime.py

@@ -145,9 +145,8 @@ def make_stagedir(self, *dirs):
                              *self._format_dirs(*dirs), wipeout=wipeout)
 
     def make_outputdir(self, *dirs):
-        wipeout = self.get_option('general/0/clean_stagedir')
         return self._makedir(self.output_prefix,
-                             *self._format_dirs(*dirs), wipeout=wipeout)
+                             *self._format_dirs(*dirs), wipeout=True)
 
     @property
     def modules_system(self):

reframe/utility/os_ext.py

@@ -95,25 +95,44 @@ def osgroup():
 
 
 def copytree(src, dst, symlinks=False, ignore=None, copy_function=shutil.copy2,
-             ignore_dangling_symlinks=False):
-    '''Same as shutil.copytree() but valid also if 'dst' exists.
-
-    In this case it will first remove it and then call the standard
-    shutil.copytree().'''
+             ignore_dangling_symlinks=False, dirs_exist_ok=False):
+    '''Compatibility version of :py:func:`shutil.copytree()` for Python <= 3.8
+    '''
     if src == os.path.commonpath([src, dst]):
         raise ValueError("cannot copy recursively the parent directory "
                          "`%s' into one of its descendants `%s'" % (src, dst))
 
-    if os.path.exists(dst):
-        shutil.rmtree(dst)
+    if sys.version_info[2] >= 8:
+        return shutil.copytree(src, dst, symlinks, ignore, copy_function,
+                               ignore_dangling_symlinks, dirs_exist_ok)
+
+    if not dirs_exist_ok:
+        return shutil.copytree(src, dst, symlinks, ignore, copy_function,
+                               ignore_dangling_symlinks)
+
+    # dirs_exist_ok=True and Python < 3.8
+    if not os.path.exists(dst):
+        return shutil.copytree(src, dst, symlinks, ignore, copy_function,
+                               ignore_dangling_symlinks)
+
+    # dst exists; manually descend into the subdirectories
+    _, subdirs, files = list(os.walk(src))[0]
+    ignore_paths = ignore(src, os.listdir(src)) if ignore else {}
+    for f in files:
+        if f not in ignore_paths:
+            copy_function(os.path.join(src, f), os.path.join(dst, f))
 
-    shutil.copytree(src, dst, symlinks, ignore, copy_function,
-                    ignore_dangling_symlinks)
+    for d in subdirs:
+        if d not in ignore_paths:
+            copytree(os.path.join(src, d), os.path.join(dst, d), symlinks,
+                     ignore, copy_function, ignore_dangling_symlinks)
+
+    return dst
 
 
 def copytree_virtual(src, dst, file_links=[],
                      symlinks=False, copy_function=shutil.copy2,
-                     ignore_dangling_symlinks=False):
+                     ignore_dangling_symlinks=False, dirs_exist_ok=False):
     '''Copy `dst` to `src`, but create symlinks for the files in `file_links`.
 
     If `file_links` is empty, this is equivalent to `copytree()`.  The rest of
@@ -134,35 +153,42 @@ def copytree_virtual(src, dst, file_links=[],
     link_targets = set()
     for f in file_links:
         if os.path.isabs(f):
-            raise ValueError("copytree_virtual() failed: `%s': "
-                             "absolute paths not allowed in file_links" % f)
+            raise ValueError(f'copytree_virtual() failed: {f!r}: '
+                             f'absolute paths not allowed in file_links')
 
         target = os.path.join(src, f)
         if not os.path.exists(target):
-            raise ValueError("copytree_virtual() failed: `%s' "
-                             "does not exist" % target)
+            raise ValueError(f'copytree_virtual() failed: {target!r} '
+                             f'does not exist')
 
         if os.path.commonpath([src, target]) != src:
-            raise ValueError("copytree_virtual() failed: "
-                             "`%s' not under `%s'" % (target, src))
+            raise ValueError(f'copytree_virtual() failed: '
+                             f'{target!r} not under {src!r}')
 
         link_targets.add(os.path.abspath(target))
 
+    if '.' in file_links or '..' in file_links:
+        raise ValueError(f"'.' or '..' are not allowed in file_links")
+
     if not file_links:
         ignore = None
     else:
         def ignore(dir, contents):
-            return [c for c in contents
-                    if os.path.join(dir, c) in link_targets]
+            return {c for c in contents
+                    if os.path.join(dir, c) in link_targets}
 
     # Copy to dst ignoring the file_links
     copytree(src, dst, symlinks, ignore,
-             copy_function, ignore_dangling_symlinks)
+             copy_function, ignore_dangling_symlinks, dirs_exist_ok)
 
     # Now create the symlinks
     for f in link_targets:
         link_name = f.replace(src, dst)
-        os.symlink(f, link_name)
+        try:
+            os.symlink(f, link_name)
+        except FileExistsError:
+            if not dirs_exist_ok:
+                raise
 
 
 def rmtree(*args, max_retries=3, **kwargs):

unittests/test_utility.py

@@ -61,15 +61,7 @@ def test_command_async(self):
     def test_copytree(self):
         dir_src = tempfile.mkdtemp()
         dir_dst = tempfile.mkdtemp()
-
-        with pytest.raises(OSError):
-            shutil.copytree(dir_src, dir_dst)
-
-        try:
-            os_ext.copytree(dir_src, dir_dst)
-        except Exception as e:
-            pytest.fail('custom copytree failed: %s' % e)
-
+        os_ext.copytree(dir_src, dir_dst, dirs_exist_ok=True)
         shutil.rmtree(dir_src)
         shutil.rmtree(dir_dst)
 
@@ -301,38 +293,50 @@ def verify_target_directory(self, file_links=[]):
             assert target_name == os.readlink(link_name)
 
     def test_virtual_copy_nolinks(self):
-        os_ext.copytree_virtual(self.prefix, self.target)
+        os_ext.copytree_virtual(self.prefix, self.target, dirs_exist_ok=True)
         self.verify_target_directory()
 
     def test_virtual_copy_valid_links(self):
         file_links = ['bar/', 'foo/bar.txt', 'foo.txt']
-        os_ext.copytree_virtual(self.prefix, self.target, file_links)
+        os_ext.copytree_virtual(self.prefix, self.target,
+                                file_links, dirs_exist_ok=True)
         self.verify_target_directory(file_links)
 
     def test_virtual_copy_inexistent_links(self):
         file_links = ['foobar/', 'foo/bar.txt', 'foo.txt']
         with pytest.raises(ValueError):
-            os_ext.copytree_virtual(self.prefix, self.target, file_links)
+            os_ext.copytree_virtual(self.prefix, self.target,
+                                    file_links, dirs_exist_ok=True)
 
     def test_virtual_copy_absolute_paths(self):
         file_links = [os.path.join(self.prefix, 'bar'),
                       'foo/bar.txt', 'foo.txt']
         with pytest.raises(ValueError):
-            os_ext.copytree_virtual(self.prefix, self.target, file_links)
+            os_ext.copytree_virtual(self.prefix, self.target,
+                                    file_links, dirs_exist_ok=True)
 
     def test_virtual_copy_irrelevenant_paths(self):
         file_links = ['/bin', 'foo/bar.txt', 'foo.txt']
         with pytest.raises(ValueError):
-            os_ext.copytree_virtual(self.prefix, self.target, file_links)
+            os_ext.copytree_virtual(self.prefix, self.target,
+                                    file_links, dirs_exist_ok=True)
 
         file_links = [os.path.dirname(self.prefix), 'foo/bar.txt', 'foo.txt']
         with pytest.raises(ValueError):
-            os_ext.copytree_virtual(self.prefix, self.target, file_links)
+            os_ext.copytree_virtual(self.prefix, self.target,
+                                    file_links, dirs_exist_ok=True)
 
     def test_virtual_copy_linkself(self):
         file_links = ['.']
-        with pytest.raises(OSError):
-            os_ext.copytree_virtual(self.prefix, self.target, file_links)
+        with pytest.raises(ValueError):
+            os_ext.copytree_virtual(self.prefix, self.target,
+                                    file_links, dirs_exist_ok=True)
+
+    def test_virtual_copy_linkparent(self):
+        file_links = ['..']
+        with pytest.raises(ValueError):
+            os_ext.copytree_virtual(self.prefix, self.target,
+                                    file_links, dirs_exist_ok=True)
 
     def tearDown(self):
         shutil.rmtree(self.prefix)