Merge pull request #3916 from easybuilders/4.5.x

boegel · web-flow · commit d18b6f0e2563 · 2021-12-13T12:46:21.000+01:00
release EasyBuild v4.5.1
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -4,6 +4,26 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.5.1 (December 13th 2021)
+---------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - also dump environment to reprod directory (#3374)
+  - determine which extensions can be skipped in parallel (if --parallel-extensions-install is enabled) (#3890)
+  - fall back to sequential installation for extensions with unknown dependencies when using --parallel-extensions-install (#3906)
+  - allow oversubscription in sanity check for OpenMPI-based toolchains (#3909)
+- various bug fixes, including:
+  - don't try to ensure absolute path for path part of repositorypath (#3893, #3899)
+  - fix typo in EULA agreement error message (#3894)
+  - only remove lock if it was created in the same EasyBuild session (not if it existed already) (#3889)
+  - introduce EasyBlock.post_init method to correctly define builddir variable when build-in-installdir mode is enabled in easyconfig or easyblock (#3900)
+  - also show download progress bar when using --inject-checksums (#3905)
+  - pick up custom extract_cmd specified for extension (#3907)
+  - make test_run_cmd_async more robust against fluke failures (#3908)
+
+
 v4.5.0 (October 29th 2021)
 --------------------------
 
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py
@@ -557,6 +557,45 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
     return '\n'.join(lines)
 
 
+def dump_env_easyblock(app, orig_env=None, ec_path=None, script_path=None, silent=False):
+    if orig_env is None:
+        orig_env = copy.deepcopy(os.environ)
+    if ec_path is None:
+        raise EasyBuildError("The path to the easyconfig relevant to this environment dump is required")
+    if script_path is None:
+        # Assume we are placing it alongside the easyconfig path
+        script_path = '%s.env' % os.path.splitext(ec_path)[0]
+    # Compose script
+    ecfile = os.path.basename(ec_path)
+    script_lines = [
+        "#!/bin/bash",
+        "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
+        "# usage: source %s" % os.path.basename(script_path),
+    ]
+
+    script_lines.extend(['', "# toolchain & dependency modules"])
+    if app.toolchain.modules:
+        script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
+    else:
+        script_lines.append("# (no modules loaded)")
+
+    script_lines.extend(['', "# build environment"])
+    if app.toolchain.vars:
+        env_vars = sorted(app.toolchain.vars.items())
+        script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
+    else:
+        script_lines.append("# (no build environment defined)")
+
+    write_file(script_path, '\n'.join(script_lines))
+    msg = "Script to set up build environment for %s dumped to %s" % (ecfile, script_path)
+    if silent:
+        _log.info(msg)
+    else:
+        print_msg(msg, prefix=False)
+
+    restore_env(orig_env)
+
+
 def dump_env_script(easyconfigs):
     """
     Dump source scripts that set up build environment for specified easyconfigs.
@@ -591,31 +630,8 @@ def dump_env_script(easyconfigs):
         app.check_readiness_step()
         app.prepare_step(start_dir=False)
 
-        # compose script
-        ecfile = os.path.basename(ec.path)
-        script_lines = [
-            "#!/bin/bash",
-            "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile),
-            "# usage: source %s" % os.path.basename(script_path),
-        ]
-
-        script_lines.extend(['', "# toolchain & dependency modules"])
-        if app.toolchain.modules:
-            script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules])
-        else:
-            script_lines.append("# (no modules loaded)")
-
-        script_lines.extend(['', "# build environment"])
-        if app.toolchain.vars:
-            env_vars = sorted(app.toolchain.vars.items())
-            script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars])
-        else:
-            script_lines.append("# (no build environment defined)")
-
-        write_file(script_path, '\n'.join(script_lines))
-        print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False)
-
-        restore_env(orig_env)
+        # create the environment dump
+        dump_env_easyblock(app, orig_env=orig_env, ec_path=ec.path, script_path=script_path)
 
 
 def categorize_files_by_type(paths):
diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py
@@ -119,6 +119,7 @@ def __init__(self, mself, ext, extra_params=None):
 
         # list of source/patch files: we use an empty list as default value like in EasyBlock
         self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values)
+        self.src_extract_cmd = self.ext.get('extract_cmd', None)
         self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values)
         self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values)
 
@@ -225,7 +226,8 @@ def async_cmd_check(self):
     @property
     def required_deps(self):
         """Return list of required dependencies for this extension."""
-        raise NotImplementedError("Don't know how to determine required dependencies for extension '%s'" % self.name)
+        self.log.info("Don't know how to determine required dependencies for extension '%s'", self.name)
+        return None
 
     @property
     def toolchain(self):
diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py
@@ -118,7 +118,7 @@ def run(self, unpack_src=False):
         if unpack_src:
             targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name))
             self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options,
-                                        change_into_dir=False)
+                                        change_into_dir=False, cmd=self.src_extract_cmd)
 
             # setting start dir must be done from unpacked source directory for extension,
             # because start_dir value is usually a relative path (if it is set)
diff --git a/easybuild/main.py b/easybuild/main.py
@@ -531,7 +531,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
         dump_env_script(easyconfigs)
 
     elif options.inject_checksums:
-        inject_checksums(ordered_ecs, options.inject_checksums)
+        with rich_live_cm():
+            inject_checksums(ordered_ecs, options.inject_checksums)
 
     # cleanup and exit after dry run, searching easyconfigs or submitting regression test
     stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
@@ -456,11 +456,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced
     _log.debug("Unpacking %s in directory %s", fn, abs_dest)
     cwd = change_dir(abs_dest)
 
-    if not cmd:
-        cmd = extract_cmd(fn, overwrite=overwrite)
-    else:
+    if cmd:
         # complete command template with filename
         cmd = cmd % fn
+        _log.debug("Using specified command to unpack %s: %s", fn, cmd)
+    else:
+        cmd = extract_cmd(fn, overwrite=overwrite)
+        _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd)
+
     if not cmd:
         raise EasyBuildError("Can't extract file %s with unknown filetype", fn)
 
@@ -1366,7 +1369,7 @@ def find_extension(filename):
     if res:
         ext = res.group('ext')
     else:
-        raise EasyBuildError('Unknown file type for file %s', filename)
+        raise EasyBuildError("%s has unknown file extension", filename)
 
     return ext
 
@@ -1379,7 +1382,9 @@ def extract_cmd(filepath, overwrite=False):
     ext = find_extension(filename)
     target = filename[:-len(ext)]
 
+    # find_extension will either return an extension listed in EXTRACT_CMDS, or raise an error
     cmd_tmpl = EXTRACT_CMDS[ext.lower()]
+
     if overwrite:
         if 'unzip -qq' in cmd_tmpl:
             cmd_tmpl = cmd_tmpl.replace('unzip -qq', 'unzip -qq -o')
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
@@ -1067,7 +1067,7 @@ def _postprocess_checks(self):
 
     def get_cfg_opt_abs_path(self, opt_name, path):
         """Get path value of configuration option as absolute path."""
-        if os.path.isabs(path):
+        if os.path.isabs(path) or path.startswith('git@'):
             abs_path = path
         else:
             abs_path = os.path.abspath(path)
@@ -1092,22 +1092,20 @@ def _ensure_abs_path(self, opt_name):
     def _postprocess_config(self):
         """Postprocessing of configuration options"""
 
-        # resolve relative paths for configuration options that specify a location;
+        # resolve relative paths for configuration options that specify a location,
+        # to avoid incorrect paths being used when EasyBuild changes the current working directory
+        # (see https://github.com/easybuilders/easybuild-framework/issues/3619);
         # ensuring absolute paths for 'robot' is handled separately below,
-        # because we need to be careful with the argument pass to --robot
+        # because we need to be careful with the argument pass to --robot;
+        # note: repositorypath is purposely not listed here, because it's a special case:
+        # - the value could consist of a 2-tuple (<path>, <relative_subdir>);
+        # - the <path> could also specify the location of a *remote* (Git( repository,
+        #   which can be done in variety of formats (git@<url>:<org>/<repo>), https://<url>, etc.)
+        #   (see also https://github.com/easybuilders/easybuild-framework/issues/3892);
         path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
                           'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
                           'robot_paths', 'sourcepath']
 
-        # repositorypath is a special case: only first part is a path;
-        # 2nd (optional) part is a relative subdir and should not be resolved to an absolute path!
-        repositorypath = self.options.repositorypath
-        if isinstance(repositorypath, (list, tuple)) and len(repositorypath) == 2:
-            abs_path = self.get_cfg_opt_abs_path('repositorypath', repositorypath[0])
-            self.options.repositorypath = (abs_path, repositorypath[1])
-        else:
-            path_opt_names.append('repositorypath')
-
         for opt_name in path_opt_names:
             self._ensure_abs_path(opt_name)
 
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.5.0')
+VERSION = LooseVersion('4.5.1')
 UNKNOWN = 'UNKNOWN'
 
 
diff --git a/test/framework/config.py b/test/framework/config.py
@@ -302,7 +302,7 @@ def test_generaloption_config_file(self):
         self.assertEqual(install_path('mod'), installpath_modules),  # via config file
         self.assertEqual(source_paths(), [testpath2])  # via command line
         self.assertEqual(build_path(), testpath1)  # via config file
-        self.assertEqual(get_repositorypath(), (os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'))  # via config file
+        self.assertEqual(get_repositorypath(), [os.path.join(topdir, 'ebfiles_repo'), 'somesubdir'])  # via config file
 
         # hardcoded first entry
         self.assertEqual(options.robot_paths[0], '/tmp/foo')
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
@@ -1335,8 +1335,7 @@ def test_buildininstalldir(self):
         self.prep()
         ec = EasyConfig(self.eb_file)
         eb = EasyBlock(ec)
-        eb.gen_builddir()
-        eb.gen_installdir()
+        eb.post_init()
         eb.make_builddir()
         eb.make_installdir()
         self.assertEqual(eb.builddir, eb.installdir)
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
@@ -114,6 +114,9 @@ def test_extract_cmd(self):
 
         self.assertEqual("unzip -qq -o test.zip", ft.extract_cmd('test.zip', True))
 
+        error_pattern = "test.foo has unknown file extension"
+        self.assertErrorRegex(EasyBuildError, error_pattern, ft.extract_cmd, 'test.foo')
+
     def test_find_extension(self):
         """Test find_extension function."""
         tests = [
diff --git a/test/framework/options.py b/test/framework/options.py
@@ -4854,8 +4854,8 @@ def test_prefix_option(self):
         expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath']
         self.assertEqual(sorted(regex.findall(txt)), expected)
 
-    def test_dump_env_config(self):
-        """Test for --dump-env-config."""
+    def test_dump_env_script(self):
+        """Test for --dump-env-script."""
 
         fftw = 'FFTW-3.3.7-gompic-2018a'
         gcc = 'GCC-4.9.2'
@@ -6037,6 +6037,53 @@ def test_sanity_check_only(self):
 
         stdout = self.mocked_main(args + ['--trace'], do_build=True, raise_error=True, testing=False)
 
+        # check whether %(builddir)s value is correct
+        # when buildininstalldir is enabled in easyconfig and --sanity-check-only is used
+        # (see https://github.com/easybuilders/easybuild-framework/issues/3895)
+        test_ec_txt += '\n' + '\n'.join([
+            "buildininstalldir = True",
+            "sanity_check_commands = [",
+            # build and install directory should be the same path
+            "    'test %(builddir)s = %(installdir)s',",
+            # build/install directory must exist (even though step that creates build dir was never run)
+            "    'test -d %(builddir)s',",
+            "]",
+        ])
+        write_file(test_ec, test_ec_txt)
+        self.eb_main(args, do_build=True, raise_error=True)
+
+        # also check when using easyblock that enables build_in_installdir in its constructor
+        test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks')
+        toy_eb = os.path.join(test_ebs, 't', 'toy.py')
+        toy_eb_txt = read_file(toy_eb)
+
+        self.assertFalse('self.build_in_installdir = True' in toy_eb_txt)
+
+        regex = re.compile(r'^(\s+)(super\(EB_toy, self\).__init__.*)\n', re.M)
+        toy_eb_txt = regex.sub(r'\1\2\n\1self.build_in_installdir = True', toy_eb_txt)
+        self.assertTrue('self.build_in_installdir = True' in toy_eb_txt)
+
+        toy_eb = os.path.join(self.test_prefix, 'toy.py')
+        write_file(toy_eb, toy_eb_txt)
+
+        test_ec_txt = test_ec_txt.replace('buildininstalldir = True', '')
+        write_file(test_ec, test_ec_txt)
+
+        orig_local_sys_path = sys.path[:]
+        args.append('--include-easyblocks=%s' % toy_eb)
+        self.eb_main(args, do_build=True, raise_error=True)
+
+        # undo import of the toy easyblock, to avoid problems with other tests
+        del sys.modules['easybuild.easyblocks.toy']
+        sys.path = orig_local_sys_path
+        import easybuild.easyblocks
+        reload(easybuild.easyblocks)
+        import easybuild.easyblocks.toy
+        reload(easybuild.easyblocks.toy)
+        # need to reload toy_extension, which imports EB_toy, to ensure right EB_toy is picked up in later tests
+        import easybuild.easyblocks.generic.toy_extension
+        reload(easybuild.easyblocks.generic.toy_extension)
+
     def test_skip_extensions(self):
         """Test use of --skip-extensions."""
         topdir = os.path.abspath(os.path.dirname(__file__))
@@ -6242,7 +6289,7 @@ def test_accept_eula_for(self):
 
         # by default, no EULAs are accepted at all
         args = [test_ec, '--force']
-        error_pattern = r"The End User License Argreement \(EULA\) for toy is currently not accepted!"
+        error_pattern = r"The End User License Agreement \(EULA\) for toy is currently not accepted!"
         self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, do_build=True, raise_error=True)
         toy_modfile = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
         if get_module_syntax() == 'Lua':
@@ -6334,7 +6381,7 @@ def test_config_abs_path(self):
             r"^containerpath\s+\(F\) = /.*/test_topdir/test_middle_dir$",
             r"^installpath\s+\(E\) = /.*/test_topdir$",
             r"^prefix\s+\(C\) = /.*/test_topdir/test_middle_dir$",
-            r"^repositorypath\s+\(F\) = \('/apps/easyconfigs_archive', ' somesubdir'\)$",
+            r"^repositorypath\s+\(F\) = /apps/easyconfigs_archive,\s+somesubdir$",
             r"^sourcepath\s+\(C\) = /.*/test_topdir/test_middle_dir/test_subdir$",
             r"^robot-paths\s+\(E\) = /.*/test_topdir$",
         ]
@@ -6361,6 +6408,25 @@ def test_config_abs_path(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (pattern, txt))
 
+    def test_config_repositorypath(self):
+        """Test how special repositorypath values are handled."""
+
+        repositorypath = 'git@github.com:boegel/my_easyconfigs.git'
+        args = [
+            '--repositorypath=%s' % repositorypath,
+            '--show-config',
+        ]
+        txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)
+
+        regex = re.compile(r'repositorypath\s+\(C\) = %s' % repositorypath, re.M)
+        self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt))
+
+        args[0] = '--repositorypath=%s,some/subdir' % repositorypath
+        txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False, strip=True)
+
+        regex = re.compile(r"repositorypath\s+\(C\) = %s, some/subdir" % repositorypath, re.M)
+        self.assertTrue(regex.search(txt), "Pattern '%s' should be found in: %s" % (regex.pattern, txt))
+
     # end-to-end testing of unknown filename
     def test_easystack_wrong_read(self):
         """Test for --easystack <easystack.yaml> when wrong name is provided"""
diff --git a/test/framework/run.py b/test/framework/run.py
@@ -605,7 +605,9 @@ def test_run_cmd_async(self):
         self.assertEqual(res, {'done': False, 'exit_code': None, 'output': 'sleeping...\n'})
 
         # 2nd check with default output size (1024) gets full output
-        res = check_async_cmd(*cmd_info, output=res['output'])
+        # (keep checking until command is fully done)
+        while not res['done']:
+            res = check_async_cmd(*cmd_info, output=res['output'])
         self.assertEqual(res, {'done': True, 'exit_code': 0, 'output': 'sleeping...\ntest123\n'})
 
         # check asynchronous running of failing command
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
