Merge branch '5.0.x' into run_shell_cmd_out_files

boegel · boegel · commit efbfd20976ec · 2023-11-06T09:31:59.000+01:00
diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml
@@ -74,7 +74,7 @@ jobs:
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
-          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test
       run: |
@@ -95,7 +95,7 @@ jobs:
           echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
           # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
           # see https://docs.easybuild.io/en/latest/Containers.html
-          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
           export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml
@@ -66,7 +66,7 @@ jobs:
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
-          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz
+          pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test
       run: |
@@ -87,7 +87,7 @@ jobs:
           echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros
           # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild;
           # see https://docs.easybuild.io/en/latest/Containers.html
-          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
+          curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
           export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
           export EASYBUILD_CONTAINER_TYPE='apptainer'
diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -25,17 +25,17 @@ jobs:
         run: |
           cd $HOME
           for pkg in easyblocks easyconfigs; do
-              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/develop.tar.gz
-              tar xfz develop.tar.gz
-              rm -f develop.tar.gz
+              curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/5.0.x.tar.gz
+              tar xfz 5.0.x.tar.gz
+              rm -f 5.0.x.tar.gz
           done
 
       - name: Set up environment
         shell: bash
         run: |
           # collect environment variables to be set in subsequent steps in script that can be sourced
           echo "export PATH=$PWD:$PATH" > /tmp/eb_env
-          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-develop:$HOME/easybuild-easyconfigs-develop" >> /tmp/eb_env
+          echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-5.0.x:$HOME/easybuild-easyconfigs-5.0.x" >> /tmp/eb_env
 
       - name: Run commands to check test environment
         shell: bash
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -101,9 +101,9 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
+        if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
             SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
             python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -4,6 +4,25 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.8.2 (29 October 2023)
+------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for `%(sysroot)s` template value (#4359)
+  - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360)
+- various bug fixes, including:
+  - add CI workflow to run unit tests with Python 2 (again) (#4333)
+  - fix typo in help message for `--silence-hook-trigger` (#4343)
+  - include major version (`*majver`) templates in auto-generated documentation (#4347)
+  - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350)
+  - fix `findPythonDeps.py` script when called with an (absolute or relative) path to an easyconfig instead of a filename (#4365)
+  - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366)
+- other changes:
+  - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192)
+
+
 v4.8.1 (11 September 2023)
 --------------------------
 
diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py
@@ -1117,6 +1117,15 @@ def dependencies(self, build_only=False):
 
         return retained_deps
 
+    def dependency_names(self, build_only=False):
+        """
+        Return a set of names of all (direct) dependencies after filtering.
+        Iterable builddependencies are flattened when not iterating.
+
+        :param build_only: only return build dependencies, discard others
+        """
+        return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']}
+
     def builddependencies(self):
         """
         Return a flat list of the parsed build dependencies
diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py
@@ -89,6 +89,8 @@
 # template values which are only generated dynamically
 TEMPLATE_NAMES_DYNAMIC = [
     ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
+    ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
+     "as specify by the --sysroot configuration option"),
     ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
     ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
      "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
@@ -197,6 +199,9 @@ def template_constant_dict(config, ignore=None, toolchain=None):
     # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
     template_values['arch'] = platform.uname()[4]
 
+    # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
+    template_values['sysroot'] = build_option('sysroot') or ''
+
     # step 1: add TEMPLATE_NAMES_EASYCONFIG
     for name in TEMPLATE_NAMES_EASYCONFIG:
         if name in ignore:
diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py
@@ -55,9 +55,11 @@ def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
         extra_args['universal_newlines'] = True
     stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE
     p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args)
-    out, _ = p.communicate()
+    out, err = p.communicate()
     if p.returncode != 0:
-        raise RuntimeError('Failed to %s: %s' % (action_desc, out))
+        if err:
+            err = "\nSTDERR:\n" + err
+        raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err))
     return out
 
 
@@ -171,20 +173,23 @@ def print_deps(package, verbose):
                               capture_stderr=False,
                               action_desc='Get missing dependencies'
                               )
+    excluded_dep = '(%s)' % os.path.basename(args.ec)
     missing_deps = [dep for dep in missing_dep_out.split('\n')
-                    if dep.startswith('*') and '(%s)' % args.ec not in dep
+                    if dep.startswith('*') and excluded_dep not in dep
                     ]
     if missing_deps:
         print('You need to install all modules on which %s depends first!' % args.ec)
         print('\n\t'.join(['Missing:'] + missing_deps))
         sys.exit(1)
 
+    # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir
+    ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec
     with temporary_directory() as tmp_dir:
         old_dir = os.getcwd()
         os.chdir(tmp_dir)
         if args.verbose:
             print('Running EasyBuild to get build environment')
-        run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment')
+        run_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment')
         os.chdir(old_dir)
 
         cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package)
diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py
@@ -50,7 +50,7 @@ class Gcc(Compiler):
 
     COMPILER_FAMILY = TC_CONSTANT_GCC
     COMPILER_UNIQUE_OPTS = {
-        'loop': (False, "Automatic loop parallellisation"),
+        'loop': (False, "Automatic loop parallelisation"),
         'f2c': (False, "Generate code compatible with f2c and f77"),
         'lto': (False, "Enable Link Time Optimization"),
     }
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
@@ -275,8 +275,8 @@ def basic_options(self):
             'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}),
             'rebuild': ("Rebuild software, even if module already exists (don't skip OS dependencies checks)",
                         None, 'store_true', False),
-            'robot': ("Enable dependency resolution, using easyconfigs in specified paths",
-                      'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}),
+            'robot': ("Enable dependency resolution, optionally consider additional paths to search for easyconfigs",
+                      'pathlist', 'store_or_None', [], 'r', {'metavar': '[PATH[%sPATH]]' % os.pathsep}),
             'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)",
                             'pathlist', 'add_flex', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}),
             'search-paths': ("Additional locations to consider in --search (next to --robot and --robot-paths paths)",
@@ -467,7 +467,7 @@ def override_options(self):
             'output-style': ("Control output style; auto implies using Rich if available to produce rich output, "
                              "with fallback to basic colored output",
                              'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES),
-            'parallel': ("Specify (maximum) level of parallellism used during build procedure",
+            'parallel': ("Specify (maximum) level of parallelism used during build procedure",
                          'int', 'store', None),
             'parallel-extensions-install': ("Install list of extensions in parallel (if supported)",
                                             None, 'store_true', False),
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
@@ -1205,7 +1205,7 @@ def get_default_parallelism():
             raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err)
 
     if maxpar is not None and maxpar < par:
-        _log.info("Limiting parallellism from %s to %s", par, maxpar)
+        _log.info("Limiting parallelism from %s to %s", par, maxpar)
         par = maxpar
 
     return par
diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py
@@ -92,7 +92,7 @@ def get_git_revision():
 
 def this_is_easybuild():
     """Standard starting message"""
-    top_version = max(FRAMEWORK_VERSION, EASYBLOCKS_VERSION)
+    top_version = max(FRAMEWORK_VERSION, LooseVersion(EASYBLOCKS_VERSION))
     msg = "This is EasyBuild %s (framework: %s, easyblocks: %s) on host %s."
     msg = msg % (top_version, FRAMEWORK_VERSION, EASYBLOCKS_VERSION, gethostname())
 
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
@@ -2097,7 +2097,7 @@ def test_extensions_sanity_check(self):
             eb.run_all_steps(True)
 
     def test_parallel(self):
-        """Test defining of parallellism."""
+        """Test defining of parallelism."""
         topdir = os.path.abspath(os.path.dirname(__file__))
         toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
         toytxt = read_file(toy_ec)
@@ -2114,7 +2114,7 @@ def test_parallel(self):
         os.close(handle)
         write_file(toy_ec3, toytxt + "\nparallel = False")
 
-        # default: parallellism is derived from # available cores + ulimit
+        # default: parallelism is derived from # available cores + ulimit
         test_eb = EasyBlock(EasyConfig(toy_ec))
         test_eb.check_readiness_step()
         self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0)
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
@@ -302,7 +302,9 @@ def test_dependency(self):
         self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3')
         self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3')
 
+        self.assertEqual(eb.dependency_names(), {'first', 'second', 'foo', 'bar'})
         # same tests for builddependencies
+        self.assertEqual(eb.dependency_names(build_only=True), {'first', 'second'})
         first = eb.builddependencies()[0]
         second = eb.builddependencies()[1]
 
@@ -355,6 +357,7 @@ def test_false_dep_version(self):
         self.assertEqual(len(deps), 2)
         self.assertEqual(deps[0]['name'], 'second_build')
         self.assertEqual(deps[1]['name'], 'first')
+        self.assertEqual(eb.dependency_names(), {'first', 'second_build'})
 
         # more realistic example: only filter dep for POWER
         self.contents = '\n'.join([
@@ -378,12 +381,14 @@ def test_false_dep_version(self):
             deps = eb.dependencies()
             self.assertEqual(len(deps), 1)
             self.assertEqual(deps[0]['name'], 'not_on_power')
+            self.assertEqual(eb.dependency_names(), {'not_on_power'})
 
         # only power, dependency gets filtered
         st.get_cpu_architecture = lambda: POWER
         eb = EasyConfig(self.eb_file)
         deps = eb.dependencies()
         self.assertEqual(deps, [])
+        self.assertEqual(eb.dependency_names(), set())
 
     def test_extra_options(self):
         """ extra_options should allow other variables to be stored """
@@ -1350,6 +1355,34 @@ def test_start_dir_template(self):
         self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt)
         self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt)
 
+    def test_sysroot_template(self):
+        """Test the %(sysroot)s template"""
+
+        test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"'
+        test_ec_txt += '\nbuildopts = "--some-opt=%(sysroot)s/"'
+        test_ec_txt += '\ninstallopts = "--some-opt=%(sysroot)s/"'
+        write_file(test_ec, test_ec_txt)
+
+        # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None)
+        ec = EasyConfig(test_ec)
+        self.assertEqual(ec['configopts'], "--some-opt=/")
+        self.assertEqual(ec['buildopts'], "--some-opt=/")
+        self.assertEqual(ec['installopts'], "--some-opt=/")
+
+        # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None)
+        # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist
+        update_build_option('sysroot', self.test_prefix)
+
+        ec = EasyConfig(test_ec)
+        self.assertEqual(ec['configopts'], "--some-opt=%s/" % self.test_prefix)
+        self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix)
+        self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix)
+
     def test_constant_doc(self):
         """test constant documentation"""
         doc = avail_easyconfig_constants()
@@ -1602,18 +1635,15 @@ def test_filter_deps(self):
         test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
         ec_file = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb')
         ec = EasyConfig(ec_file)
-        deps = sorted([dep['name'] for dep in ec.dependencies()])
-        self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'])
+        self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'})
 
         # test filtering multiple deps
         init_config(build_options={'filter_deps': ['FFTW', 'ScaLAPACK']})
-        deps = sorted([dep['name'] for dep in ec.dependencies()])
-        self.assertEqual(deps, ['GCC', 'OpenBLAS', 'OpenMPI'])
+        self.assertEqual(ec.dependency_names(), {'GCC', 'OpenBLAS', 'OpenMPI'})
 
         # test filtering of non-existing dep
         init_config(build_options={'filter_deps': ['zlib']})
-        deps = sorted([dep['name'] for dep in ec.dependencies()])
-        self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'])
+        self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'})
 
         # test parsing of value passed to --filter-deps
         opts = init_config(args=[])
@@ -1647,6 +1677,7 @@ def test_filter_deps(self):
         init_config(build_options=build_options)
         ec = EasyConfig(ec_file, validate=False)
         self.assertEqual(ec.dependencies(), [])
+        self.assertEqual(ec.dependency_names(), set())
 
     def test_replaced_easyconfig_parameters(self):
         """Test handling of replaced easyconfig parameters."""
@@ -1835,6 +1866,9 @@ def test_external_dependencies(self):
         }
         self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata)
 
+        # External module names are omitted
+        self.assertEqual(ec.dependency_names(), {'intel'})
+
         # provide file with partial metadata for some external modules;
         # metadata obtained from probing modules should be added to it...
         metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg')
@@ -1863,6 +1897,7 @@ def test_external_dependencies(self):
         deps = ec.dependencies()
 
         self.assertEqual(len(deps), 8)
+        self.assertEqual(ec.dependency_names(), {'intel'})
 
         for idx in [0, 1, 2, 6]:
             self.assertEqual(deps[idx]['external_module_metadata'], {})
@@ -3229,6 +3264,7 @@ def test_template_constant_dict(self):
             'nameletter': 'g',
             'nameletterlower': 'g',
             'parallel': None,
+            'sysroot': '',
             'toolchain_name': 'foss',
             'toolchain_version': '2018a',
             'version': '1.5',
@@ -3311,6 +3347,7 @@ def test_template_constant_dict(self):
             'pyminver': '7',
             'pyshortver': '3.7',
             'pyver': '3.7.2',
+            'sysroot': '',
             'version': '0.01',
             'version_major': '0',
             'version_major_minor': '0.01',
@@ -3375,6 +3412,7 @@ def test_template_constant_dict(self):
             'namelower': 'foo',
             'nameletter': 'f',
             'nameletterlower': 'f',
+            'sysroot': '',
             'version': '1.2.3',
             'version_major': '1',
             'version_major_minor': '1.2',
diff --git a/test/framework/github.py b/test/framework/github.py
diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ class Gcc(Compiler):`
`50`	`50`
`51`	`51`	`COMPILER_FAMILY = TC_CONSTANT_GCC`
`52`	`52`	`COMPILER_UNIQUE_OPTS = {`
`53`		`- 'loop': (False, "Automatic loop parallellisation"),`
	`53`	`+ 'loop': (False, "Automatic loop parallelisation"),`
`54`	`54`	`'f2c': (False, "Generate code compatible with f2c and f77"),`
`55`	`55`	`'lto': (False, "Enable Link Time Optimization"),`
`56`	`56`	`}`