Merge branch 'easybuilders:develop' into optarch-map

Author: Flamefire

RELEASE_NOTES

@@ -4,6 +4,28 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.2 (12 June 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - improve behavior when using extension which has 'nosource' enabled (#4506)
+  - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513)
+  - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553)
+  - add support for easyconfig parameter 'module_only' (#4537)
+- various bug fixes, including:
+  - fix typo in patch_step logging (#4505)
+  - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507)
+  - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512)
+  - fix resolved (template) values in case of failure  (#4532)
+  - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551)
+  - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552)
+- other changes:
+  - code cleanup in 'easyblock.py' (#4519)
+  - stop running unit tests on Python 3.5 (#4530)
+
+
 v4.9.1 (5 April 2024)
 ---------------------
 

easybuild/toolchains/linalg/libsci.py

@@ -65,13 +65,20 @@ def _get_software_root(self, name, required=True):
         """Get install prefix for specified software name; special treatment for Cray modules."""
         if name == 'cray-libsci':
             # Cray-provided LibSci module
-            env_var = 'CRAY_LIBSCI_PREFIX_DIR'
-            root = os.getenv(env_var, None)
+            root = None
+            # consider both $CRAY_LIBSCI_PREFIX_DIR and $CRAY_PE_LIBSCI_PREFIX_DIR,
+            # cfr. https://github.com/easybuilders/easybuild-framework/issues/4536
+            env_vars = ('CRAY_LIBSCI_PREFIX_DIR', 'CRAY_PE_LIBSCI_PREFIX_DIR')
+            for env_var in env_vars:
+                root = os.getenv(env_var, None)
+                if root is not None:
+                    self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
+                    break
+
             if root is None:
                 if required:
-                    raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var)
-            else:
-                self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root)
+                    env_vars_str = ', '.join('$' + e for e in env_vars)
+                    raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_vars_str)
         else:
             root = super(LibSci, self)._get_software_root(name, required=required)
 

easybuild/tools/github.py

@@ -589,13 +589,19 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi
             raise EasyBuildError("Couldn't find path to patched file %s", full_path)
 
     if github_repo == GITHUB_EASYCONFIGS_REPO:
-        ver = _get_version_for_repo(os.path.join(final_path, 'setup.py'))
+        ver_file = os.path.join(final_path, 'setup.py')
     elif github_repo == GITHUB_EASYBLOCKS_REPO:
-        ver = _get_version_for_repo(os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py'))
+        ver_file = os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py')
+    else:
+        raise EasyBuildError("Don't know how to determine version for repo %s", github_repo)
 
-    if different_major_versions(FRAMEWORK_VERSION, ver):
-        raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)",
-                             FRAMEWORK_VERSION, github_account, github_repo, pr, ver)
+    # take into account that the file we need to determine repo version may not be available,
+    # for example when a closed PR is used (since then we only download files patched by the PR)
+    if os.path.exists(ver_file):
+        ver = _get_version_for_repo(ver_file)
+        if different_major_versions(FRAMEWORK_VERSION, ver):
+            raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)",
+                                 FRAMEWORK_VERSION, github_account, github_repo, pr, ver)
 
     return files
 
@@ -643,6 +649,13 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
     if github_repo is None:
         github_repo = GITHUB_EASYCONFIGS_REPO
 
+    if github_repo == GITHUB_EASYCONFIGS_REPO:
+        easybuild_subdir = os.path.join('easybuild', 'easyconfigs')
+    elif github_repo == GITHUB_EASYBLOCKS_REPO:
+        easybuild_subdir = os.path.join('easybuild', 'easyblocks')
+    else:
+        raise EasyBuildError("Unknown repo: %s", github_repo)
+
     if path is None:
         if github_repo == GITHUB_EASYCONFIGS_REPO:
             extra_ec_paths = build_option('extra_ec_paths')
@@ -686,6 +699,12 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None,
     else:
         raise EasyBuildError("Unknown repo: %s" % github_repo)
 
+    # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path
+    mkdir(path, parents=True)
+    dirpath = os.path.join(repo_commit, easybuild_subdir)
+    for subdir in os.listdir(dirpath):
+        symlink(os.path.join(dirpath, subdir), os.path.join(path, subdir))
+
     # copy specified files to directory where they're expected to be found
     file_paths = []
     for file in files:

easybuild/tools/modules.py

@@ -37,6 +37,7 @@
 * Jens Timmerman (Ghent University)
 * David Brown (Pacific Northwest National Laboratory)
 """
+import glob
 import os
 import re
 import shlex
@@ -52,6 +53,7 @@
 from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
 from easybuild.tools.py2vs3 import subprocess_popen_text
 from easybuild.tools.run import run_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.utilities import get_subclasses, nub
 
 # software root/version environment variable name prefixes
@@ -1671,6 +1673,7 @@ def get_software_libdir(name, only_one=True, fs=None):
 
     Returns the library subdirectory, relative to software root.
     It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs.
+    If only_one is True and fs is None, select the one subdirectory with shared or static libraries, if possible.
 
     :param name: name of the software package
     :param only_one: indicates whether only one lib path is expected to be found
@@ -1703,6 +1706,16 @@ def get_software_libdir(name, only_one=True, fs=None):
             if len(res) == 1:
                 res = res[0]
             else:
+                if fs is None and len(res) == 2:
+                    # if both lib and lib64 were found, check if only one (exactly) has libraries;
+                    # this is needed for software with library archives in lib64 but other files/directories in lib
+                    lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]]
+                    has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res]
+                    if has_libs[0] and not has_libs[1]:
+                        return res[0]
+                    elif has_libs[1] and not has_libs[0]:
+                        return res[1]
+
                 raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s",
                                      name, root, ', '.join(res))
         return res

easybuild/tools/version.py

@@ -45,7 +45,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.9.2.dev0')
+VERSION = LooseVersion('4.9.3.dev0')
 UNKNOWN = 'UNKNOWN'
 UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
 

test/framework/modules.py

@@ -51,6 +51,7 @@
 from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version
 from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches
 from easybuild.tools.run import run_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
 
 
 # number of modules included for testing purposes
@@ -674,10 +675,23 @@ def test_get_software_root_version_libdir(self):
             os.environ.pop('EBROOT%s' % env_var_name)
             os.environ.pop('EBVERSION%s' % env_var_name)
 
-        # check expected result of get_software_libdir with multiple lib subdirs
+        # if only 'lib' has a library archive, use it
         root = os.path.join(tmpdir, name)
         mkdir(os.path.join(root, 'lib64'))
         os.environ['EBROOT%s' % env_var_name] = root
+        write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo')
+        self.assertEqual(get_software_libdir(name), 'lib')
+
+        remove_file(os.path.join(root, 'lib', 'libfoo.a'))
+
+        # also check vice versa with *shared* library in lib64
+        shlib_ext = get_shared_lib_ext()
+        write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo')
+        self.assertEqual(get_software_libdir(name), 'lib64')
+
+        remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext))
+
+        # check expected result of get_software_libdir with multiple lib subdirs
         self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name)
         self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64'])
 

test/framework/options.py

@@ -1927,7 +1927,7 @@ def test_github_from_pr(self):
 
             # make sure that *only* these modules are listed, no others
             regex = re.compile(r"^ \* \[.\] .*/(?P<filepath>.*) \(module: (?P<module>.*)\)$", re.M)
-            self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules))
+            self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
 
             pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424')
             regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M)
@@ -1962,12 +1962,12 @@ def test_github_from_pr(self):
 
             # make sure that *only* these modules are listed, no others
             regex = re.compile(r"^ \* \[.\] .*/(?P<filepath>.*) \(module: (?P<module>.*)\)$", re.M)
-            self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules))
+            self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
 
             for pr in ('12150', '12366'):
                 pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr)
                 regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M)
-                self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+                self.assertTrue(regex.search(outtxt), "Found pattern '%s' in: %s" % (regex.pattern, outtxt))
 
         except URLError as err:
             print("Ignoring URLError '%s' in test_from_pr" % err)
@@ -2144,6 +2144,50 @@ def test_from_commit(self):
             print("Ignoring URLError '%s' in test_from_commit" % err)
             shutil.rmtree(tmpdir)
 
+        easyblock_template = '\n'.join([
+            "from easybuild.framework.easyblock import EasyBlock",
+            "class %s(EasyBlock):",
+            "    pass",
+        ])
+
+        # create fake custom easyblock for CMake that is required by easyconfig used in test below
+        easyblock_file = os.path.join(self.test_prefix, 'easyblocks', 'cmake.py')
+        write_file(easyblock_file, easyblock_template % 'EB_CMake')
+
+        # also test with an easyconfig that requires additional easyconfigs to resolve dependencies,
+        # cfr. https://github.com/easybuilders/easybuild-framework/issues/4540;
+        # using commit that adds CMake-3.18.4.eb (which requires ncurses-6.2.eb),
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/13156
+        test_commit = '41eee3fe2e5102f52319481ca8dde16204dab590'
+        args = [
+            '--from-commit=%s' % test_commit,
+            '--dry-run',
+            '--tmpdir=%s' % tmpdir,
+            '--include-easyblocks=' + os.path.join(self.test_prefix, 'easyblocks', '*.py'),
+        ]
+        try:
+            outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)
+            modules = [
+                (tmpdir, 'ncurses/6.2'),
+                (tmpdir, 'CMake/3.18.4'),
+            ]
+            for path_prefix, module in modules:
+                ec_fn = "%s.eb" % '-'.join(module.split('/'))
+                path = '.*%s' % os.path.dirname(path_prefix)
+                regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M)
+                self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+
+            # make sure that *only* these modules are listed, no others
+            regex = re.compile(r"^ \* \[.\] .*/(?P<filepath>.*) \(module: (?P<module>.*)\)$", re.M)
+            self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules))
+
+            pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit)
+            regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M)
+            self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt))
+        except URLError as err:
+            print("Ignoring URLError '%s' in test_from_commit" % err)
+            shutil.rmtree(tmpdir)
+
     # must be run after test for --list-easyblocks, hence the '_xxx_'
     # cleaning up the imported easyblocks is quite difficult...
     def test_xxx_include_easyblocks_from_commit(self):