Merge branch 'develop' into 5.0.x

Author: boegel

easybuild/framework/easyblock.py

@@ -4410,7 +4410,9 @@ def build_and_install_one(ecdict, init_env):
         def ensure_writable_log_dir(log_dir):
             """Make sure we can write into the log dir"""
             if build_option('read_only_installdir'):
-                # temporarily re-enable write permissions for copying log/easyconfig to install dir
+                # temporarily re-enable write permissions for copying log/easyconfig to install dir,
+                # ensuring that we resolve symlinks
+                log_dir = os.path.realpath(log_dir)
                 if os.path.exists(log_dir):
                     adjust_permissions(log_dir, stat.S_IWUSR, add=True, recursive=True)
                 else:

easybuild/framework/easyconfig/templates.py

@@ -103,6 +103,8 @@
     'cuda_sm_comma_sep': 'Comma-separated list of sm_* values that correspond with CUDA compute capabilities',
     'cuda_sm_space_sep': 'Space-separated list of sm_* values that correspond with CUDA compute capabilities',
     'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)',
+    # can't be a boolean (True/False), must be a string value since it's a string template
+    'rpath_enabled': "String value indicating whether or not RPATH linking is used ('true' or 'false')",
     'software_commit': "Git commit id to use for the software as specified by --software-commit command line option",
     'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include"
                "as specify by the --sysroot configuration option",
@@ -298,6 +300,9 @@ def template_constant_dict(config, ignore=None, toolchain=None):
     # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value
     template_values['arch'] = platform.uname()[4]
 
+    # set 'rpath' template based on 'rpath' configuration option, using empty string as fallback
+    template_values['rpath_enabled'] = 'true' if build_option('rpath') else 'false'
+
     # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback
     template_values['sysroot'] = build_option('sysroot') or ''
 

easybuild/scripts/findPythonDeps.py

@@ -95,22 +95,35 @@ def get_dep_tree(package_spec, verbose):
 
 def find_deps(pkgs, dep_tree):
     """Recursively resolve dependencies of the given package(s) and return them"""
+    MAX_PACKAGES = 1000
     res = []
-    for orig_pkg in pkgs:
-        pkg = canonicalize_name(orig_pkg)
-        matching_entries = [entry for entry in dep_tree
-                            if pkg in (entry['package']['package_name'], entry['package']['key'])]
-        if not matching_entries:
+    next_pkgs = set(pkgs)
+    # Don't check any package multiple times to avoid infinite recursion
+    seen_pkgs = set()
+    count = 0
+    while next_pkgs:
+        cur_pkgs = next_pkgs - seen_pkgs
+        seen_pkgs.update(cur_pkgs)
+        next_pkgs = set()
+        for orig_pkg in cur_pkgs:
+            count += 1
+            if count > MAX_PACKAGES:
+                raise RuntimeError("Aborting after checking %s packages. Possibly cycle detected!" % MAX_PACKAGES)
+            pkg = canonicalize_name(orig_pkg)
             matching_entries = [entry for entry in dep_tree
-                                if orig_pkg in (entry['package']['package_name'], entry['package']['key'])]
-        if not matching_entries:
-            raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree))
-        if len(matching_entries) > 1:
-            raise RuntimeError("Found multiple installed packages for '%s' in %s" % (pkg, dep_tree))
-        entry = matching_entries[0]
-        res.append((entry['package']['package_name'], entry['package']['installed_version']))
-        deps = (dep['package_name'] for dep in entry['dependencies'])
-        res.extend(find_deps(deps, dep_tree))
+                                if pkg in (entry['package']['package_name'], entry['package']['key'])]
+            if not matching_entries:
+                matching_entries = [entry for entry in dep_tree
+                                    if orig_pkg in (entry['package']['package_name'], entry['package']['key'])]
+            if not matching_entries:
+                raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree))
+            if len(matching_entries) > 1:
+                raise RuntimeError("Found multiple installed packages for '%s' in %s" % (pkg, dep_tree))
+            entry = matching_entries[0]
+            res.append((entry['package']['package_name'], entry['package']['installed_version']))
+            # Add dependencies to list of packages to check next
+            # Could call this function recursively but that might exceed the max recursion depth
+            next_pkgs.update(dep['package_name'] for dep in entry['dependencies'])
     return res
 
 

easybuild/toolchains/compiler/gcc.py

@@ -85,7 +85,7 @@ class Gcc(Compiler):
         COMPILER_UNIQUE_OPTION_MAP['strict'] = []
         COMPILER_UNIQUE_OPTION_MAP['precise'] = []
         COMPILER_UNIQUE_OPTION_MAP['loose'] = ['fno-math-errno']
-        COMPILER_UNIQUE_OPTION_MAP['verloose'] = ['fno-math-errno']
+        COMPILER_UNIQUE_OPTION_MAP['veryloose'] = ['fno-math-errno']
 
     # used when 'optarch' toolchain option is enabled (and --optarch is not specified)
     COMPILER_OPTIMAL_ARCHITECTURE_OPTION = {

easybuild/tools/github.py

@@ -1097,11 +1097,16 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
     elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and all(file_info['new']):
         commit_msg = "adding easyblocks: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo'])
     else:
-        raise EasyBuildError(
-            "A meaningful commit message must be specified via --pr-commit-msg when "
-            "modifying/deleting files or targeting the framework repo.",
-            exit_code=EasyBuildExit.OPTION_ERROR
-        )
+        msg = ''
+        modified_files = [os.path.basename(p) for new, p in zip(file_info['new'], file_info['paths_in_repo'])
+                          if not new]
+        if modified_files:
+            msg += '\nModified: ' + ', '.join(modified_files)
+        if paths['files_to_delete']:
+            msg += '\nDeleted: ' + ', '.join(paths['files_to_delete'])
+        raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when "
+                             "modifying/deleting files or targeting the framework repo." + msg,
+                             exit_code=EasyBuildExit.OPTION_ERROR)
 
     # figure out to which software name patches relate, and copy them to the right place
     if paths['patch_files']:
@@ -1144,8 +1149,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_
         # only consider new easyconfig files for dependencies (not updated ones)
         for idx in range(len(all_dep_info['ecs'])):
             if all_dep_info['new'][idx]:
-                for key, values in dep_info.items():
-                    values.append(all_dep_info[key][idx])
+                for key, info in dep_info.items():
+                    info.append(all_dep_info[key][idx])
 
     # checkout target branch
     if pr_branch is None:
@@ -1788,7 +1793,7 @@ def post_pr_labels(pr, labels):
 
         pr_url = g.repos[pr_target_account][pr_target_repo].issues[pr]
         try:
-            status, data = pr_url.labels.post(body=labels)
+            status, _ = pr_url.labels.post(body=labels)
             if status == HTTP_STATUS_OK:
                 print_msg("Added labels %s to PR#%s" % (', '.join(labels), pr), log=_log, prefix=False)
                 return True
@@ -2104,13 +2109,11 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None):
                     patch = patch[0]
                 elif isinstance(patch, dict):
                     patch_info = {}
-                    for key in patch.keys():
-                        patch_info[key] = patch[key]
-                    if 'name' not in patch_info.keys():
-                        raise EasyBuildError(
-                            "Wrong patch spec '%s', when using a dict 'name' entry must be supplied", str(patch),
-                            exit_code=EasyBuildExit.EASYCONFIG_ERROR
-                        )
+                    for key, cur_patch in patch.items():
+                        patch_info[key] = cur_patch
+                    if 'name' not in patch_info:
+                        raise EasyBuildError(f"Wrong patch spec '{patch}', when using a dict 'name' entry must be supplied",
+                                             exit_code=EasyBuildExit.EASYCONFIG_ERROR)
                     patch = patch_info['name']
 
                 if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path),

test/framework/easyconfig.py

@@ -1442,6 +1442,30 @@ def test_start_dir_template(self):
         self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt)
         self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt)
 
+    def test_rpath_template(self):
+        """Test the %(rpath)s template"""
+        test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += "configopts = '--with-rpath=%(rpath_enabled)s'"
+        write_file(test_ec, test_ec_txt)
+
+        ec = EasyConfig(test_ec)
+        expected = '--with-rpath=true' if get_os_name() == 'Linux' else '--with-rpath=false'
+        self.assertEqual(ec['configopts'], expected)
+
+        # force True
+        update_build_option('rpath', True)
+        ec = EasyConfig(test_ec)
+        self.assertEqual(ec['configopts'], "--with-rpath=true")
+
+        # force False
+        update_build_option('rpath', False)
+        ec = EasyConfig(test_ec)
+        self.assertEqual(ec['configopts'], "--with-rpath=false")
+
     def test_sysroot_template(self):
         """Test the %(sysroot)s template"""
 
@@ -3483,6 +3507,8 @@ def test_template_constant_dict(self):
 
         arch_regex = re.compile('^[a-z0-9_]+$')
 
+        rpath = 'true' if get_os_name() == 'Linux' else 'false'
+
         expected = {
             'bitbucket_account': 'gzip',
             'github_account': 'gzip',
@@ -3492,6 +3518,7 @@ def test_template_constant_dict(self):
             'nameletter': 'g',
             'nameletterlower': 'g',
             'parallel': None,
+            'rpath_enabled': rpath,
             'software_commit': '',
             'sysroot': '',
             'toolchain_name': 'foss',
@@ -3576,6 +3603,7 @@ def test_template_constant_dict(self):
             'pyminver': '7',
             'pyshortver': '3.7',
             'pyver': '3.7.2',
+            'rpath_enabled': rpath,
             'software_commit': '',
             'sysroot': '',
             'version': '0.01',
@@ -3642,6 +3670,7 @@ def test_template_constant_dict(self):
             'namelower': 'foo',
             'nameletter': 'f',
             'nameletterlower': 'f',
+            'rpath_enabled': rpath,
             'software_commit': '',
             'sysroot': '',
             'version': '1.2.3',

test/framework/options.py

@@ -4708,7 +4708,7 @@ def test_github_new_update_pr(self):
             '--git-working-dirs-path=%s' % git_working_dir,
             ':bzip2-1.0.6.eb',
         ])
-        error_msg = "A meaningful commit message must be specified via --pr-commit-msg"
+        error_msg = "A meaningful commit message must be specified via --pr-commit-msg.*\nDeleted: bzip2-1.0.6.eb"
 
         self.mock_stdout(True)
         self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True, testing=False)
@@ -4787,7 +4787,8 @@ def test_github_new_update_pr(self):
             gcc_ec,
             '-D',
         ]
-        error_msg = "A meaningful commit message must be specified via --pr-commit-msg"
+        error_msg = "A meaningful commit message must be specified via --pr-commit-msg.*\n"
+        error_msg += "Modified: " + os.path.basename(gcc_ec)
         self.mock_stdout(True)
         self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True)
         self.mock_stdout(False)