Add a build option to trigger ignoring versionsuffixes when updating deps

Author: Alan O'Cais

easybuild/framework/easyconfig/tweak.py

@@ -124,6 +124,10 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
             pruned_build_specs = copy.copy(build_specs)
 
             update_dependencies = pruned_build_specs.pop('update_deps', None)
+            ignore_versionsuffixes = pruned_build_specs.pop('ignore_versionsuffixes', None)
+            if ignore_versionsuffixes and not update_dependencies:
+                print_warning("--try-ignore-versionsuffixes is ignored if --try-update-deps is not True")
+                ignore_versionsuffixes = False
             if 'toolchain' in pruned_build_specs:
                 target_toolchain = pruned_build_specs.pop('toolchain')
                 pruned_build_specs.pop('toolchain_name', '')
@@ -195,7 +199,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                     new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
                                                                         targetdir=tweaked_ecs_path,
                                                                         update_build_specs=pruned_build_specs,
-                                                                        update_dep_versions=update_dependencies)
+                                                                        update_dep_versions=update_dependencies,
+                                                                        ignore_versionsuffixes=ignore_versionsuffixes)
                     # Need to update the toolchain in the build_specs to match the toolchain mapping
                     keys = verification_build_specs.keys()
                     if 'toolchain_name' in keys:
@@ -217,7 +222,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                     # Note pruned_build_specs are not passed down for dependencies
                     map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
                                                           targetdir=tweaked_ecs_deps_path,
-                                                          update_dep_versions=update_dependencies)
+                                                          update_dep_versions=update_dependencies,
+                                                          ignore_versionsuffixes=ignore_versionsuffixes)
             else:
                 tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
 
@@ -939,7 +945,7 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain):
 
 
 def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_build_specs=None,
-                                          update_dep_versions=False):
+                                          update_dep_versions=False, ignore_versionsuffixes=False):
     """
     Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
 
@@ -1033,8 +1039,12 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
             elif update_dep_versions:
                 # search for available updates for this dependency:
                 # first get highest version candidate paths for this (include search through subtoolchains)
-                potential_version_mappings = find_potential_version_mappings(dep, toolchain_mapping,
-                                                                             versionsuffix_mapping=versonsuffix_mapping)
+                potential_version_mappings = find_potential_version_mappings(
+                    dep,
+                    toolchain_mapping,
+                    versionsuffix_mapping=versonsuffix_mapping,
+                    ignore_versionsuffixes=ignore_versionsuffixes
+                )
                 # only highest version match is retained by default in potential_version_mappings,
                 # compare that version to the original version and replace if appropriate (upgrades only).
                 if potential_version_mappings:
@@ -1091,7 +1101,7 @@ def list_deps_versionsuffixes(ec_spec):
 
 
 def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True,
-                                    ignore_versionsuffix=False):
+                                    ignore_versionsuffixes=False):
     """
     Find potential version mapping for a dependency in a new hierarchy
 
@@ -1207,7 +1217,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
 
     exclude_alternate_versionsuffixes = False
     if ignored_versionsuffix_greater:
-        if ignore_versionsuffix:
+        if ignore_versionsuffixes:
             highest_version = highest_version_ignoring_versionsuffix
         else:
             print_warning(

easybuild/tools/options.py

@@ -320,6 +320,8 @@ def software_options(self):
         opts['try-update-deps'] = ("Try to update versions of the dependencies of an easyconfig based on what is "
                                    "available in the robot path",
                                    None, 'store_true', False)
+        opts['try-ignore-versionsuffixes'] = ("Ignore versionsuffix differences when --try-update-deps is used",
+                                              None, 'store_true', False)
 
         self.log.debug("software_options: descr %s opts %s" % (descr, opts))
         self.add_group_parser(opts, descr)
@@ -1490,7 +1492,8 @@ def process_software_build_specs(options):
         'version': options.try_software_version,
         'toolchain_name': options.try_toolchain_name,
         'toolchain_version': options.try_toolchain_version,
-        'update_deps': options.try_update_deps
+        'update_deps': options.try_update_deps,
+        'ignore_versionsuffixes': options.try_ignore_versionsuffixes,
     }
 
     # process easy options

test/framework/options.py

@@ -1277,6 +1277,57 @@ def test_try_update_deps(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt))
 
+        # construct another toy easyconfig that is well suited for testing ignoring versionsuffix
+        test_ectxt = '\n'.join([
+            "easyblock = 'ConfigureMake'",
+            '',
+            "name = 'test'",
+            "version = '1.2.3'",
+            ''
+            "homepage = 'https://test.org'",
+            "description = 'this is just a test'",
+            '',
+            "toolchain = {'name': 'GCC', 'version': '4.8.2'}",
+            '',
+            "dependencies = [('OpenBLAS', '0.2.8', '-LAPACK-3.4.2')]",
+        ])
+        write_file(test_ec, test_ectxt)
+        self.mock_stderr(True)
+        outtxt = self.eb_main(args, raise_error=True, do_build=True)
+        errtxt = self.get_stderr()
+        warning_stub = "\nWARNING: There may be newer version(s) of dep 'OpenBLAS' available with a different " \
+                       "versionsuffix to '-LAPACK-3.4.2'"
+        self.mock_stderr(False)
+        self.assertTrue(warning_stub in errtxt)
+        patterns = [
+            # toolchain got updated
+            r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$",
+            # no version update for OpenBLAS (because there's no corresponding ec using GCC/6.4.0-2.28 (sub)toolchain)
+            r"^ \* \[ \] .*/tweaked_dep_easyconfigs/OpenBLAS-0.2.8-GCC-6.4.0-2.28-LAPACK-3.4.2.eb "
+            r"\(module: OpenBLAS/0.2.8-GCC-6.4.0-2.28-LAPACK-3.4.2\)$",
+            # also generated easyconfig for test/1.2.3 with expected toolchain
+            r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt))
+
+        # Now verify that we can ignore versionsuffixes
+        args.append('--try-ignore-versionsuffixes')
+        outtxt = self.eb_main(args, raise_error=True, do_build=True)
+        patterns = [
+            # toolchain got updated
+            r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$",
+            # no version update for OpenBLAS (because there's no corresponding ec using GCC/6.4.0-2.28 (sub)toolchain)
+            r"^ \* \[x\] .*/test_ecs/o/OpenBLAS/OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb "
+            r"\(module: OpenBLAS/0.2.20-GCC-6.4.0-2.28\)$",
+            # also generated easyconfig for test/1.2.3 with expected toolchain
+            r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt))
+
     def test_dry_run_hierarchical(self):
         """Test dry run using a hierarchical module naming scheme."""
         fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')

test/framework/tweak.py

@@ -405,7 +405,7 @@ def test_find_potential_version_mappings(self):
         self.mock_stderr(False)
         self.assertTrue(errtxt.startswith(warning_stub))
         self.assertEqual(len(potential_versions), 0)
-        potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffix=True)
+        potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffixes=True)
         self.assertEqual(len(potential_versions), 1)
         expected = {
             'path': os.path.join(test_easyconfigs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'),