Merge pull request #3353 from ocaisa/allow_ignoring_versionsuffix

Allow ignoring versionsuffix in try-update-deps

Author: boegel

easybuild/framework/easyconfig/tweak.py

@@ -64,10 +64,8 @@
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES
 from easybuild.tools.utilities import flatten, nub, quote_str
 
-
 _log = fancylogger.getLogger('easyconfig.tweak', fname=False)
 
-
 EASYCONFIG_TEMPLATE = "TEMPLATE"
 
 
@@ -126,6 +124,10 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
             pruned_build_specs = copy.copy(build_specs)
 
             update_dependencies = pruned_build_specs.pop('update_deps', None)
+            ignore_versionsuffixes = pruned_build_specs.pop('ignore_versionsuffixes', None)
+            if ignore_versionsuffixes and not update_dependencies:
+                print_warning("--try-ignore-versionsuffixes is ignored if --try-update-deps is not True")
+                ignore_versionsuffixes = False
             if 'toolchain' in pruned_build_specs:
                 target_toolchain = pruned_build_specs.pop('toolchain')
                 pruned_build_specs.pop('toolchain_name', '')
@@ -197,7 +199,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                     new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
                                                                         targetdir=tweaked_ecs_path,
                                                                         update_build_specs=pruned_build_specs,
-                                                                        update_dep_versions=update_dependencies)
+                                                                        update_dep_versions=update_dependencies,
+                                                                        ignore_versionsuffixes=ignore_versionsuffixes)
                     # Need to update the toolchain in the build_specs to match the toolchain mapping
                     keys = verification_build_specs.keys()
                     if 'toolchain_name' in keys:
@@ -219,7 +222,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                     # Note pruned_build_specs are not passed down for dependencies
                     map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
                                                           targetdir=tweaked_ecs_deps_path,
-                                                          update_dep_versions=update_dependencies)
+                                                          update_dep_versions=update_dependencies,
+                                                          ignore_versionsuffixes=ignore_versionsuffixes)
             else:
                 tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
 
@@ -277,6 +281,7 @@ def tweak_one(orig_ec, tweaked_ec, tweaks, targetdir=None):
 
         class TcDict(dict):
             """A special dict class that represents trivial toolchains properly."""
+
             def __repr__(self):
                 return "{'name': '%(name)s', 'version': '%(version)s'}" % self
 
@@ -899,7 +904,7 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp
                 'versionsuffix': versionsuffix or '',
             }
             # See what this dep would be mapped to
-            version_matches = find_potential_version_mappings(software_as_dep, toolchain_mapping)
+            version_matches = find_potential_version_mappings(software_as_dep, toolchain_mapping, quiet=True)
             if version_matches:
                 target_version = version_matches[0]['version']
                 if LooseVersion(target_version) > LooseVersion(version):
@@ -940,7 +945,7 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain):
 
 
 def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_build_specs=None,
-                                          update_dep_versions=False):
+                                          update_dep_versions=False, ignore_versionsuffixes=False):
     """
     Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
 
@@ -961,6 +966,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
     if update_dep_versions and (list_deps_versionsuffixes(ec_spec) or parsed_ec['versionsuffix']):
         # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8`
         versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['toolchain'], toolchain_mapping)
+        versonsuffix_mapping.update(map_common_versionsuffixes('Perl', parsed_ec['toolchain'], toolchain_mapping))
 
     if update_build_specs is not None:
         if 'version' in update_build_specs:
@@ -1033,21 +1039,25 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
             elif update_dep_versions:
                 # search for available updates for this dependency:
                 # first get highest version candidate paths for this (include search through subtoolchains)
-                potential_version_mappings = find_potential_version_mappings(dep, toolchain_mapping,
-                                                                             versionsuffix_mapping=versonsuffix_mapping)
+                potential_version_mappings = find_potential_version_mappings(
+                    dep,
+                    toolchain_mapping,
+                    versionsuffix_mapping=versonsuffix_mapping,
+                    ignore_versionsuffixes=ignore_versionsuffixes
+                )
                 # only highest version match is retained by default in potential_version_mappings,
                 # compare that version to the original version and replace if appropriate (upgrades only).
                 if potential_version_mappings:
                     highest_version_match = potential_version_mappings[0]['version']
+                    highest_versionsuffix_match = potential_version_mappings[0]['versionsuffix']
                     if LooseVersion(highest_version_match) > LooseVersion(dep['version']):
                         _log.info("Updating version of %s dependency from %s to %s", dep['name'], dep['version'],
                                   highest_version_match)
                         _log.info("Depending on your configuration, this will be resolved with one of the following "
                                   "easyconfigs: \n%s", '\n'.join(cand['path'] for cand in potential_version_mappings))
                         orig_dep['version'] = highest_version_match
-                        if orig_dep['versionsuffix'] in versonsuffix_mapping:
-                            dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']]
-                            orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']]
+                        dep['versionsuffix'] = highest_versionsuffix_match
+                        orig_dep['versionsuffix'] = highest_versionsuffix_match
                         dep_changed = True
 
             if dep_changed:
@@ -1090,7 +1100,8 @@ def list_deps_versionsuffixes(ec_spec):
     return list(set(versionsuffix_list))
 
 
-def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True):
+def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True,
+                                    ignore_versionsuffixes=False, quiet=False):
     """
     Find potential version mapping for a dependency in a new hierarchy
 
@@ -1139,7 +1150,9 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
     if len(version_components) > 1:  # Have at least major.minor
         candidate_ver_list.append(r'%s\..*' % major_version)
     candidate_ver_list.append(r'.*')  # Include a major version search
-    potential_version_mappings, highest_version = [], None
+    potential_version_mappings = []
+    highest_version = None
+    highest_version_ignoring_versionsuffix = None
 
     for candidate_ver in candidate_ver_list:
 
@@ -1152,7 +1165,8 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
                     toolchain_suffix = ''
                 else:
                     toolchain_suffix = '-%s-%s' % (toolchain['name'], toolchain['version'])
-                full_versionsuffix = toolchain_suffix + versionsuffix + EB_FORMAT_EXTENSION
+                # Search for any version suffix but only use what we are allowed to
+                full_versionsuffix = toolchain_suffix + r'.*' + EB_FORMAT_EXTENSION
                 depver = '^' + prefix_to_version + candidate_ver + full_versionsuffix
                 cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False,
                                                 case_sensitive=True)
@@ -1178,14 +1192,46 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
 
                 # add what is left to the possibilities
                 for path in cand_paths:
-                    version = fetch_parameters_from_easyconfig(read_file(path), ['version'])[0]
+                    version, newversionsuffix = fetch_parameters_from_easyconfig(read_file(path), ['version',
+                                                                                                   'versionsuffix'])
+                    if not newversionsuffix:
+                        newversionsuffix = ''
                     if version:
-                        if highest_version is None or LooseVersion(version) > LooseVersion(highest_version):
-                            highest_version = version
+                        if versionsuffix == newversionsuffix:
+                            if highest_version is None or LooseVersion(version) > LooseVersion(highest_version):
+                                highest_version = version
+                        else:
+                            if highest_version_ignoring_versionsuffix is None or \
+                                    LooseVersion(version) > LooseVersion(highest_version_ignoring_versionsuffix):
+                                highest_version_ignoring_versionsuffix = version
                     else:
                         raise EasyBuildError("Failed to determine version from contents of %s", path)
 
-                    potential_version_mappings.append({'path': path, 'toolchain': toolchain, 'version': version})
+                    potential_version_mappings.append({'path': path, 'toolchain': toolchain, 'version': version,
+                                                       'versionsuffix': newversionsuffix})
+
+    ignored_versionsuffix_greater = \
+        highest_version_ignoring_versionsuffix is not None and highest_version is None or \
+        (highest_version_ignoring_versionsuffix is not None and highest_version is not None and
+         LooseVersion(highest_version_ignoring_versionsuffix) > LooseVersion(highest_version))
+
+    exclude_alternate_versionsuffixes = False
+    if ignored_versionsuffix_greater:
+        if ignore_versionsuffixes:
+            highest_version = highest_version_ignoring_versionsuffix
+        else:
+            if not quiet:
+                print_warning(
+                    "There may be newer version(s) of dep '%s' available with a different versionsuffix to '%s': %s",
+                    dep['name'], versionsuffix, [d['path'] for d in potential_version_mappings if
+                                                 d['version'] == highest_version_ignoring_versionsuffix])
+            # exclude candidates with a different versionsuffix
+            exclude_alternate_versionsuffixes = True
+    else:
+        # If the other version suffixes are not greater, then just ignore them
+        exclude_alternate_versionsuffixes = True
+    if exclude_alternate_versionsuffixes:
+        potential_version_mappings = [d for d in potential_version_mappings if d['versionsuffix'] == versionsuffix]
 
     if highest_versions_only and highest_version is not None:
         potential_version_mappings = [d for d in potential_version_mappings if d['version'] == highest_version]

easybuild/tools/options.py

@@ -320,6 +320,8 @@ def software_options(self):
         opts['try-update-deps'] = ("Try to update versions of the dependencies of an easyconfig based on what is "
                                    "available in the robot path",
                                    None, 'store_true', False)
+        opts['try-ignore-versionsuffixes'] = ("Ignore versionsuffix differences when --try-update-deps is used",
+                                              None, 'store_true', False)
 
         self.log.debug("software_options: descr %s opts %s" % (descr, opts))
         self.add_group_parser(opts, descr)
@@ -1490,7 +1492,8 @@ def process_software_build_specs(options):
         'version': options.try_software_version,
         'toolchain_name': options.try_toolchain_name,
         'toolchain_version': options.try_toolchain_version,
-        'update_deps': options.try_update_deps
+        'update_deps': options.try_update_deps,
+        'ignore_versionsuffixes': options.try_ignore_versionsuffixes,
     }
 
     # process easy options

test/framework/options.py

@@ -1315,6 +1315,57 @@ def test_try_update_deps(self):
             regex = re.compile(pattern, re.M)
             self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt))
 
+        # construct another toy easyconfig that is well suited for testing ignoring versionsuffix
+        test_ectxt = '\n'.join([
+            "easyblock = 'ConfigureMake'",
+            '',
+            "name = 'test'",
+            "version = '1.2.3'",
+            ''
+            "homepage = 'https://test.org'",
+            "description = 'this is just a test'",
+            '',
+            "toolchain = {'name': 'GCC', 'version': '4.8.2'}",
+            '',
+            "dependencies = [('OpenBLAS', '0.2.8', '-LAPACK-3.4.2')]",
+        ])
+        write_file(test_ec, test_ectxt)
+        self.mock_stderr(True)
+        outtxt = self.eb_main(args, raise_error=True, do_build=True)
+        errtxt = self.get_stderr()
+        warning_stub = "\nWARNING: There may be newer version(s) of dep 'OpenBLAS' available with a different " \
+                       "versionsuffix to '-LAPACK-3.4.2'"
+        self.mock_stderr(False)
+        self.assertTrue(warning_stub in errtxt)
+        patterns = [
+            # toolchain got updated
+            r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$",
+            # no version update for OpenBLAS (because there's no corresponding ec using GCC/6.4.0-2.28 (sub)toolchain)
+            r"^ \* \[ \] .*/tweaked_dep_easyconfigs/OpenBLAS-0.2.8-GCC-6.4.0-2.28-LAPACK-3.4.2.eb "
+            r"\(module: OpenBLAS/0.2.8-GCC-6.4.0-2.28-LAPACK-3.4.2\)$",
+            # also generated easyconfig for test/1.2.3 with expected toolchain
+            r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt))
+
+        # Now verify that we can ignore versionsuffixes
+        args.append('--try-ignore-versionsuffixes')
+        outtxt = self.eb_main(args, raise_error=True, do_build=True)
+        patterns = [
+            # toolchain got updated
+            r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$",
+            # no version update for OpenBLAS (because there's no corresponding ec using GCC/6.4.0-2.28 (sub)toolchain)
+            r"^ \* \[x\] .*/test_ecs/o/OpenBLAS/OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb "
+            r"\(module: OpenBLAS/0.2.20-GCC-6.4.0-2.28\)$",
+            # also generated easyconfig for test/1.2.3 with expected toolchain
+            r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt))
+
     def test_dry_run_hierarchical(self):
         """Test dry run using a hierarchical module naming scheme."""
         fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')

test/framework/tweak.py

@@ -378,6 +378,40 @@ def test_find_potential_version_mappings(self):
             'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'),
             'toolchain': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'},
             'version': '1.6',
+            'versionsuffix': '',
+        }
+        self.assertEqual(potential_versions[0], expected)
+
+        # Test that we can override respecting the versionsuffix
+
+        # Create toolchain mapping for OpenBLAS
+        gcc_4_tc = {'name': 'GCC', 'version': '4.8.2'}
+        gcc_6_tc = {'name': 'GCC', 'version': '6.4.0-2.28'}
+        tc_mapping = map_toolchain_hierarchies(gcc_4_tc, gcc_6_tc, self.modtool)
+        # Create a dep with the necessary params (including versionsuffix)
+        openblas_dep = {
+            'toolchain': {'version': '4.8.2', 'name': 'GCC'},
+            'name': 'OpenBLAS',
+            'system': False,
+            'versionsuffix': '-LAPACK-3.4.2',
+            'version': '0.2.8'
+        }
+
+        self.mock_stderr(True)
+        potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping)
+        errtxt = self.get_stderr()
+        warning_stub = "\nWARNING: There may be newer version(s) of dep 'OpenBLAS' available with a different " \
+                       "versionsuffix to '-LAPACK-3.4.2'"
+        self.mock_stderr(False)
+        self.assertTrue(errtxt.startswith(warning_stub))
+        self.assertEqual(len(potential_versions), 0)
+        potential_versions = find_potential_version_mappings(openblas_dep, tc_mapping, ignore_versionsuffixes=True)
+        self.assertEqual(len(potential_versions), 1)
+        expected = {
+            'path': os.path.join(test_easyconfigs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.20-GCC-6.4.0-2.28.eb'),
+            'toolchain': {'version': '6.4.0-2.28', 'name': 'GCC'},
+            'version': '0.2.20',
+            'versionsuffix': '',
         }
         self.assertEqual(potential_versions[0], expected)