Merge pull request #3111 from boegel/pick_dep_version

take into account that dependency version could be a dict rather than a string value in template_constant_dict

Author: migueldiascosta

easybuild/framework/easyconfig/easyconfig.py

@@ -68,7 +68,7 @@
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
-from easybuild.tools.systemtools import check_os_dependency, get_cpu_architecture
+from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
 from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
@@ -1231,31 +1231,6 @@ def get_parsed_multi_deps(self):
 
         return multi_deps
 
-    def find_dep_version_match(self, dep_version):
-        """Identify the correct version for this system from the choices provided. This returns the version to use."""
-        if isinstance(dep_version, string_type):
-            self.log.debug("Version is already a string ('%s'), OK", dep_version)
-            return dep_version
-        elif dep_version is None:
-            self.log.debug("Version is None, OK")
-            return None
-        elif isinstance(dep_version, dict):
-            # figure out matches based on dict keys (after splitting on '=')
-            my_arch_key = 'arch=%s' % get_cpu_architecture()
-            arch_keys = [x for x in dep_version.keys() if x.startswith('arch=')]
-            other_keys = [x for x in dep_version.keys() if x not in arch_keys]
-            if other_keys:
-                raise EasyBuildError("Unexpected keys in version: %s. Only 'arch=' keys are supported", other_keys)
-            if arch_keys:
-                if my_arch_key in dep_version:
-                    ver = dep_version[my_arch_key]
-                    self.log.info("Version selected from %s using key %s: %s", dep_version, my_arch_key, ver)
-                    return ver
-                else:
-                    raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, my_arch_key)
-
-        raise EasyBuildError("Unknown value type for version: %s", dep_version)
-
     # private method
     def _parse_dependency(self, dep, hidden=False, build_only=False):
         """
@@ -1337,7 +1312,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
             raise EasyBuildError("Dependency %s of unsupported type: %s", dep, type(dep))
 
         # Find the version to use on this system
-        dependency['version'] = self.find_dep_version_match(dependency['version'])
+        dependency['version'] = pick_dep_version(dependency['version'])
 
         if dependency['external_module']:
             # check whether the external module is hidden

easybuild/framework/easyconfig/templates.py

@@ -38,7 +38,7 @@
 from easybuild.base import fancylogger
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.systemtools import get_shared_lib_ext
+from easybuild.tools.systemtools import get_shared_lib_ext, pick_dep_version
 
 
 _log = fancylogger.getLogger('easyconfig.templates', fname=False)
@@ -228,6 +228,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None):
                 raise EasyBuildError("Unexpected type for dependency: %s", dep)
 
             if isinstance(dep_name, string_type) and dep_name.lower() == name.lower():
+                dep_version = pick_dep_version(dep_version)
                 template_values['%sver' % pref] = dep_version
                 dep_version_parts = dep_version.split('.')
                 template_values['%smajver' % pref] = dep_version_parts[0]

easybuild/tools/systemtools.py

@@ -45,6 +45,7 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import is_readable, read_file, which
+from easybuild.tools.py2vs3 import string_type
 from easybuild.tools.run import run_cmd
 
 
@@ -864,3 +865,40 @@ def check_python_version():
         raise EasyBuildError("EasyBuild is not compatible (yet) with Python %s", python_ver)
 
     return (python_maj_ver, python_min_ver)
+
+
+def pick_dep_version(dep_version):
+    """
+    Pick the correct dependency version to use for this system.
+    Input can either be:
+    * a string value (or None)
+    * a dict with options to choose from
+
+    Return value is the version to use.
+    """
+    if isinstance(dep_version, string_type):
+        _log.debug("Version is already a string ('%s'), OK", dep_version)
+        result = dep_version
+
+    elif dep_version is None:
+        _log.debug("Version is None, OK")
+        result = None
+
+    elif isinstance(dep_version, dict):
+        # figure out matches based on dict keys (after splitting on '=')
+        my_arch_key = 'arch=%s' % get_cpu_architecture()
+        arch_keys = [x for x in dep_version.keys() if x.startswith('arch=')]
+        other_keys = [x for x in dep_version.keys() if x not in arch_keys]
+        if other_keys:
+            raise EasyBuildError("Unexpected keys in version: %s. Only 'arch=' keys are supported", other_keys)
+        if arch_keys:
+            if my_arch_key in dep_version:
+                result = dep_version[my_arch_key]
+                _log.info("Version selected from %s using key %s: %s", dep_version, my_arch_key, result)
+            else:
+                raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, my_arch_key)
+
+    else:
+        raise EasyBuildError("Unknown value type for version: %s", dep_version)
+
+    return result

test/framework/easyconfig.py

@@ -52,7 +52,7 @@
 from easybuild.framework.easyconfig.easyconfig import letter_dir_for, process_easyconfig, resolve_template
 from easybuild.framework.easyconfig.easyconfig import triage_easyconfig_params, verify_easyconfig_filename
 from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3
-from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
+from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig
 from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str
 from easybuild.framework.easyconfig.style import check_easyconfigs_style
 from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph
@@ -2560,25 +2560,48 @@ def test_template_constant_dict(self):
         expected['parallel'] = 42
         self.assertEqual(res, expected)
 
-        ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb'))
+        toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-deps.eb')
+        toy_ec_txt = read_file(toy_ec)
+
         # fiddle with version to check version_minor template ('0' should be retained)
-        ec['version'] = '0.01'
+        toy_ec_txt = re.sub('version = .*', 'version = "0.01"', toy_ec_txt)
+
+        my_arch = st.get_cpu_architecture()
+
+        # add Java dep with version specified using a dict value
+        toy_ec_txt += '\n'.join([
+            "dependencies += [",
+            "  ('Python', '3.7.2'),"
+            "  ('Java', {",
+            "    'arch=%s': '1.8.0_221'," % my_arch,
+            "    'arch=fooarch': '1.8.0-foo',",
+            "  })",
+            "]",
+        ])
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, toy_ec_txt)
+
+        # only perform shallow/quick parse (as is done in list_software function)
+        ec = EasyConfigParser(filename=test_ec).get_config_dict()
 
         expected = {
-            'bitbucket_account': 'toy',
-            'github_account': 'toy',
+            'javamajver': '1',
+            'javashortver': '1.8',
+            'javaver': '1.8.0_221',
             'name': 'toy',
             'namelower': 'toy',
             'nameletter': 't',
             'toolchain_name': 'system',
             'toolchain_version': 'system',
             'nameletterlower': 't',
-            'parallel': None,
+            'pymajver': '3',
+            'pyshortver': '3.7',
+            'pyver': '3.7.2',
             'version': '0.01',
             'version_major': '0',
             'version_major_minor': '0.01',
             'version_minor': '01',
-            'versionprefix': '',
             'versionsuffix': '-deps',
         }
         res = template_constant_dict(ec)

test/framework/systemtools.py

@@ -43,7 +43,7 @@
 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN
 from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL
 from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP
-from easybuild.tools.systemtools import check_python_version
+from easybuild.tools.systemtools import check_python_version, pick_dep_version
 from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_architecture, get_cpu_family
 from easybuild.tools.systemtools import get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor
 from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version
@@ -837,6 +837,26 @@ def mock_python_ver(py_maj_ver, py_min_ver):
         if sys.version_info[:2] == (2, 6):
             self.assertTrue(stderr.startswith(py26_depr_warning))
 
+    def test_pick_dep_version(self):
+        """Test pick_dep_version function."""
+
+        self.assertEqual(pick_dep_version(None), None)
+        self.assertEqual(pick_dep_version('1.2.3'), '1.2.3')
+
+        dep_ver_dict = {
+            'arch=x86_64': '1.2.3-amd64',
+            'arch=POWER': '1.2.3-ppc64le',
+        }
+
+        st.get_cpu_architecture = lambda: X86_64
+        self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-amd64')
+
+        st.get_cpu_architecture = lambda: POWER
+        self.assertEqual(pick_dep_version(dep_ver_dict), '1.2.3-ppc64le')
+
+        error_pattern = "Unknown value type for version"
+        self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, ('1.2.3', '4.5.6'))
+
 
 def suite():
     """ returns all the testcases in this module """