Merge pull request #3845 from Flamefire/alternative_sanity_check_paths

Add option to make sanity_check_paths arch dependent

Author: akesandgren

easybuild/framework/easyblock.py

@@ -100,7 +100,7 @@
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw
-from easybuild.tools.systemtools import get_shared_lib_ext, use_group
+from easybuild.tools.systemtools import get_shared_lib_ext, pick_system_specific_value, use_group
 from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
@@ -3327,6 +3327,15 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
             error_msg += "values should be lists (at least one non-empty)."
             raise EasyBuildError(error_msg % ', '.join("'%s'" % k for k in known_keys))
 
+        # Resolve arch specific entries
+        for values in paths.values():
+            new_values = []
+            for value in values:
+                value = pick_system_specific_value('sanity_check_paths', value, allow_none=True)
+                if value is not None:
+                    new_values.append(value)
+            values[:] = new_values
+
         # if enhance_sanity_check is not enabled, only sanity_check_commands specified in the easyconfig file are used,
         # the ones provided by the easyblock (via custom_commands) are ignored
         if ec_commands and not enhance_sanity_check:

easybuild/framework/easyconfig/types.py

@@ -377,6 +377,42 @@ def to_list_of_strings_and_tuples_and_dicts(spec):
     return str_tup_list
 
 
+def to_sanity_check_paths_entry(spec):
+    """
+    Convert a 'list of lists and strings' to a 'list of tuples and strings' while allowing dicts of lists or strings
+
+    Example:
+        ['foo', ['bar', 'baz'], {'f42': ['a', 'b']}]
+        to
+        ['foo', ('bar', 'baz'), {'f42': ('a', 'b')}]
+    """
+    result = []
+
+    if not isinstance(spec, (list, tuple)):
+        raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec))
+
+    for elem in spec:
+        if isinstance(elem, (string_type, tuple)):
+            result.append(elem)
+        elif isinstance(elem, list):
+            result.append(tuple(elem))
+        elif isinstance(elem, dict):
+            for key, value in elem.items():
+                if not isinstance(key, string_type):
+                    raise EasyBuildError("Expected keys to be of type string, got %s (%s)", key, type(key))
+                elif isinstance(value, list):
+                    elem[key] = tuple(value)
+                elif not isinstance(value, (string_type, tuple)):
+                    raise EasyBuildError("Expected elements to be of type string, tuple or list, got %s (%s)",
+                                         value, type(value))
+            result.append(elem)
+        else:
+            raise EasyBuildError("Expected elements to be of type string, tuple/list or dict, got %s (%s)",
+                                 elem, type(elem))
+
+    return result
+
+
 def to_sanity_check_paths_dict(spec):
     """
     Convert a sanity_check_paths dict as received by yaml (a dict with list values that contain either lists or strings)
@@ -391,7 +427,7 @@ def to_sanity_check_paths_dict(spec):
 
     sanity_check_dict = {}
     for key in spec:
-        sanity_check_dict[key] = to_list_of_strings_and_tuples(spec[key])
+        sanity_check_dict[key] = to_sanity_check_paths_entry(spec[key])
     return sanity_check_dict
 
 
@@ -554,11 +590,18 @@ def ensure_iterable_license_specs(specs):
         'key_types': [str],
     }
 ))
+STRING_OR_TUPLE_DICT = (dict, as_hashable(
+    {
+        'elem_types': [str],
+        'key_types': [str, TUPLE_OF_STRINGS],
+    }
+))
 STRING_OR_TUPLE_OR_DICT_LIST = (list, as_hashable({'elem_types': [str, TUPLE_OF_STRINGS, STRING_DICT]}))
+SANITY_CHECK_PATHS_ENTRY = (list, as_hashable({'elem_types': [str, TUPLE_OF_STRINGS, STRING_OR_TUPLE_DICT]}))
 SANITY_CHECK_PATHS_DICT = (dict, as_hashable({
     'elem_types': {
-        SANITY_CHECK_PATHS_FILES: [STRING_OR_TUPLE_LIST],
-        SANITY_CHECK_PATHS_DIRS: [STRING_OR_TUPLE_LIST],
+        SANITY_CHECK_PATHS_FILES: [SANITY_CHECK_PATHS_ENTRY],
+        SANITY_CHECK_PATHS_DIRS: [SANITY_CHECK_PATHS_ENTRY],
     },
     'opt_keys': [],
     'req_keys': [SANITY_CHECK_PATHS_FILES, SANITY_CHECK_PATHS_DIRS],
@@ -573,8 +616,8 @@ def ensure_iterable_license_specs(specs):
 CHECKSUMS = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT, CHECKSUM_LIST]}))
 
 CHECKABLE_TYPES = [CHECKSUM_LIST, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS,
-                   SANITY_CHECK_PATHS_DICT, STRING_DICT, STRING_OR_TUPLE_LIST, STRING_OR_TUPLE_OR_DICT_LIST,
-                   TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
+                   SANITY_CHECK_PATHS_DICT, SANITY_CHECK_PATHS_ENTRY, STRING_DICT, STRING_OR_TUPLE_LIST,
+                   STRING_OR_TUPLE_DICT, STRING_OR_TUPLE_OR_DICT_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
 
 # easy types, that can be verified with isinstance
 EASY_TYPES = [string_type, bool, dict, int, list, str, tuple]

easybuild/tools/systemtools.py

@@ -1257,48 +1257,63 @@ def check_python_version():
     return (python_maj_ver, python_min_ver)
 
 
+def pick_system_specific_value(description, options_or_value, allow_none=False):
+    """Pick an entry for the current system when the input has multiple options
+
+    :param description: Descriptive string about the value to be retrieved. Used for logging.
+    :param options_or_value: Either a dictionary with options to choose from or a value of any other type
+    :param allow_none: When True and no matching arch key was found, return None instead of an error
+
+    :return options_or_value when it is not a dictionary or the matching entry (if existing)
+    """
+    result = options_or_value
+    if isinstance(options_or_value, dict):
+        if not options_or_value:
+            raise EasyBuildError("Found empty dict as %s!", description)
+        other_keys = [x for x in options_or_value.keys() if not x.startswith(ARCH_KEY_PREFIX)]
+        if other_keys:
+            other_keys = ','.join(sorted(other_keys))
+            raise EasyBuildError("Unexpected keys in %s: %s (only '%s' keys are supported)",
+                                 description, other_keys, ARCH_KEY_PREFIX)
+        host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture()
+        star_arch_key = ARCH_KEY_PREFIX + '*'
+        # check for specific 'arch=' key first
+        try:
+            result = options_or_value[host_arch_key]
+            _log.info("Selected %s from %s for %s (using key %s)",
+                      result, options_or_value, description, host_arch_key)
+        except KeyError:
+            # fall back to 'arch=*'
+            try:
+                result = options_or_value[star_arch_key]
+                _log.info("Selected %s from %s for %s (using fallback key %s)",
+                          result, options_or_value, description, star_arch_key)
+            except KeyError:
+                if allow_none:
+                    result = None
+                else:
+                    raise EasyBuildError("No matches for %s in %s (looking for %s)",
+                                         description, options_or_value, host_arch_key)
+    return result
+
+
 def pick_dep_version(dep_version):
     """
     Pick the correct dependency version to use for this system.
     Input can either be:
     * a string value (or None)
     * a dict with options to choose from
 
-    Return value is the version to use.
+    Return value is the version to use or False to skip this dependency.
     """
-    if isinstance(dep_version, string_type):
-        _log.debug("Version is already a string ('%s'), OK", dep_version)
-        result = dep_version
-
-    elif dep_version is None:
+    if dep_version is None:
         _log.debug("Version is None, OK")
         result = None
-
-    elif isinstance(dep_version, dict):
-        arch_keys = [x for x in dep_version.keys() if x.startswith(ARCH_KEY_PREFIX)]
-        other_keys = [x for x in dep_version.keys() if x not in arch_keys]
-        if other_keys:
-            other_keys = ','.join(sorted(other_keys))
-            raise EasyBuildError("Unexpected keys in version: %s (only 'arch=' keys are supported)", other_keys)
-        if arch_keys:
-            host_arch_key = ARCH_KEY_PREFIX + get_cpu_architecture()
-            star_arch_key = ARCH_KEY_PREFIX + '*'
-            # check for specific 'arch=' key first
-            if host_arch_key in dep_version:
-                result = dep_version[host_arch_key]
-                _log.info("Version selected from %s using key %s: %s", dep_version, host_arch_key, result)
-            # fall back to 'arch=*'
-            elif star_arch_key in dep_version:
-                result = dep_version[star_arch_key]
-                _log.info("Version selected for %s using fallback key %s: %s", dep_version, star_arch_key, result)
-            else:
-                raise EasyBuildError("No matches for version in %s (looking for %s)", dep_version, host_arch_key)
-        else:
-            raise EasyBuildError("Found empty dict as version!")
-
     else:
-        typ = type(dep_version)
-        raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version)
+        result = pick_system_specific_value("version", dep_version)
+        if not isinstance(result, string_type) and result is not False:
+            typ = type(dep_version)
+            raise EasyBuildError("Unknown value type for version: %s (%s), should be string value", typ, dep_version)
 
     return result
 

test/framework/easyblock.py

@@ -38,6 +38,7 @@
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
 
+import easybuild.tools.systemtools as st
 from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.easyconfig import EasyConfig
@@ -2744,6 +2745,32 @@ def test_avail_easyblocks(self):
         self.assertEqual(hpl['class'], 'EB_HPL')
         self.assertTrue(hpl['loc'].endswith('sandbox/easybuild/easyblocks/h/hpl.py'))
 
+    def test_arch_specific_sanity_check(self):
+        """Tests that the correct version is chosen for this architecture"""
+
+        my_arch = st.get_cpu_architecture()
+
+        self.contents = '\n'.join([
+            'easyblock = "ConfigureMake"',
+            'name = "test"',
+            'version = "0.2"',
+            'homepage = "https://example.com"',
+            'description = "test"',
+            'toolchain = SYSTEM',
+            'sanity_check_paths = {',
+            "  'files': [{'arch=%s': 'correct.a'}, 'default.a']," % my_arch,
+            "  'dirs': [{'arch=%s': ('correct', 'alternative')}, {'arch=no-arch': 'not-used'}]," % my_arch,
+            '}',
+        ])
+        self.writeEC()
+        ec = EasyConfig(self.eb_file)
+        eb = EasyBlock(ec)
+        paths, _, _ = eb._sanity_check_step_common(None, None)
+
+        self.assertEqual(set(paths.keys()), set(('files', 'dirs')))
+        self.assertEqual(paths['files'], ['correct.a', 'default.a'])
+        self.assertEqual(paths['dirs'], [('correct', 'alternative')])
+
     def test_sanity_check_paths_verification(self):
         """Test verification of sanity_check_paths w.r.t. keys & values."""
 

test/framework/systemtools.py

@@ -51,7 +51,7 @@
 from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor
 from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version
 from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory
-from easybuild.tools.systemtools import find_library_path, locate_solib, pick_dep_version
+from easybuild.tools.systemtools import find_library_path, locate_solib, pick_dep_version, pick_system_specific_value
 
 
 PROC_CPUINFO_TXT = None
@@ -933,6 +933,38 @@ def test_pick_dep_version(self):
         error_pattern = r"Unknown value type for version: .* \(1.23\), should be string value"
         self.assertErrorRegex(EasyBuildError, error_pattern, pick_dep_version, 1.23)
 
+    def test_pick_system_specific_value(self):
+        """Test pick_system_specific_value function."""
+
+        self.assertEqual(pick_system_specific_value('test-desc', None), None)
+        self.assertEqual(pick_system_specific_value('test-desc', '1.2.3'), '1.2.3')
+        self.assertEqual(pick_system_specific_value('test-desc', (42, 'foobar')), (42, 'foobar'))
+
+        option_dict = {
+            'arch=x86_64': '1.2.3-amd64',
+            'arch=POWER': '1.2.3-ppc64le',
+            'arch=*': '1.2.3-other',
+        }
+
+        st.get_cpu_architecture = lambda: X86_64
+        self.assertEqual(pick_system_specific_value('test-desc', option_dict), '1.2.3-amd64')
+
+        st.get_cpu_architecture = lambda: POWER
+        self.assertEqual(pick_system_specific_value('test-desc', option_dict), '1.2.3-ppc64le')
+
+        st.get_cpu_architecture = lambda: "NON_EXISTING_ARCH"
+        self.assertEqual(pick_system_specific_value('test-desc', option_dict), '1.2.3-other')
+
+        error_pattern = "Found empty dict as test-desc"
+        self.assertErrorRegex(EasyBuildError, error_pattern, pick_system_specific_value, 'test-desc', {})
+
+        error_pattern = r"Unexpected keys in test-desc: foo \(only 'arch=' keys are supported\)"
+        self.assertErrorRegex(EasyBuildError, error_pattern, pick_system_specific_value, 'test-desc',
+                              {'foo': '1'})
+        error_pattern = r"Unexpected keys in test-desc: foo \(only 'arch=' keys are supported\)"
+        self.assertErrorRegex(EasyBuildError, error_pattern, pick_system_specific_value, 'test-desc',
+                              {'foo': '1', 'arch=POWER': '2'})
+
     def test_check_os_dependency(self):
         """Test check_os_dependency."""
 

test/framework/type_checking.py

@@ -701,7 +701,7 @@ def test_to_sanity_check_paths_dict(self):
         self.assertErrorRegex(EasyBuildError, "Expected value to be a dict", to_sanity_check_paths_dict, [])
         error_msg = "Expected value to be a list"
         self.assertErrorRegex(EasyBuildError, error_msg, to_sanity_check_paths_dict, {'files': 'foo', 'dirs': []})
-        error_msg = "Expected elements to be of type string, tuple or list"
+        error_msg = "Expected elements to be of type string, tuple/list or dict"
         self.assertErrorRegex(EasyBuildError, error_msg, to_sanity_check_paths_dict, {'files': [], 'dirs': [1]})
 
     def test_to_checksums(self):