Merge pull request #2599 from ocaisa/allow_dep_upgrades

Add ability to upgrade dependencies based on available easyconfigs

Author: boegel

easybuild/framework/easyconfig/format/one.py

@@ -71,7 +71,7 @@ def dump_dependency(dep, toolchain):
     if dep['external_module']:
         res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name'])
     else:
-        # mininal spec: (name, version)
+        # minimal spec: (name, version)
         tup = (dep['name'], dep['version'])
         if dep['toolchain'] != toolchain:
             if dep[SYSTEM_TOOLCHAIN_NAME]:

easybuild/framework/easyconfig/tweak.py

@@ -758,7 +758,8 @@ def find_easyconfigs(path, ignore_dirs=None):
     return files
 
 
-def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False):
+def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False,
+                case_sensitive=False):
     """
     Search for files using in specified paths using specified search query (regular expression)
 
@@ -782,7 +783,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen
 
     # compile regex, case-insensitive
     try:
-        query = re.compile(query, re.I)
+        if case_sensitive:
+            query = re.compile(query)
+        else:
+            # compile regex, case-insensitive
+            query = re.compile(query, re.I)
     except re.error as err:
         raise EasyBuildError("Invalid search query: %s", err)
 

easybuild/tools/filetools.py

@@ -318,6 +318,9 @@ def software_options(self):
 
         opts['map-toolchains'] = ("Enable mapping of (sub)toolchains when --try-toolchain(-version) is used",
                                   None, 'store_true', True)
+        opts['try-update-deps'] = ("Try to update versions of the dependencies of an easyconfig based on what is "
+                                   "available in the robot path",
+                                   None, 'store_true', False)
 
         self.log.debug("software_options: descr %s opts %s" % (descr, opts))
         self.add_group_parser(opts, descr)
@@ -1481,6 +1484,7 @@ def process_software_build_specs(options):
         'version': options.try_software_version,
         'toolchain_name': options.try_toolchain_name,
         'toolchain_version': options.try_toolchain_version,
+        'update_deps': options.try_update_deps
     }
 
     # process easy options

easybuild/tools/options.py

@@ -469,20 +469,36 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro
     return ordered_ecs
 
 
-def search_easyconfigs(query, short=False, filename_only=False, terse=False):
-    """Search for easyconfigs, if a query is provided."""
+def search_easyconfigs(query, short=False, filename_only=False, terse=False, consider_extra_paths=True,
+                       print_result=True, case_sensitive=False):
+    """
+    Search for easyconfigs, if a query is provided.
+
+    :param query: regex query string
+    :param short: figure out common prefix of hits, use variable to factor it out
+    :param filename_only: only print filenames, not paths
+    :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing
+    :param consider_extra_paths: consider all paths when searching
+    :param print_result: print the list of easyconfigs
+    :param case_sensitive: boolean to decide whether search is case sensitive
+
+    :return: return a list of paths for the query
+    """
     search_path = build_option('robot_path')
     if not search_path:
         search_path = [os.getcwd()]
     extra_search_paths = build_option('search_paths')
-    if extra_search_paths:
-        search_path.extend(extra_search_paths)
+    # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths
+    if extra_search_paths and consider_extra_paths:
+        # we shouldn't use += or .extend here but compose a new list,
+        # to avoid adding a path to the list returned by build_option('robot_path') !
+        search_path = search_path + extra_search_paths
 
     ignore_dirs = build_option('ignore_dirs')
 
     # note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs
     var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse,
-                                  silent=True, filename_only=False)
+                                  silent=True, filename_only=False, case_sensitive=case_sensitive)
 
     # filter out archived easyconfigs, these are handled separately
     hits, archived_hits = [], []
@@ -492,32 +508,42 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False):
         else:
             hits.append(hit)
 
-    # check whether only filenames should be printed
+    # check whether only filenames should be used
     if filename_only:
         hits = [os.path.basename(hit) for hit in hits]
         archived_hits = [os.path.basename(hit) for hit in archived_hits]
 
-    # prepare output format
-    if terse:
-        lines, tmpl = [], '%s'
+    if print_result:
+        # prepare output format
+        if terse:
+            lines, tmpl = [], '%s'
+        else:
+            lines = ['%s=%s' % var_def for var_def in var_defs]
+            tmpl = ' * %s'
+
+        # non-archived hits are shown first
+        lines.extend(tmpl % hit for hit in hits)
+
+        # also take into account archived hits
+        if archived_hits:
+            if build_option('consider_archived_easyconfigs'):
+                if not terse:
+                    lines.extend(['', "Matching archived easyconfigs:", ''])
+                lines.extend(tmpl % hit for hit in archived_hits)
+            elif not terse:
+                cnt = len(archived_hits)
+                lines.extend([
+                    '',
+                    "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them"
+                    % cnt,
+                ])
+
+        print('\n'.join(lines))
+
+    # if requested return the matches as a list
+    if build_option('consider_archived_easyconfigs'):
+        final_hits = hits + archived_hits
     else:
-        lines = ['%s=%s' % var_def for var_def in var_defs]
-        tmpl = ' * %s'
-
-    # non-archived hits are shown first
-    lines.extend(tmpl % hit for hit in hits)
-
-    # also take into account archived hits
-    if archived_hits:
-        if build_option('consider_archived_easyconfigs'):
-            if not terse:
-                lines.extend(['', "Matching archived easyconfigs:", ''])
-            lines.extend(tmpl % hit for hit in archived_hits)
-        elif not terse:
-            cnt = len(archived_hits)
-            lines.extend([
-                '',
-                "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt,
-            ])
-
-    print('\n'.join(lines))
+        final_hits = hits
+
+    return final_hits

easybuild/tools/robot.py

@@ -1765,11 +1765,11 @@ def run_checks():
 
         # full check also catches checksum issues with extensions
         res = eb.check_checksums()
-        self.assertEqual(len(res), 5)
+        self.assertEqual(len(res), 4)
         run_checks()
 
         idx = 2
-        for ext in ['bar', 'barbar', 'toy']:
+        for ext in ['bar', 'barbar']:
             expected = "Checksums missing for one or more sources/patches of extension %s in " % ext
             self.assertTrue(res[idx].startswith(expected))
             idx += 1

test/framework/easyblock.py

@@ -0,0 +1,40 @@
+##
+# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
+#
+# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC
+# Authors::   Thekla Loizou <[email protected]>
+# License::   MIT/GPL
+# $Id$
+#
+# This work implements a part of the HPCBIOS project and is a component of the policy:
+# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html
+##
+easyblock = 'ConfigureMake'
+
+name = 'gzip'
+version = '1.4'
+
+homepage = "http://www.gzip.org/"
+description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"
+
+# test toolchain specification
+toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}
+
+# source tarball filename
+sources = ['%(name)s-%(version)s.tar.gz']
+
+# download location for source files
+source_urls = [GNU_SOURCE]
+
+# make sure the gzip and gunzip binaries are available after installation
+sanity_check_paths = {
+    'files': ["bin/gunzip", "bin/gzip"],
+    'dirs': [],
+}
+
+# run 'gzip -h' and 'gzip --version' after installation
+sanity_check_commands = [True, ('gzip', '--version')]
+
+software_license = GPLv3
+
+moduleclass = 'tools'

test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb

@@ -0,0 +1,40 @@
+##
+# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
+#
+# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC
+# Authors::   Thekla Loizou <[email protected]>
+# License::   MIT/GPL
+# $Id$
+#
+# This work implements a part of the HPCBIOS project and is a component of the policy:
+# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html
+##
+easyblock = 'ConfigureMake'
+
+name = 'gzip'
+version = '1.6'
+
+homepage = "http://www.gzip.org/"
+description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"
+
+# test toolchain specification
+toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}
+
+# source tarball filename
+sources = ['%(name)s-%(version)s.tar.gz']
+
+# download location for source files
+source_urls = [GNU_SOURCE]
+
+# make sure the gzip and gunzip binaries are available after installation
+sanity_check_paths = {
+    'files': ["bin/gunzip", "bin/gzip"],
+    'dirs': [],
+}
+
+# run 'gzip -h' and 'gzip --version' after installation
+sanity_check_commands = [True, ('gzip', '--version')]
+
+software_license = GPLv3
+
+moduleclass = 'tools'

test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb

@@ -15,7 +15,11 @@ toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}
 
 source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
 sources = [SOURCE_TAR_GZ]
+checksums = ['aa9d9ca75c7d7164f6bf3a52ecd77340eec02c18']
 
 builddependencies = [('binutils', '2.26')]
 
+# introduce fake dependency for testing dep upgrades
+dependencies = [('gzip', '1.4')]
+
 moduleclass = 'system'

test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb

@@ -47,6 +47,7 @@ exts_list = [
     (name, version, {
         'sanity_check_paths': {'files': ['lib/libtoy.a'], 'dirs': []},
         'exts_filter': ("ls -l lib/libtoy.a", ''),
+        'checksums': ['44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc']
     }),
 ]