Merge pull request #4669 from bartoldeman/job-local-tweak

Let jobs retweak easyconfigs themselves by passing down `--try-*` options

Author: boegel

easybuild/framework/easyconfig/tweak.py

@@ -36,6 +36,7 @@
 * Fotis Georgatos (Uni.Lu, NTUA)
 * Alan O'Cais (Juelich Supercomputing Centre)
 * Maxime Boissonneault (Universite Laval, Calcul Quebec, Compute Canada)
+* Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
 """
 import copy
 import functools
@@ -82,8 +83,9 @@ def ec_filename_for(path):
     return fn
 
 
-def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
-    """Tweak list of easyconfigs according to provided build specifications."""
+def tweak(easyconfigs, build_specs, modtool, targetdirs=None, return_map=False):
+    """Tweak list of easyconfigs according to provided build specifications.
+       If return_map=True, also returns tweaked to original file mapping"""
     # keep track of originally listed easyconfigs (via their path)
     listed_ec_paths = [ec['spec'] for ec in easyconfigs]
 
@@ -92,6 +94,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
         tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
     modifying_toolchains_or_deps = False
     src_to_dst_tc_mapping = {}
+    tweak_map = {}
     revert_to_regex = False
 
     if 'update_deps' in build_specs:
@@ -223,13 +226,18 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
             if modifying_toolchains_or_deps:
                 if tc_name in src_to_dst_tc_mapping:
                     # Note pruned_build_specs are not passed down for dependencies
-                    map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
-                                                          targetdir=tweaked_ecs_deps_path,
-                                                          update_dep_versions=update_dependencies,
-                                                          ignore_versionsuffixes=ignore_versionsuffixes)
+                    new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
+                                                                        targetdir=tweaked_ecs_deps_path,
+                                                                        update_dep_versions=update_dependencies,
+                                                                        ignore_versionsuffixes=ignore_versionsuffixes)
             else:
-                tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
+                new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
+
+        if new_ec_file:
+            tweak_map[new_ec_file] = orig_ec['spec']
 
+    if return_map:
+        return tweaked_easyconfigs, tweak_map
     return tweaked_easyconfigs
 
 

easybuild/main.py

@@ -37,6 +37,7 @@
 * Ward Poelmans (Ghent University)
 * Fotis Georgatos (Uni.Lu, NTUA)
 * Maxime Boissonneault (Compute Canada)
+* Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
 """
 import copy
 import os
@@ -430,7 +431,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
     # if easyconfig files for the dependencies are not available
     if try_to_generate and build_specs and not generated_ecs:
-        easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
+        easyconfigs, tweak_map = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths, return_map=True)
+    else:
+        tweak_map = None
 
     if options.containerize:
         # if --containerize/-C create a container recipe (and optionally container image), and stop
@@ -552,7 +555,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
 
     # submit build as job(s), clean up and exit
     if options.job:
-        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
+        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing, tweak_map=tweak_map)
         if not testing:
             print_msg("Submitted parallel build jobs, exiting now")
             return True

easybuild/tools/job/backend.py

@@ -32,6 +32,7 @@
 """
 
 from abc import ABCMeta, abstractmethod
+from types import SimpleNamespace
 
 from easybuild.base import fancylogger
 from easybuild.tools.config import get_job_backend
@@ -69,7 +70,7 @@ def make_job(self, script, name, env_vars=None, hours=None, cores=None):
         See the `Job`:class: constructor for an explanation of what
         the arguments are.
         """
-        pass
+        return SimpleNamespace()
 
     @abstractmethod
     def queue(self, job, dependencies=frozenset()):

easybuild/tools/parallelbuild.py

@@ -33,6 +33,7 @@
 * Toon Willems (Ghent University)
 * Kenneth Hoste (Ghent University)
 * Stijn De Weirdt (Ghent University)
+* Bart Oldeman (McGill University, Calcul Quebec, Digital Research Alliance of Canada)
 """
 import math
 import os
@@ -45,7 +46,7 @@
 from easybuild.tools.config import build_option, get_repository, get_repositorypath
 from easybuild.tools.filetools import get_cwd
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
-from easybuild.tools.job.backend import job_backend
+from easybuild.tools.job.backend import job_backend, JobBackend
 from easybuild.tools.repository.repository import init_repository
 
 
@@ -57,7 +58,8 @@ def _to_key(dep):
     return ActiveMNS().det_full_module_name(dep)
 
 
-def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True):
+def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', testing=False,
+                                  prepare_first=True, tweak_map=None, try_opts=''):
     """
     Build easyconfigs in parallel by submitting jobs to a batch-queuing system.
     Return list of jobs submitted.
@@ -69,11 +71,14 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu
     :param build_command: build command to use
     :param easyconfigs: list of easyconfig files
     :param output_dir: output directory
+    :param testing: If `True`, skip actual job submission
     :param prepare_first: prepare by runnning fetch step first for each easyconfig
+    :param tweak_map: Mapping from tweaked to original easyconfigs
+    :param try_opts: --try-* options to pass if the easyconfig is tweaked
     """
     _log.info("going to build these easyconfigs in parallel: %s", [os.path.basename(ec['spec']) for ec in easyconfigs])
 
-    active_job_backend = job_backend()
+    active_job_backend = JobBackend() if testing else job_backend()
     if active_job_backend is None:
         raise EasyBuildError("Can not use --job if no job backend is available.")
 
@@ -93,12 +98,17 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu
         # this is very important, otherwise we might have race conditions
         # e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
         # running this step here, prevents this
-        if prepare_first:
+        if prepare_first and not testing:
             prepare_easyconfig(easyconfig)
 
+        # convert <tweaked easyconfig.eb> to <original-easyconfig.eb --try-xxx> to avoid needing a shared tmpdir
+        spec = easyconfig['spec']
+        if spec in (tweak_map or {}):
+            spec = tweak_map[spec] + try_opts
+
         # the new job will only depend on already submitted jobs
-        _log.info("creating job for ec: %s" % os.path.basename(easyconfig['spec']))
-        new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir)
+        _log.info("creating job for ec: %s using %s" % (os.path.basename(easyconfig['spec']), spec))
+        new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir, spec=spec)
 
         # filter out dependencies marked as external modules
         deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)]
@@ -116,24 +126,27 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu
 
     active_job_backend.complete()
 
-    return jobs
+    return build_command if testing else jobs
 
 
-def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
+def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True, tweak_map=None):
     """
     Submit jobs.
     :param ordered_ecs: list of easyconfigs, in the order they should be processed
     :param cmd_line_opts: list of command line options (in 'longopt=value' form)
     :param testing: If `True`, skip actual job submission
     :param prepare_first: prepare by runnning fetch step first for each easyconfig
+    :param tweak_map: Mapping from tweaked to original easyconfigs
     """
     curdir = get_cwd()
 
-    # regex pattern for options to ignore (help options can't reach here)
+    # regex patterns for options to ignore and tweak options (help options can't reach here)
     ignore_opts = re.compile('^--robot$|^--job|^--try-.*$|^--easystack$')
+    try_opts_re = re.compile('^--try-.*$')
 
     # generate_cmd_line returns the options in form --longopt=value
     opts = [o for o in cmd_line_opts if not ignore_opts.match(o.split('=')[0])]
+    try_opts = [o for o in cmd_line_opts if try_opts_re.match(o.split('=')[0])]
 
     # add --disable-job to make sure the submitted job doesn't submit a job itself,
     # resulting in an infinite cycle of jobs;
@@ -143,6 +156,7 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
 
     # compose string with command line options, properly quoted and with '%' characters escaped
     opts_str = ' '.join(opts).replace('%', '%%')
+    try_opts_str = ' ' + ' '.join(try_opts).replace('%', '%%')
 
     eb_cmd = build_option('job_eb_cmd')
 
@@ -154,19 +168,19 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True):
     _log.info("Command template for jobs: %s", command)
     if testing:
         _log.debug("Skipping actual submission of jobs since testing mode is enabled")
-        return command
-    else:
-        return build_easyconfigs_in_parallel(command, ordered_ecs, prepare_first=prepare_first)
+    return build_easyconfigs_in_parallel(command, ordered_ecs, testing=testing, prepare_first=prepare_first,
+                                         tweak_map=tweak_map, try_opts=try_opts_str)
 
 
-def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'):
+def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build', spec=''):
     """
     Creates a job to build a *single* easyconfig.
 
     :param job_backend: A factory object for querying server parameters and creating actual job objects
     :param build_command: format string for command, full path to an easyconfig file will be substituted in it
     :param easyconfig: easyconfig as processed by process_easyconfig
     :param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
+    :param spec: untweaked easyconfig name with optional --try-* options
 
     returns the job
     """
@@ -183,7 +197,7 @@ def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-bui
     command = build_command % {
         'add_opts': add_opts,
         'output_dir': os.path.join(os.path.abspath(output_dir), name),
-        'spec': easyconfig['spec'],
+        'spec': spec or easyconfig['spec'],
     }
 
     # just use latest build stats

test/framework/easyconfig.py

@@ -805,7 +805,8 @@ def test_tweak_multiple_tcs(self):
         untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb')
         easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)])
         tweak_specs = {'moduleclass': 'debugger'}
-        easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths)
+        easyconfigs, tweak_map = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths,
+                                       return_map=True)
         # Check that all expected tweaked easyconfigs exists
         tweaked_openmpi_1 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_1))
         tweaked_openmpi_2 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_2))
@@ -817,6 +818,7 @@ def test_tweak_multiple_tcs(self):
                         "Tweaked value not found in " + tweaked_openmpi_content_1)
         self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_2,
                         "Tweaked value not found in " + tweaked_openmpi_content_2)
+        self.assertEqual(tweak_map, {tweaked_openmpi_1: untweaked_openmpi_1, tweaked_openmpi_2: untweaked_openmpi_2})
 
     def test_installversion(self):
         """Test generation of install version."""

test/framework/options.py

@@ -508,9 +508,10 @@ def test_job(self):
         """Test submitting build as a job."""
 
         # use gzip-1.4.eb easyconfig file that comes with the tests
-        eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb')
+        test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+        eb_file = os.path.join(test_ecs, 'g', 'gzip', 'gzip-1.4.eb')
 
-        def check_args(job_args, passed_args=None):
+        def check_args(job_args, passed_args=None, msgstrs=None, try_opts='', tweaked_eb_file='gzip-1.4.eb'):
             """Check whether specified args yield expected result."""
             if passed_args is None:
                 passed_args = job_args[:]
@@ -529,16 +530,43 @@ def check_args(job_args, passed_args=None):
             assertmsg = "Info log msg with job command template for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt)
             self.assertTrue(re.search(job_msg, outtxt), assertmsg)
 
+            if msgstrs is None:
+                msgstrs = [(tweaked_eb_file, eb_file + try_opts)]
+
+            assertmsg = "Info log msg with creating job for --job (job_msg: %s, outtxt: %s)" % (job_msg, outtxt)
+            for msgstr in msgstrs:
+                job_msg = r"INFO creating job for ec: %s using %s\n" % msgstr
+                self.assertTrue(re.search(job_msg, outtxt), assertmsg)
+
         # options passed are reordered, so order here matters to make tests pass
         check_args(['--debug'])
         check_args(['--debug', '--stop=configure', '--try-software-name=foo'],
-                   passed_args=['--debug', "--stop='configure'"])
+                   passed_args=['--debug', "--stop='configure'"],
+                   try_opts=" --try-software-name='foo'",
+                   tweaked_eb_file="foo-1.4.eb")
         check_args(['--debug', '--robot-paths=/tmp/foo:/tmp/bar'],
                    passed_args=['--debug', "--robot-paths='/tmp/foo:/tmp/bar'"])
         # --robot has preference over --robot-paths, --robot is not passed down
         check_args(['--debug', '--robot-paths=/tmp/foo', '--robot=%s' % self.test_prefix],
                    passed_args=['--debug', "--robot-paths='%s:/tmp/foo'" % self.test_prefix])
 
+        # check if libtoy dep uses --try-toolchain but gzip does not (easyconfig exists already)
+        eb_file = os.path.join(self.test_buildpath, 'toy-0.0-with-deps.eb')
+        copy_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'), eb_file)
+        write_file(eb_file, "dependencies = [('libtoy', '0.0'), ('gzip', '1.4')]\n", append=True)
+        try_opts = " --try-toolchain='GCC,4.9.3-2.26'"
+        tweaked_eb_file = "toy-0.0-GCC-4.9.3-2.26.eb"
+        gzip_eb_file = 'gzip-1.4-GCC-4.9.3-2.26.eb'
+        check_args(['--debug', '--stop=configure', '--try-toolchain=GCC,4.9.3-2.26', '--robot'],
+                   passed_args=['--debug', "--stop='configure'"],
+                   msgstrs=[
+                       (tweaked_eb_file, eb_file + try_opts),
+                       ('libtoy-0.0-GCC-4.9.3-2.26.eb',
+                        os.path.join(test_ecs, 'l', 'libtoy', 'libtoy-0.0.eb') + try_opts),
+                       (gzip_eb_file, os.path.join(test_ecs, 'g', 'gzip', gzip_eb_file))],
+                   try_opts=try_opts,
+                   tweaked_eb_file=tweaked_eb_file)
+
     # 'zzz' prefix in the test name is intentional to make this test run last,
     # since it fiddles with the logging infrastructure which may break things
     def test_zzz_logtostdout(self):

test/framework/parallelbuild.py

@@ -290,7 +290,7 @@ def test_build_easyconfigs_in_parallel_gc3pie(self):
     def test_submit_jobs(self):
         """Test submit_jobs"""
         test_easyconfigs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
-        toy_ec = os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb')
+        toy_ec = process_easyconfig(os.path.join(test_easyconfigs_dir, 't', 'toy', 'toy-0.0.eb'))
 
         args = [
             '--debug',
@@ -303,7 +303,7 @@ def test_submit_jobs(self):
             '--job-cores=3',
         ]
         eb_go = parse_options(args=args)
-        cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True)
+        cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True)
 
         # these patterns must be found
         regexs = [
@@ -331,7 +331,7 @@ def test_submit_jobs(self):
 
         # test again with custom EasyBuild command to use in jobs
         update_build_option('job_eb_cmd', "/just/testing/bin/eb --debug")
-        cmd = submit_jobs([toy_ec], eb_go.generate_cmd_line(), testing=True)
+        cmd = submit_jobs(toy_ec, eb_go.generate_cmd_line(), testing=True)
         regex = re.compile(r" && /just/testing/bin/eb --debug %\(spec\)s ")
         self.assertTrue(regex.search(cmd), "Pattern '%s' found in: %s" % (regex.pattern, cmd))
 

test/framework/robot.py

@@ -1139,7 +1139,8 @@ def test_tweak_robotpath(self):
 
         # Tweak the toolchain version of the easyconfig
         tweak_specs = {'toolchain_version': '6.4.0-2.28'}
-        easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths)
+        easyconfigs, tweak_map = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths,
+                                       return_map=True)
 
         # Check that all expected tweaked easyconfigs exists
         tweaked_openmpi = os.path.join(tweaked_ecs_paths[0], 'OpenMPI-2.1.2-GCC-6.4.0-2.28.eb')
@@ -1155,6 +1156,10 @@ def test_tweak_robotpath(self):
         # Check it picks up the untweaked dependency of the tweaked OpenMPI
         untweaked_hwloc = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')
         self.assertIn(untweaked_hwloc, specs)
+        # Check correctness of tweak_map (maps back to the original untweaked file, even for hwloc, where the
+        # tweaked version is generated but not used)
+        self.assertEqual(tweak_map, {tweaked_openmpi: untweaked_openmpi,
+                                     tweaked_hwloc: untweaked_hwloc.replace("6.4.0-2.28", "4.6.4")})
 
     def test_robot_find_subtoolchain_for_dep(self):
         """Test robot_find_subtoolchain_for_dep."""