Merge branch 'feat/alt_flex_alloc' of github.com:ekouts/reframe into feat/alt_flex_alloc

Author: Vasileios Karakasis

reframe/core/buildsystems.py

@@ -264,10 +264,10 @@ class Make(BuildSystem):
     def emit_build_commands(self, environ):
         cmd_parts = ['make']
         if self.makefile:
-            cmd_parts += ['-f %s' % self.makefile]
+            cmd_parts += [f'-f {self.makefile}']
 
         if self.srcdir:
-            cmd_parts += ['-C %s' % self.srcdir]
+            cmd_parts += [f'-C {self.srcdir}']
 
         cmd_parts += ['-j']
         if self.max_concurrency is not None:
@@ -283,31 +283,36 @@ def emit_build_commands(self, environ):
         fflags   = self._fflags(environ)
         ldflags  = self._ldflags(environ)
         if cc:
-            cmd_parts += ['CC="%s"' % cc]
+            cmd_parts += [f'CC="{cc}"']
 
         if cxx:
-            cmd_parts += ['CXX="%s"' % cxx]
+            cmd_parts += [f'CXX="{cxx}"']
 
         if ftn:
-            cmd_parts += ['FC="%s"' % ftn]
+            cmd_parts += [f'FC="{ftn}"']
 
         if nvcc:
-            cmd_parts += ['NVCC="%s"' % nvcc]
+            cmd_parts += [f'NVCC="{nvcc}"']
 
         if cppflags:
-            cmd_parts += ['CPPFLAGS="%s"' % ' '.join(cppflags)]
+            flags = ' '.join(cppflags)
+            cmd_parts += [f'CPPFLAGS="{flags}"']
 
         if cflags:
-            cmd_parts += ['CFLAGS="%s"' % ' '.join(cflags)]
+            flags = ' '.join(cflags)
+            cmd_parts += [f'CFLAGS="{flags}"']
 
         if cxxflags:
-            cmd_parts += ['CXXFLAGS="%s"' % ' '.join(cxxflags)]
+            flags = ' '.join(cxxflags)
+            cmd_parts += [f'CXXFLAGS="{flags}"']
 
         if fflags:
-            cmd_parts += ['FCFLAGS="%s"' % ' '.join(fflags)]
+            flags = ' '.join(fflags)
+            cmd_parts += [f'FCFLAGS="{flags}"']
 
         if ldflags:
-            cmd_parts += ['LDFLAGS="%s"' % ' '.join(ldflags)]
+            flags = ' '.join(ldflags)
+            cmd_parts += [f'LDFLAGS="{flags}"']
 
         if self.options:
             cmd_parts += self.options
@@ -384,13 +389,13 @@ class SingleSource(BuildSystem):
     lang = variable(str, type(None), value=None)
 
     def _auto_exec_name(self):
-        return '%s.x' % os.path.splitext(self.srcfile)[0]
+        return os.path.splitext(self.srcfile)[0] + '.x'
 
     def emit_build_commands(self, environ):
         if not self.srcfile:
-            raise BuildSystemError(
-                'a source file is required when using the %s build system' %
-                type(self).__name__)
+            bname = type(self).__name__
+            raise BuildSystemError(f'a source file is required when using '
+                                   f'the {bname!r} build system')
 
         cc = self._cc(environ)
         cxx = self._cxx(environ)
@@ -527,11 +532,11 @@ def _combine_flags(self, cppflags, xflags):
     def emit_build_commands(self, environ):
         prepare_cmd = []
         if self.srcdir:
-            prepare_cmd += ['cd %s' % self.srcdir]
+            prepare_cmd += [f'cd {self.srcdir}']
 
         if self.builddir:
-            prepare_cmd += ['mkdir -p %s' % self.builddir,
-                            'cd %s' % self.builddir]
+            prepare_cmd += [f'mkdir -p {self.builddir}',
+                            f'cd {self.builddir}']
 
         cmake_cmd = ['cmake']
         cc = self._cc(environ)
@@ -544,28 +549,32 @@ def emit_build_commands(self, environ):
         fflags   = self._combine_flags(cppflags, self._fflags(environ))
         ldflags  = self._ldflags(environ)
         if cc:
-            cmake_cmd += ['-DCMAKE_C_COMPILER="%s"' % cc]
+            cmake_cmd += [f'-DCMAKE_C_COMPILER="{cc}"']
 
         if cxx:
-            cmake_cmd += ['-DCMAKE_CXX_COMPILER="%s"' % cxx]
+            cmake_cmd += [f'-DCMAKE_CXX_COMPILER="{cxx}"']
 
         if ftn:
-            cmake_cmd += ['-DCMAKE_Fortran_COMPILER="%s"' % ftn]
+            cmake_cmd += [f'-DCMAKE_Fortran_COMPILER="{ftn}"']
 
         if nvcc:
-            cmake_cmd += ['-DCMAKE_CUDA_COMPILER="%s"' % nvcc]
+            cmake_cmd += [f'-DCMAKE_CUDA_COMPILER="{nvcc}"']
 
         if cflags:
-            cmake_cmd += ['-DCMAKE_C_FLAGS="%s"' % ' '.join(cflags)]
+            flags = ' '.join(cflags)
+            cmake_cmd += [f'-DCMAKE_C_FLAGS="{flags}"']
 
         if cxxflags:
-            cmake_cmd += ['-DCMAKE_CXX_FLAGS="%s"' % ' '.join(cxxflags)]
+            flags = ' '.join(cxxflags)
+            cmake_cmd += [f'-DCMAKE_CXX_FLAGS="{flags}"']
 
         if fflags:
-            cmake_cmd += ['-DCMAKE_Fortran_FLAGS="%s"' % ' '.join(fflags)]
+            flags = ' '.join(fflags)
+            cmake_cmd += [f'-DCMAKE_Fortran_FLAGS="{flags}"']
 
         if ldflags:
-            cmake_cmd += ['-DCMAKE_EXE_LINKER_FLAGS="%s"' % ' '.join(ldflags)]
+            flags = ' '.join(ldflags)
+            cmake_cmd += [f'-DCMAKE_EXE_LINKER_FLAGS="{flags}"']
 
         if self.config_opts:
             cmake_cmd += self.config_opts
@@ -609,11 +618,11 @@ class Autotools(ConfigureBasedBuildSystem):
     def emit_build_commands(self, environ):
         prepare_cmd = []
         if self.srcdir:
-            prepare_cmd += ['cd %s' % self.srcdir]
+            prepare_cmd += [f'cd {self.srcdir}']
 
         if self.builddir:
-            prepare_cmd += ['mkdir -p %s' % self.builddir,
-                            'cd %s' % self.builddir]
+            prepare_cmd += [f'mkdir -p {self.builddir}',
+                            f'cd {self.builddir}']
 
         if self.builddir:
             configure_cmd = [os.path.join(
@@ -631,28 +640,33 @@ def emit_build_commands(self, environ):
         fflags   = self._fflags(environ)
         ldflags  = self._ldflags(environ)
         if cc:
-            configure_cmd += ['CC="%s"' % cc]
+            configure_cmd += [f'CC="{cc}"']
 
         if cxx:
-            configure_cmd += ['CXX="%s"' % cxx]
+            configure_cmd += [f'CXX="{cxx}"']
 
         if ftn:
-            configure_cmd += ['FC="%s"' % ftn]
+            configure_cmd += [f'FC="{ftn}"']
 
         if cppflags:
-            configure_cmd += ['CPPFLAGS="%s"' % ' '.join(cppflags)]
+            flags = ' '.join(cppflags)
+            configure_cmd += [f'CPPFLAGS="{flags}"']
 
         if cflags:
-            configure_cmd += ['CFLAGS="%s"' % ' '.join(cflags)]
+            flags = ' '.join(cflags)
+            configure_cmd += [f'CFLAGS="{flags}"']
 
         if cxxflags:
-            configure_cmd += ['CXXFLAGS="%s"' % ' '.join(cxxflags)]
+            flags = ' '.join(cxxflags)
+            configure_cmd += [f'CXXFLAGS="{flags}"']
 
         if fflags:
-            configure_cmd += ['FCFLAGS="%s"' % ' '.join(fflags)]
+            flags = ' '.join(fflags)
+            configure_cmd += [f'FCFLAGS="{flags}"']
 
         if ldflags:
-            configure_cmd += ['LDFLAGS="%s"' % ' '.join(ldflags)]
+            flags = ' '.join(ldflags)
+            configure_cmd += [f'LDFLAGS="{flags}"']
 
         if self.config_opts:
             configure_cmd += self.config_opts
@@ -897,6 +911,29 @@ def prepare_cmds(self):
         return cmds
 
 
+class CustomBuild(BuildSystem):
+    '''Custom build system.
+
+    This build system backend allows users to use custom build scripts to
+    build the test code. It does not do any interpretation of the current test
+    environment and it simply runs the supplied :attr:`commands`.
+
+    Users should use this build system with caution, because environment
+    management, reproducibility and any potential side effects are all
+    controlled by the user's custom build system.
+
+    .. versionadded:: 3.11.0
+    '''
+
+    #: The commands to run for building the test code.
+    #:
+    #: :type: :class:`List[str]`
+    commands = variable(typ.List[str])
+
+    def emit_build_commands(self, environ):
+        return self.commands
+
+
 class BuildSystemField(fields.TypedField):
     def __init__(self, fieldname, *other_types):
         super().__init__(fieldname, BuildSystem, *other_types)
@@ -906,6 +943,6 @@ def __set__(self, obj, value):
             try:
                 value = globals()[value]()
             except KeyError:
-                raise ValueError('unknown build system: %s' % value) from None
+                raise ValueError(f'unknown build system: {value}') from None
 
         super().__set__(obj, value)

reframe/core/schedulers/slurm.py

@@ -7,6 +7,7 @@
 import glob
 import itertools
 import re
+import shlex
 import time
 from argparse import ArgumentParser
 from contextlib import suppress
@@ -305,6 +306,8 @@ def filternodes(self, job, nodes):
         # create a mutable list out of the immutable SequenceView that
         # sched_access is
         options = job.sched_access + job.options + job.cli_options
+        options = list(itertools.chain.from_iterable(shlex.split(opt)
+                                                     for opt in options))
         option_parser = ArgumentParser()
         option_parser.add_argument('--reservation')
         option_parser.add_argument('-p', '--partition')

reframe/frontend/cli.py

@@ -1037,6 +1037,14 @@ def _case_failed(t):
 
         if options.distribute:
             node_map = getallnodes(options.distribute, parsed_job_options)
+            # In the distribute option we need to remove the cli options that
+            # begin with '--nodelist' and '-w', so that we don't include them
+            # in the job scripts
+            parsed_job_options = list(
+                filter(lambda x: (not x.startswith('-w') and
+                                  not x.startswith('--nodelist')),
+                       parsed_job_options)
+            )
             testcases = distribute_tests(testcases, node_map)
             testcases_all = testcases
 
@@ -1249,7 +1257,9 @@ def module_unuse(*paths):
             success = True
             if runner.stats.failed():
                 success = False
-                runner.stats.print_failure_report(printer, not options.distribute)
+                runner.stats.print_failure_report(
+                    printer, not options.distribute
+                )
                 if options.failure_stats:
                     runner.stats.print_failure_stats(printer)
 

reframe/frontend/distribute_tests.py

@@ -15,7 +15,7 @@
 from reframe.frontend.executors import generate_testcases
 
 
-def getallnodes(state='all', jobs_cli_options=[]):
+def getallnodes(state='all', jobs_cli_options=None):
     rt = runtime.runtime()
     nodes = {}
     for part in rt.system.partitions:

unittests/test_buildsystems.py

@@ -25,7 +25,8 @@ def environ():
                            ldflags=['-dynamic'])
 
 
-@pytest.fixture(params=['Autotools', 'CMake', 'Make', 'SingleSource'])
+@pytest.fixture(params=['Autotools', 'CMake',
+                        'CustomBuild', 'Make', 'SingleSource'])
 def build_system(request):
     return bs.__dict__[request.param]()
 
@@ -127,6 +128,12 @@ def _emit_from_env_SingleSource(build_system, environ):
     assert expected == build_system.emit_build_commands(environ)
 
 
+def _emit_from_env_CustomBuild(build_system, environ):
+    build_system.commands = ['./custom-configure --foo', 'make']
+    expected = ['./custom-configure --foo', 'make']
+    assert expected == build_system.emit_build_commands(environ)
+
+
 def _emit_from_buildsystem_Make(build_system_with_flags, environ):
     build_system_with_flags.makefile = 'Makefile_foo'
     build_system_with_flags.srcdir = 'foodir'
@@ -188,6 +195,12 @@ def _emit_from_buildsystem_SingleSource(build_system_with_flags, environ):
     assert expected == build_system_with_flags.emit_build_commands(environ)
 
 
+def _emit_from_buildsystem_CustomBuild(build_system_with_flags, environ):
+    build_system_with_flags.commands = ['./custom-configure --foo', 'make']
+    expected = ['./custom-configure --foo', 'make']
+    assert expected == build_system_with_flags.emit_build_commands(environ)
+
+
 def _emit_no_env_defaults_Make(build_system, environ):
     build_system.flags_from_environ = False
     assert ['make -j 1'] == build_system.emit_build_commands(environ)
@@ -213,6 +226,12 @@ def _emit_no_env_defaults_SingleSource(build_system, environ):
             build_system.emit_build_commands(environ))
 
 
+def _emit_no_env_defaults_CustomBuild(build_system, environ):
+    build_system.commands = ['./custom-configure --foo', 'make']
+    expected = ['./custom-configure --foo', 'make']
+    assert expected == build_system.emit_build_commands(environ)
+
+
 @pytest.fixture(params=['C', 'C++', 'Fortran', 'CUDA'])
 def lang(request):
     return request.param