Merge pull request #452 from vkarak/feat/autotools-build-system

[feat] Add support for Autotools-based projects

Author: vkarak

docs/advanced.rst

@@ -123,7 +123,7 @@ Add a configuration step before compiling the code
 
 It is often the case that a configuration step is needed before compiling a code with ``make``.
 To address this kind of projects, ReFrame aims to offer specific abstractions for "configure-make"-style build systems.
-Currently, it supports only `CMake <https://cmake.org/>`__-based projects through its :class:`CMake <reframe.core.buildsystems.CMake>` build system.
+It supports `CMake-based <https://cmake.org/>`__ projects through the :class:`CMake <reframe.core.buildsystems.CMake>` build system, as well as `Autotools-based <https://www.gnu.org/software/automake/>`__ projects through the :class:`Autotools <reframe.core.buildsystems.Autotools>` build system.
 
 For other build systems, you can achieve the same effect using the :class:`Make <reframe.core.buildsystems.Make>` build system and the :attr:`prebuild_cmd <reframe.core.pipeline.RegressionTest.prebuild_cmd>` for performing the configuration step.
 The following code snippet will configure a code with ``./custom_configure`` before invoking ``make``:

reframe/core/buildsystems.py

@@ -181,7 +181,7 @@ class Make(BuildSystem):
 
     .. code::
 
-      make -j [N] [-f MAKEFILE] [-C SRCDIR] CC='X' CXX='X' FC='X' NVCC='X' CPPFLAGS='X' CFLAGS='X' CXXFLAGS='X' FFLAGS='X' LDFLAGS='X' OPTIONS
+      make -j [N] [-f MAKEFILE] [-C SRCDIR] CC='X' CXX='X' FC='X' NVCC='X' CPPFLAGS='X' CFLAGS='X' CXXFLAGS='X' FCFLAGS='X' LDFLAGS='X' OPTIONS
 
     The compiler and compiler flags variables will only be passed if they are
     not :class:`None`.
@@ -276,7 +276,7 @@ def emit_build_commands(self, environ):
             cmd_parts += ["CXXFLAGS='%s'" % ' '.join(cxxflags)]
 
         if fflags is not None:
-            cmd_parts += ["FFLAGS='%s'" % ' '.join(fflags)]
+            cmd_parts += ["FCFLAGS='%s'" % ' '.join(fflags)]
 
         if ldflags is not None:
             cmd_parts += ["LDFLAGS='%s'" % ' '.join(ldflags)]
@@ -301,7 +301,7 @@ class SingleSource(BuildSystem):
       the source file and pick the correct compiler.
       See also the :attr:`SingleSource.lang` attribute.
     - ``CPPFLAGS`` are the preprocessor flags and are passed to any compiler.
-    - ``XFLAGS`` is any of ``CFLAGS``, ``CXXFLAGS`` or ``FFLAGS`` depending on
+    - ``XFLAGS`` is any of ``CFLAGS``, ``CXXFLAGS`` or ``FCFLAGS`` depending on
       the programming language of the source file.
     - ``SRCFILE`` is the source file to be compiled.
       This is set up automatically by the framework.
@@ -443,17 +443,8 @@ def _guess_language(self, filename):
             return 'CUDA'
 
 
-class CMake(BuildSystem):
-    """A build system for compiling CMake-based projects.
-
-    This build system will emit the following commands:
-
-    1. Create a build directory if :attr:`builddir` is not :class:`None` and
-       change to it.
-    2. Invoke ``cmake`` to configure the project by setting the corresponding
-       CMake flags for compilers and compiler flags.
-    3. Issue ``make`` to compile the code.
-    """
+class ConfigureBasedBuildSystem(BuildSystem):
+    """Abstract base class for configured-based build systems."""
 
     #: The top-level directory of the code.
     #:
@@ -476,13 +467,39 @@ class CMake(BuildSystem):
     #: :default: ``[]``
     config_opts = fields.TypedListField('config_opts', str)
 
+    #: Options to be passed to the subsequent ``make`` invocation.
+    #:
+    #: :type: :class:`list[str]`
+    #: :default: ``[]``
+    make_opts = fields.TypedListField('make_opts', str)
+
+    #: Same as for the :attr:`Make` build system.
+    #:
+    #: :type: integer
+    #: :default: :class:`None`
+    max_concurrency = fields.IntegerField('max_concurrency', allow_none=True)
+
     def __init__(self):
         super().__init__()
         self.srcdir = None
         self.builddir = None
         self.config_opts = []
+        self.make_opts = []
         self.max_concurrency = None
 
+
+class CMake(ConfigureBasedBuildSystem):
+    """A build system for compiling CMake-based projects.
+
+    This build system will emit the following commands:
+
+    1. Create a build directory if :attr:`builddir` is not :class:`None` and
+       change to it.
+    2. Invoke ``cmake`` to configure the project by setting the corresponding
+       CMake flags for compilers and compiler flags.
+    3. Issue ``make`` to compile the code.
+    """
+
     def _combine_flags(self, cppflags, xflags):
         if cppflags is None:
             return xflags
@@ -548,9 +565,85 @@ def emit_build_commands(self, environ):
         if self.max_concurrency is not None:
             make_cmd += [str(self.max_concurrency)]
 
+        if self.make_opts:
+            make_cmd += self.make_opts
+
         return prepare_cmd + [' '.join(cmake_cmd), ' '.join(make_cmd)]
 
 
+class Autotools(ConfigureBasedBuildSystem):
+    """A build system for compiling Autotools-based projects.
+
+    This build system will emit the following commands:
+
+    1. Create a build directory if :attr:`builddir` is not :class:`None` and
+       change to it.
+    2. Invoke ``configure`` to configure the project by setting the corresponding
+       flags for compilers and compiler flags.
+    3. Issue ``make`` to compile the code.
+    """
+
+    def emit_build_commands(self, environ):
+        prepare_cmd = []
+        if self.srcdir:
+            prepare_cmd += ['cd %s' % self.srcdir]
+
+        if self.builddir:
+            prepare_cmd += ['mkdir -p %s' % self.builddir,
+                            'cd %s' % self.builddir]
+
+        if self.builddir:
+            configure_cmd = [os.path.join(
+                os.path.relpath('.', self.builddir), 'configure')]
+        else:
+            configure_cmd = ['./configure']
+
+        cc = self._cc(environ)
+        cxx = self._cxx(environ)
+        ftn = self._ftn(environ)
+        nvcc = self._nvcc(environ)
+        cppflags = self._cppflags(environ)
+        cflags   = self._cflags(environ)
+        cxxflags = self._cxxflags(environ)
+        fflags   = self._fflags(environ)
+        ldflags  = self._ldflags(environ)
+        if cc is not None:
+            configure_cmd += ["CC='%s'" % cc]
+
+        if cxx is not None:
+            configure_cmd += ["CXX='%s'" % cxx]
+
+        if ftn is not None:
+            configure_cmd += ["FC='%s'" % ftn]
+
+        if cppflags is not None:
+            configure_cmd += ["CPPFLAGS='%s'" % ' '.join(cppflags)]
+
+        if cflags is not None:
+            configure_cmd += ["CFLAGS='%s'" % ' '.join(cflags)]
+
+        if cxxflags is not None:
+            configure_cmd += ["CXXFLAGS='%s'" % ' '.join(cxxflags)]
+
+        if fflags is not None:
+            configure_cmd += ["FCFLAGS='%s'" % ' '.join(fflags)]
+
+        if ldflags is not None:
+            configure_cmd += ["LDFLAGS='%s'" % ' '.join(ldflags)]
+
+        if self.config_opts:
+            configure_cmd += self.config_opts
+
+        make_cmd = ['make -j']
+        if self.max_concurrency is not None:
+            make_cmd += [str(self.max_concurrency)]
+
+        if self.make_opts:
+            make_cmd += self.make_opts
+
+        return prepare_cmd + [' '.join(configure_cmd), ' '.join(make_cmd)]
+
+
 class BuildSystemField(fields.TypedField):
     """A field representing a build system.
 

reframe/core/pipeline.py

@@ -930,10 +930,16 @@ def compile(self, **compile_opts):
         if os.path.isdir(staged_sourcepath):
             if not self.build_system:
                 # Try to guess the build system
-                if os.path.exists(os.path.join(staged_sourcepath,
-                                               'CMakeLists.txt')):
+                cmakelists = os.path.join(staged_sourcepath, 'CMakeLists.txt')
+                configure_ac = os.path.join(staged_sourcepath, 'configure.ac')
+                configure_in = os.path.join(staged_sourcepath, 'configure.in')
+                if os.path.exists(cmakelists):
                     self.build_system = 'CMake'
                     self.build_system.builddir = 'rfm_build'
+                elif (os.path.exists(configure_ac) or
+                      os.path.exists(configure_in)):
+                    self.build_system = 'Autotools'
+                    self.build_system.builddir = 'rfm_build'
                 else:
                     self.build_system = 'Make'
 

unittests/test_buildsystems.py

@@ -48,7 +48,7 @@ def test_emit_from_env(self):
             "make -f Makefile_foo -C foodir -j 32 CC='gcc' CXX='g++' "
             "FC='gfortran' NVCC='nvcc' CPPFLAGS='-DNDEBUG' "
             "CFLAGS='-Wall -std=c99' CXXFLAGS='-Wall -std=c++11' "
-            "FFLAGS='-Wall' LDFLAGS='-dynamic' FOO=1"
+            "FCFLAGS='-Wall' LDFLAGS='-dynamic' FOO=1"
         ]
         self.assertEqual(expected,
                          self.build_system.emit_build_commands(self.environ))
@@ -62,7 +62,7 @@ def test_emit_from_buildsystem(self):
         expected = [
             "make -f Makefile_foo -C foodir -j CC='cc' CXX='CC' FC='ftn' "
             "NVCC='clang' CPPFLAGS='-DFOO' CFLAGS='-Wall -std=c99 -O3' "
-            "CXXFLAGS='-Wall -std=c++11 -O3' FFLAGS='-Wall -O3' "
+            "CXXFLAGS='-Wall -std=c++11 -O3' FCFLAGS='-Wall -O3' "
             "LDFLAGS='-static' FOO=1"
         ]
         self.assertEqual(expected,
@@ -82,6 +82,7 @@ def test_emit_from_env(self):
         self.build_system.srcdir = 'src'
         self.build_system.builddir = 'build/foo'
         self.build_system.config_opts = ['-DFOO=1']
+        self.build_system.make_opts = ['install']
         self.build_system.max_concurrency = 32
         expected = [
             "cd src",
@@ -94,7 +95,7 @@ def test_emit_from_env(self):
             "-DCMAKE_CXX_FLAGS='-DNDEBUG -Wall -std=c++11' "
             "-DCMAKE_Fortran_FLAGS='-DNDEBUG -Wall' "
             "-DCMAKE_EXE_LINKER_FLAGS='-dynamic' -DFOO=1 ../..",
-            "make -j 32"
+            "make -j 32 install"
 
         ]
         self.assertEqual(expected,
@@ -127,6 +128,55 @@ def test_emit_no_env_defaults(self):
                          self.build_system.emit_build_commands(self.environ))
 
 
+class TestAutotools(_BuildSystemTest, unittest.TestCase):
+    def create_build_system(self):
+        return bs.Autotools()
+
+    def test_emit_from_env(self):
+        self.build_system.srcdir = 'src'
+        self.build_system.builddir = 'build/foo'
+        self.build_system.config_opts = ['FOO=1']
+        self.build_system.make_opts = ['check']
+        self.build_system.max_concurrency = 32
+        expected = [
+            "cd src",
+            "mkdir -p build/foo",
+            "cd build/foo",
+            "../../configure CC='gcc' CXX='g++' FC='gfortran' "
+            "CPPFLAGS='-DNDEBUG' CFLAGS='-Wall -std=c99' "
+            "CXXFLAGS='-Wall -std=c++11' FCFLAGS='-Wall' "
+            "LDFLAGS='-dynamic' FOO=1",
+            "make -j 32 check"
+
+        ]
+        self.assertEqual(expected,
+                         self.build_system.emit_build_commands(self.environ))
+
+    def test_emit_from_buildsystem(self):
+        super().setup_base_buildsystem()
+        self.build_system.builddir = 'build/foo'
+        self.build_system.config_opts = ['FOO=1']
+        self.build_system.max_concurrency = None
+        expected = [
+            "mkdir -p build/foo",
+            "cd build/foo",
+            "../../configure CC='cc' CXX='CC' FC='ftn' "
+            "CPPFLAGS='-DFOO' CFLAGS='-Wall -std=c99 -O3' "
+            "CXXFLAGS='-Wall -std=c++11 -O3' FCFLAGS='-Wall -O3' "
+            "LDFLAGS='-static' FOO=1",
+            "make -j"
+
+        ]
+        print(self.build_system.emit_build_commands(self.environ))
+        self.assertEqual(expected,
+                         self.build_system.emit_build_commands(self.environ))
+
+    def test_emit_no_env_defaults(self):
+        self.build_system.flags_from_environ = False
+        self.assertEqual(['./configure', 'make -j'],
+                         self.build_system.emit_build_commands(self.environ))
+
+
 class TestSingleSource(_BuildSystemTest, unittest.TestCase):
     def create_build_system(self):
         return bs.SingleSource()