Update CVODES compilation procedure (#637)

* Compile CVODES functions with each Stan model

In 2017, model compilation takes ~40-50 seconds. Compiling CVODES
functions with each model adds about 7 seconds.

Closes #209.

* update Sundials in model.py

* Update Sundials building procedure.

* add test for Sundials

* update platform.platform to platform.system and use tmp folder for Windows

* minimal change

* add travis tests

Author: ahartikainen

.appveyor.yml

@@ -96,6 +96,7 @@ test_script:
   - ECHO "Starting the tests"
   - "python -c \"import nose; nose.main()\" --nocapture pystan.tests.test_basic \
      pystan.tests.test_basic_array pystan.tests.test_basic_matrix \
+     pystan.tests.test_cvodes \
      pystan.tests.test_basic_pars pystan.tests.test_ess \
      pystan.tests.test_extract pystan.tests.test_fixed_param \
      pystan.tests.test_linear_regression pystan.tests.test_misc \

.travis.yml

@@ -5,6 +5,7 @@ branches:
   only:
   - master
   - develop
+  - cvodes
 os:
   - linux
 git:

pystan/model.py

@@ -25,7 +25,8 @@
 import time
 
 import distutils
-from distutils.core import Extension
+from distutils.core import Extension, Distribution
+from distutils.command.build_clib import build_clib
 
 import Cython
 from Cython.Build.Inline import _get_build_extension
@@ -39,6 +40,66 @@
 
 logger = logging.getLogger('pystan')
 
+def _build_libraries(self, libraries):
+    """Fork distutils.build_clib.build_libraries to enable compiler flags."""
+    for (lib_name, build_info) in libraries:
+        sources = build_info.get('sources')
+        if sources is None or not isinstance(sources, (list, tuple)):
+            raise DistutilsSetupError(
+                   "in 'libraries' option (library '%s'), "
+                   "'sources' must be present and must be "
+                   "a list of source filenames" % lib_name)
+        sources = list(sources)
+
+        distutils.command.build_clib.log.info("building '%s' library", lib_name)
+
+        macros = build_info.get('macros')
+        include_dirs = build_info.get('include_dirs')
+        extra_postargs = build_info.get('extra_postargs')
+        objects = self.compiler.compile(sources,
+                                        output_dir=self.build_temp,
+                                        macros=macros,
+                                        include_dirs=include_dirs,
+                                        extra_postargs=extra_postargs,
+                                        debug=self.debug)
+
+        self.compiler.create_static_lib(objects, lib_name,
+                                        output_dir=self.build_clib,
+                                        debug=self.debug)
+
+# overwrite the default class method
+build_clib.build_libraries = _build_libraries
+
+def _get_build_clib():
+    """Convenience function to create build_clib class instance."""
+    dist = Distribution()
+    config_files = dist.find_config_files()
+    dist.parse_config_files(config_files)
+    build_clibrary = build_clib(dist)
+    build_clibrary.finalize_options()
+    return build_clibrary
+
+def _build_clib(sources, include_dirs_c, lib_dir, extra_args):
+    """Build C-library."""
+    libraries = [("libsundials", {
+        "sources" : sources,
+        "include_dirs" : include_dirs_c,
+        "extra_postargs" : extra_args,
+    })]
+
+    build_clibrary = _get_build_clib()
+    build_clibrary.libraries = libraries
+    build_clibrary.include_dirs = include_dirs_c
+    build_clibrary.build_temp = lib_dir
+    build_clibrary.build_clib = lib_dir
+    build_clibrary.run()
+
+    objects = []
+    for _, build_info in libraries:
+        obj = build_clibrary.compiler.object_filenames(build_info["sources"], output_dir=lib_dir)
+        objects.extend(obj)
+
+    return objects
 
 def load_module(module_name, module_path):
     """Load the module named `module_name` from  `module_path`
@@ -309,58 +370,95 @@ def __init__(self, file=None, charset='utf-8', model_name="anon_model",
             s = template.safe_substitute(model_cppname=self.model_cppname)
             outfile.write(s)
 
-        ## cvodes sources
+        if extra_compile_args is None:
+            extra_compile_args = []
+
+        distutils.log.set_verbosity(verbose)
+        build_extension = _get_build_extension()
+        ## sundials sources
 
-        # cvodes sources are complied and linked together with the Stan model
+        # sundials sources are compiled and linked together with the Stan model
         # extension module. This is not ideal. In theory, build_clib could be
         # used to build a library once and models would be complied and then
         # linked with this library. This would save 7 or more seconds from every build.
         # But such a strategy is frustrated by the
         # lack of ``install_clib`` functionality in Python's distutils.
         #
         # TODO: numpy provides install_clib functionality, use that.
-        cvodes_src_path = os.path.join(pystan_dir, 'stan', 'lib', 'stan_math', 'lib', 'cvodes_2.9.0', 'src')
-        cvodes_sources = [
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodea.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodea_io.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_band.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_bandpre.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_bbdpre.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_dense.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_diag.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_direct.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_io.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_sparse.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_spbcgs.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_spgmr.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_spils.c'),
-            os.path.join(cvodes_src_path, 'cvodes', 'cvodes_sptfqmr.c'),
-            os.path.join(cvodes_src_path, 'nvec_ser', 'nvector_serial.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_band.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_dense.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_direct.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_iterative.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_math.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_nvector.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_spbcgs.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_spgmr.c'),
-            os.path.join(cvodes_src_path, 'sundials', 'sundials_sptfqmr.c'),
+
+        sundials_excluded = {
+            "nvector/openmp",
+            "nvector/openmpdev",
+            "nvector/parallel",
+            "nvector/parhyp",
+            "nvector/petsc",
+            "nvector/pthreads",
+            "sundials_mpi",
+            "sunlinsol/klu",
+            "sunlinsol/lapack",
+            "sunlinsol/super",
+        }
+
+        sundials_src_path = os.path.join(pystan_dir, 'stan', 'lib', 'stan_math', 'lib', 'sundials_4.1.0', 'src')
+        sundials_sources = []
+        for sun_root, _, sunfiles in os.walk(sundials_src_path):
+            for sunfile in sunfiles:
+                if os.path.splitext(sunfile)[1] == ".c":
+                    path = os.path.join(sun_root, sunfile).replace("\\", "/")
+                    if not any(item in path for item in sundials_excluded):
+                        sundials_sources.append(path)
+
+        include_dirs_c = [
+            lib_dir,
+            os.path.join(pystan_dir, "stan", "lib", "stan_math", "lib", "sundials_4.1.0", "include"),
         ]
 
+        extra_compile_args_c = []
+        if platform.system() == 'Windows':
+            if build_extension.compiler in (None, 'msvc'):
+                logger.warning("MSVC compiler is not supported")
+                extra_compile_args_c.extend([
+                    '/EHsc',
+                    '-DBOOST_DATE_TIME_NO_LIB'
+                ])
+            else:
+                # Windows, but not msvc, likely mingw
+                # fix bug in MingW-W64
+                # use posix threads
+                extra_compile_args_c.extend([
+                    '-O2',
+                    '-Wno-unused-function',
+                    '-Wno-uninitialized',
+                    '-std=c11',
+                    "-D_hypot=hypot",
+                    "-pthread",
+                    "-fexceptions",
+                    "-include",
+                    "stan_sundials_printf_override.hpp",
+                ] + [item for item in extra_compile_args if "std=c++" not in extra_compile_args])
+        else:
+            # linux or macOS
+            extra_compile_args_c.extend([
+                '-O2',
+                '-Wno-unused-function',
+                '-Wno-uninitialized',
+                '-std=c11',
+                "-include",
+                "stan_sundials_printf_override.hpp",
+            ] + [item for item in extra_compile_args if "std=c++" not in extra_compile_args])
+
+        sundials_objects = _build_clib(sundials_sources, include_dirs_c, lib_dir, extra_compile_args_c)
+
         stan_macros = [
             ('BOOST_RESULT_OF_USE_TR1', None),
             ('BOOST_NO_DECLTYPE', None),
             ('BOOST_DISABLE_ASSERTS', None),
         ]
 
-        build_extension = _get_build_extension()
         # compile stan models with optimization (-O2)
         # (stanc is compiled without optimization (-O0) currently, see #33)
-        if extra_compile_args is None:
-            extra_compile_args = []
 
-        if platform.platform().startswith('Win'):
+        if platform.system() == 'Windows':
             if build_extension.compiler in (None, 'msvc'):
                 logger.warning("MSVC compiler is not supported")
                 extra_compile_args = [
@@ -381,6 +479,8 @@ def __init__(self, file=None, charset='utf-8', model_name="anon_model",
                     "-D_hypot=hypot",
                     "-pthread",
                     "-fexceptions",
+                    "-include",
+                    "stan_sundials_printf_override.hpp",
                 ] + extra_compile_args
         else:
             # linux or macOS
@@ -390,15 +490,18 @@ def __init__(self, file=None, charset='utf-8', model_name="anon_model",
                 '-Wno-unused-function',
                 '-Wno-uninitialized',
                 '-std=c++1y',
+                '-include',
+                'stan_sundials_printf_override.hpp',
             ] + extra_compile_args
 
-        distutils.log.set_verbosity(verbose)
         extension = Extension(name=self.module_name,
                               language="c++",
-                              sources=[pyx_file] + cvodes_sources,
+                              sources=[pyx_file],
                               define_macros=stan_macros,
                               include_dirs=include_dirs,
-                              extra_compile_args=extra_compile_args)
+                              extra_objects=sundials_objects,
+                              extra_compile_args=extra_compile_args
+                              )
 
         cython_include_dirs = ['.', pystan_dir]