Merge pull request #52 from tbenthompson/linting_ci

CI pipeline and linting

Author: tbenthompson

.github/workflows/lint_and_test.yml

@@ -0,0 +1,34 @@
+name: Lint and Test
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+    - name: Install dependencies
+      run: |
+        conda env update --file environment.yml --name base
+    - name: Lint with flake8
+      run: |
+        flake8 . 
+    - name: Check formatting with black
+      run: |
+        black . 
+    - name: Test with pytest
+      run: |
+        pip install --no-use-pep517 --no-deps --disable-pip-version-check -e .
+        py.test
+

cppimport/__init__.py

@@ -1,5 +1,10 @@
-from cppimport.config import set_quiet, force_rebuild, file_exts,\
-    turn_off_strict_prototypes, set_rtld_flags
+from cppimport.config import (
+    set_quiet,
+    force_rebuild,
+    file_exts,
+    turn_off_strict_prototypes,
+    set_rtld_flags,
+)
 from cppimport.importer import imp, imp_from_filepath, build
 from cppimport.templating import setup_pybind11
 from cppimport.importer import imp as cppimport

cppimport/build_module.py

@@ -16,6 +16,7 @@
 else:
     import io
 
+
 @contextlib.contextmanager
 def stdchannel_redirected(stdchannel):
     """
@@ -30,13 +31,15 @@ def stdchannel_redirected(stdchannel):
     finally:
         setattr(sys, stdchannel, old)
 
+
 # Subclass setuptools Extension to add a parameter specifying where the shared
 # library should be placed after being compiled
 class ImportCppExt(setuptools.Extension):
     def __init__(self, libdest, *args, **kwargs):
         self.libdest = libdest
         setuptools.Extension.__init__(self, *args, **kwargs)
 
+
 # Subclass setuptools build_ext to put the compiled shared library in the
 # appropriate place in the source tree.
 class BuildImportCppExt(setuptools.command.build_ext.build_ext):
@@ -48,15 +51,23 @@ def copy_extensions_to_source(self):
             dest_filename = os.path.join(ext.libdest, os.path.basename(filename))
 
             distutils.file_util.copy_file(
-                src_filename, dest_filename,
-                verbose = self.verbose, dry_run = self.dry_run
+                src_filename, dest_filename, verbose=self.verbose, dry_run=self.dry_run
             )
 
+
 # Patch for parallel compilation with distutils
-# From: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils
-def parallel_compile(self, sources, output_dir = None, macros = None,
-        include_dirs = None, debug = 0, extra_preargs = None, extra_postargs = None,
-        depends = None):
+# From: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils # noqa: E501
+def parallel_compile(
+    self,
+    sources,
+    output_dir=None,
+    macros=None,
+    include_dirs=None,
+    debug=0,
+    extra_preargs=None,
+    extra_postargs=None,
+    depends=None,
+):
 
     # these lines are copied directly from distutils.ccompiler.CCompiler
     macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
@@ -70,6 +81,7 @@ def parallel_compile(self, sources, output_dir = None, macros = None,
     try:
         import multiprocessing
         import multiprocessing.pool
+
         N = multiprocessing.cpu_count()
     except (ImportError, NotImplementedError):
         pass
@@ -86,68 +98,66 @@ def _single_compile(obj):
         # print("took " + str(end - start) + " to compile " + str(obj))
 
     # imap is evaluated on demand, converting to list() forces execution
-    list(multiprocessing.pool.ThreadPool(N).imap(_single_compile,objects))
+    list(multiprocessing.pool.ThreadPool(N).imap(_single_compile, objects))
     return objects
 
+
 def build_module(module_data):
     build_path = tempfile.mkdtemp()
 
-    full_module_name = module_data['fullname']
-    filepath = module_data['filepath']
-    cfg = module_data['cfg']
+    full_module_name = module_data["fullname"]
+    filepath = module_data["filepath"]
+    cfg = module_data["cfg"]
 
-    module_data['abs_include_dirs'] = [
-        make_absolute(module_data['filedirname'], d)
-        for d in cfg.get('include_dirs', [])
+    module_data["abs_include_dirs"] = [
+        make_absolute(module_data["filedirname"], d)
+        for d in cfg.get("include_dirs", [])
     ] + [os.path.dirname(filepath)]
-    module_data['abs_library_dirs'] = [
-        make_absolute(module_data['filedirname'], d)
-        for d in cfg.get('library_dirs', [])
+    module_data["abs_library_dirs"] = [
+        make_absolute(module_data["filedirname"], d)
+        for d in cfg.get("library_dirs", [])
     ]
-    module_data['dependency_dirs'] = (
-        module_data['abs_include_dirs'] + [module_data['filedirname']]
-    )
-    module_data['extra_source_filepaths'] = [
-        make_absolute(module_data['filedirname'], s)
-        for s in cfg.get('sources', [])
+    module_data["dependency_dirs"] = module_data["abs_include_dirs"] + [
+        module_data["filedirname"]
+    ]
+    module_data["extra_source_filepaths"] = [
+        make_absolute(module_data["filedirname"], s) for s in cfg.get("sources", [])
     ]
 
     ext = ImportCppExt(
         os.path.dirname(filepath),
         full_module_name,
-        language = 'c++',
-        sources = (
-            module_data['extra_source_filepaths'] +
-            [module_data['rendered_src_filepath']]
+        language="c++",
+        sources=(
+            module_data["extra_source_filepaths"]
+            + [module_data["rendered_src_filepath"]]
         ),
-        include_dirs = module_data['abs_include_dirs'],
-        extra_compile_args = cfg.get('extra_compile_args', []),
-        extra_link_args = cfg.get('extra_link_args', []),
-        library_dirs = module_data['abs_library_dirs'],
-        libraries = cfg.get('libraries', [])
+        include_dirs=module_data["abs_include_dirs"],
+        extra_compile_args=cfg.get("extra_compile_args", []),
+        extra_link_args=cfg.get("extra_link_args", []),
+        library_dirs=module_data["abs_library_dirs"],
+        libraries=cfg.get("libraries", []),
     )
 
-    args = ['build_ext', '--inplace']
-    args.append('--build-temp=' + build_path)
-    args.append('--build-lib=' + build_path)
+    args = ["build_ext", "--inplace"]
+    args.append("--build-temp=" + build_path)
+    args.append("--build-lib=" + build_path)
 
     if cppimport.config.quiet:
-        args.append('-q')
+        args.append("-q")
     else:
-        args.append('-v')
+        args.append("-v")
 
     setuptools_args = dict(
-        name = full_module_name,
-        ext_modules = [ext],
-        script_args = args,
-        cmdclass = {
-            'build_ext': BuildImportCppExt
-        }
+        name=full_module_name,
+        ext_modules=[ext],
+        script_args=args,
+        cmdclass={"build_ext": BuildImportCppExt},
     )
 
     # Monkey patch in the parallel compiler if requested.
     py33orgreater = sys.version_info[0] >= 3 and sys.version_info[1] >= 3
-    parallelize = cfg.get('parallel') and py33orgreater
+    parallelize = cfg.get("parallel") and py33orgreater
     if parallelize:
         old_compile = distutils.ccompiler.CCompiler.compile
         distutils.ccompiler.CCompiler.compile = parallel_compile

cppimport/checksum.py

@@ -7,25 +7,28 @@
 from cppimport.filepaths import make_absolute
 
 
-_TAG = b'cppimport'
-_FMT = struct.Struct('q' + str(len(_TAG)) + 's')
+_TAG = b"cppimport"
+_FMT = struct.Struct("q" + str(len(_TAG)) + "s")
+
 
 def calc_cur_checksum(file_lst, module_data):
     text = b""
     for filepath in file_lst:
         try:
-            with open(filepath, 'rb') as f:
+            with open(filepath, "rb") as f:
                 text += f.read()
         except OSError as e:
             cppimport.config.quiet_print(
                 "Checksummed file not found while checking cppimport checksum "
-                "(%s); rebuilding." % e)
+                "(%s); rebuilding." % e
+            )
             return None
     return hashlib.md5(text).hexdigest()
 
+
 def load_checksum_trailer(module_data):
     try:
-        with open(module_data['ext_path'], 'rb') as f:
+        with open(module_data["ext_path"], "rb") as f:
             f.seek(-_FMT.size, 2)
             json_len, tag = _FMT.unpack(f.read(_FMT.size))
             if tag != _TAG:
@@ -42,33 +45,37 @@ def load_checksum_trailer(module_data):
     except ValueError:
         cppimport.config.quiet_print(
             "Failed to load checksum trailer info from already existing "
-            "compiled extension; rebuilding.")
+            "compiled extension; rebuilding."
+        )
         return None, None
     return deps, old_checksum
 
+
 # Use a checksum to see if the file has been changed since the last compilation
 def is_checksum_current(module_data):
     deps, old_checksum = load_checksum_trailer(module_data)
     if old_checksum is None:
         return False  # Already logged error in load_checksum_trailer.
     return old_checksum == calc_cur_checksum(deps, module_data)
 
+
 def save_checksum_trailer(module_data, dep_filepaths, cur_checksum):
     # We can just append the checksum to the shared object; this is effectively
     # legal (see e.g. https://stackoverflow.com/questions/10106447).
-    dump = json.dumps([dep_filepaths, cur_checksum]).encode('ascii')
+    dump = json.dumps([dep_filepaths, cur_checksum]).encode("ascii")
     dump += _FMT.pack(len(dump), _TAG)
-    with open(module_data['ext_path'], 'ab') as file:
+    with open(module_data["ext_path"], "ab") as file:
         file.write(dump)
 
+
 def checksum_save(module_data):
     dep_filepaths = (
         [
-            make_absolute(module_data['filedirname'], d)
-            for d in module_data['cfg'].get('dependencies', [])
-        ] +
-        module_data['extra_source_filepaths'] +
-        [module_data['filepath']]
+            make_absolute(module_data["filedirname"], d)
+            for d in module_data["cfg"].get("dependencies", [])
+        ]
+        + module_data["extra_source_filepaths"]
+        + [module_data["filepath"]]
     )
     cur_checksum = calc_cur_checksum(dep_filepaths, module_data)
     save_checksum_trailer(module_data, dep_filepaths, cur_checksum)

cppimport/config.py

@@ -2,29 +2,35 @@
 
 quiet = True
 should_force_rebuild = False
-file_exts = ['.cpp', '.c']
+file_exts = [".cpp", ".c"]
 rtld_flags = ctypes.RTLD_LOCAL
 
+
 def set_quiet(to):
     global quiet
     quiet = to
 
-def force_rebuild(to = True):
+
+def force_rebuild(to=True):
     global should_force_rebuild
     should_force_rebuild = to
 
+
 def quiet_print(a):
     global quiet
     if not quiet:
         print(a)
 
+
 def turn_off_strict_prototypes():
     import distutils.sysconfig
+
     cfg_vars = distutils.sysconfig.get_config_vars()
     for key, value in cfg_vars.items():
         if type(value) == str:
             cfg_vars[key] = value.replace("-Wstrict-prototypes", "")
 
+
 def set_rtld_flags(flags):
     global rtld_flags
     rtld_flags = flags

cppimport/cpprun.py

@@ -1,4 +1,3 @@
-import time
 import os
 import sys
 import argparse
@@ -21,17 +20,16 @@
 
 
 def cpp_run():
-    parser = argparse.ArgumentParser(description='Run a C++ file with cppimport')
-    parser.add_argument('filename', help = 'The file to run.')
+    parser = argparse.ArgumentParser(description="Run a C++ file with cppimport")
+    parser.add_argument("filename", help="The file to run.")
     parser.add_argument(
-        '--add_main_caller', '-m',
-        action = 'store_true',
-        help = 'Add a pybind11 function that will call your main()'
+        "--add_main_caller",
+        "-m",
+        action="store_true",
+        help="Add a pybind11 function that will call your main()",
     )
     parser.add_argument(
-        '--verbose', '-v',
-        action = 'store_true',
-        help = 'Tell me everything!'
+        "--verbose", "-v", action="store_true", help="Tell me everything!"
     )
     args = parser.parse_args()
 
@@ -44,11 +42,11 @@ def cpp_run():
     module_name, file_extension = os.path.splitext(filebasename)
 
     if args.add_main_caller:
-        cpprun_dir = '.cpprunfiles'
+        cpprun_dir = ".cpprunfiles"
         if not os.path.exists(cpprun_dir):
             os.makedirs(cpprun_dir)
         src = os.path.join(cpprun_dir, filebasename)
-        open(src, 'w').write(open(filename, 'r').read() + footer)
+        open(src, "w").write(open(filename, "r").read() + footer)
         sys.path.append(cpprun_dir)
     else:
         sys.path.append(filedir)
@@ -60,5 +58,5 @@ def cpp_run():
     module.main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     cpp_run()

cppimport/filepaths.py

@@ -1,5 +1,6 @@
 import os
 
+
 def make_absolute(this_dir, s):
     if os.path.isabs(s):
         return s