Combined setup scripts, only deploy once using python 3.8

Author: Sean Narenthiran

.travis.yml

@@ -1,19 +1,29 @@
 dist: bionic
 language: python
 python:
-- '3.5'
-- '3.6'
-- '3.7'
-- '3.8'
+  - '3.5'
+  - '3.6'
+  - '3.7'
+  - '3.8'
 install:
-- pip install -r requirements.txt
-- pip install .
+  - pip install -r requirements.txt
+  - pip install .
 script:
-- pytest tests/
-deploy:
-  provider: pypi
-  username: __token__
-  password:
-    secure: vUbqVM3nP1587JuwCcp4MPutC5qaSQ6tJcMw9d9oahQM/S/tMXOOcZEGAhc8Toh2FesrupxJ4uInJhK1++ydQw832MaUnNG9OsjdSM8fdMTsziLOvIgifFe8vsS2NsGJTE9ty5JIiJYFXBrx187YECu1AttyrzB4Y/QsIIZuUmMHZtRE0tJt4ZPp8KWY91wuq5TcgyXjbRYODlcHscMYoJqZBOIwXcb/rRFciIIrm9hhwJvVfZ1tI3wZ5Lnl3rj9dmBangc+wHa1gFI88A+kewgM3e76Z+07S0va8pXte6oD/X9SYKCPsisctEE0nIqV4pvr7sYyeS1yerNBC+LlmbR0xfU9nA/WcXQszX2Chz9Cz/3jIVV/Z7Jx8ddg78nqvIxb38AduUIB8t68CRpx6MR66QujLyNnARQjOuVA8thOFOmw8fVxBVhIcOpW7AAb9pZKW/m15IqvOgTBLp6x2lPPUi6eSYLR/C/qbPGNNiI3tgxXRBSs/kK+/dMun4pMFJ2oNmMvz1ZgSnMKJnkzjBN7PmzkZuurrgSuzmq61mrCyeaLHDJ/xjj00pV7Nn4mNbywzjNeiouYtgZDontZ/e2m5NU/fsr1YKNbZ0PtQT+ZT7qdiA2R3Hf3hLEVl+kADipvx0VTBg5ZjBOESpIXpa/JMj/JHApMCzQD6bZcU78=
-  on:
-    tags: true
+  - pytest tests/
+
+jobs:
+  include:
+    - stage: deploy
+      python: 3.8
+      install:
+        - pip install -r requirements.txt
+        - pip install .
+      script:
+        - pytest tests/
+      deploy:
+        provider: pypi
+        username: __token__
+        password:
+          secure: vUbqVM3nP1587JuwCcp4MPutC5qaSQ6tJcMw9d9oahQM/S/tMXOOcZEGAhc8Toh2FesrupxJ4uInJhK1++ydQw832MaUnNG9OsjdSM8fdMTsziLOvIgifFe8vsS2NsGJTE9ty5JIiJYFXBrx187YECu1AttyrzB4Y/QsIIZuUmMHZtRE0tJt4ZPp8KWY91wuq5TcgyXjbRYODlcHscMYoJqZBOIwXcb/rRFciIIrm9hhwJvVfZ1tI3wZ5Lnl3rj9dmBangc+wHa1gFI88A+kewgM3e76Z+07S0va8pXte6oD/X9SYKCPsisctEE0nIqV4pvr7sYyeS1yerNBC+LlmbR0xfU9nA/WcXQszX2Chz9Cz/3jIVV/Z7Jx8ddg78nqvIxb38AduUIB8t68CRpx6MR66QujLyNnARQjOuVA8thOFOmw8fVxBVhIcOpW7AAb9pZKW/m15IqvOgTBLp6x2lPPUi6eSYLR/C/qbPGNNiI3tgxXRBSs/kK+/dMun4pMFJ2oNmMvz1ZgSnMKJnkzjBN7PmzkZuurrgSuzmq61mrCyeaLHDJ/xjj00pV7Nn4mNbywzjNeiouYtgZDontZ/e2m5NU/fsr1YKNbZ0PtQT+ZT7qdiA2R3Hf3hLEVl+kADipvx0VTBg5ZjBOESpIXpa/JMj/JHApMCzQD6bZcU78=
+        on:
+          tags: true

build.py

@@ -1,11 +1,80 @@
 #!/usr/bin/env python
+import glob
 import multiprocessing.pool
 import os
+import tarfile
+import urllib.request
+import warnings
 
 from setuptools import setup, find_packages, distutils
 from torch.utils.cpp_extension import BuildExtension
+from torch.utils.cpp_extension import CppExtension, include_paths
 
-this_file = os.path.dirname(__file__)
+
+def download_extract(url, dl_path):
+    if not os.path.isfile(dl_path):
+        # Already downloaded
+        urllib.request.urlretrieve(url, dl_path)
+    if dl_path.endswith(".tar.gz") and os.path.isdir(dl_path[:-len(".tar.gz")]):
+        # Already extracted
+        return
+    tar = tarfile.open(dl_path)
+    tar.extractall('third_party/')
+    tar.close()
+
+
+# Download/Extract openfst, boost
+download_extract('https://github.com/parlance/ctcdecode/releases/download/v1.0/openfst-1.6.7.tar.gz',
+                 'third_party/openfst-1.6.7.tar.gz')
+download_extract('https://github.com/parlance/ctcdecode/releases/download/v1.0/boost_1_67_0.tar.gz',
+                 'third_party/boost_1_67_0.tar.gz')
+
+for file in ['third_party/kenlm/setup.py', 'third_party/ThreadPool/ThreadPool.h']:
+    if not os.path.exists(file):
+        warnings.warn('File `{}` does not appear to be present. Did you forget `git submodule update`?'.format(file))
+
+
+# Does gcc compile with this header and library?
+def compile_test(header, library):
+    dummy_path = os.path.join(os.path.dirname(__file__), "dummy")
+    command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path \
+              + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\""
+    return os.system(command) == 0
+
+
+compile_args = ['-O3', '-DKENLM_MAX_ORDER=6', '-std=c++14', '-fPIC']
+ext_libs = []
+if compile_test('zlib.h', 'z'):
+    compile_args.append('-DHAVE_ZLIB')
+    ext_libs.append('z')
+
+if compile_test('bzlib.h', 'bz2'):
+    compile_args.append('-DHAVE_BZLIB')
+    ext_libs.append('bz2')
+
+if compile_test('lzma.h', 'lzma'):
+    compile_args.append('-DHAVE_XZLIB')
+    ext_libs.append('lzma')
+
+third_party_libs = ["kenlm", "openfst-1.6.7/src/include", "ThreadPool", "boost_1_67_0", "utf8"]
+compile_args.extend(['-DINCLUDE_KENLM', '-DKENLM_MAX_ORDER=6'])
+lib_sources = glob.glob('third_party/kenlm/util/*.cc') + glob.glob('third_party/kenlm/lm/*.cc') + glob.glob(
+    'third_party/kenlm/util/double-conversion/*.cc') + glob.glob('third_party/openfst-1.6.7/src/lib/*.cc')
+lib_sources = [fn for fn in lib_sources if not (fn.endswith('main.cc') or fn.endswith('test.cc'))]
+
+third_party_includes = [os.path.realpath(os.path.join("third_party", lib)) for lib in third_party_libs]
+ctc_sources = glob.glob('ctcdecode/src/*.cpp')
+
+extension = CppExtension(
+    name='ctcdecode._ext.ctc_decode',
+    package=True,
+    with_cuda=False,
+    sources=ctc_sources + lib_sources,
+    include_dirs=third_party_includes + include_paths(),
+    libraries=ext_libs,
+    extra_compile_args=compile_args,
+    language='c++'
+)
 
 
 # monkey-patch for parallel compilation
@@ -40,7 +109,6 @@ def _single_compile(obj):
 
 # hack compile to support parallel compiling
 distutils.ccompiler.CCompiler.compile = parallelCCompile
-import build 
 
 setup(
     name="ctcdecode",
@@ -51,6 +119,6 @@ def _single_compile(obj):
     author_email="[email protected]",
     # Exclude the build files.
     packages=find_packages(exclude=["build"]),
-    ext_modules = [build.extension],
+    ext_modules=[extension],
     cmdclass={'build_ext': BuildExtension}
 )