Working container build

Author: Shillaker

docker/cpython.dockerfile

@@ -21,12 +21,15 @@ RUN inv cpython
 # Set up crossenv
 RUN ./bin/crossenv_setup.sh
 
+# Install cross-compiled python packages
+RUN . ./cross_venv/bin/activate && inv libs.install
+
+# TODO - enable these once the MXNet/ Horovod work is completed
 # Build mxnet
-RUN inv mxnet
+# RUN inv mxnet
 
-# Build cpython
-RUN inv cpython
+# Install experimental pacakges
+# RUN . ./cross_venv/bin/activate && inv libs.install --experimental
 
-# Cross-compile packages
-RUN . ./cross_venv/bin/activate && inv libs.install
-RUN . ./cross_venv/bin/activate && inv libs.install --experimental
+# Copy files into place
+RUN inv runtime

tasks/__init__.py

@@ -6,6 +6,7 @@
     git,
     libs,
     mxnet,
+    runtime,
 )
 
 ns = Collection(
@@ -14,4 +15,5 @@
     git,
     libs,
     mxnet,
+    runtime,
 )

tasks/runtime.py

@@ -0,0 +1,99 @@
+import glob
+
+from os.path import join, exists
+from shutil import rmtree, copytree
+from subprocess import run
+from os import makedirs, remove
+
+from invoke import task
+
+from tasks.env import PROJ_ROOT, THIRD_PARTY_DIR
+
+FAASM_RUNTIME_ROOT = "/usr/local/faasm/runtime_root"
+CPYTHON_SRC = join(THIRD_PARTY_DIR, "cpython")
+CPYTHON_INSTALL_DIR = join(CPYTHON_SRC, "install", "wasm")
+CROSSENV_WASM_DIR = join(PROJ_ROOT, "cross_venv", "cross")
+
+
+def _glob_remove(glob_pattern, recursive=False, directory=False):
+    print("Recursive remove: {}".format(glob_pattern))
+    for filename in glob.iglob(glob_pattern, recursive=recursive):
+        print("Removing {}".format(filename))
+        if directory:
+            rmtree(filename)
+        else:
+            remove(filename)
+
+
+def _clear_pyc_files(dir_path):
+    pyc_glob = "{}/**/*.pyc".format(dir_path)
+    _glob_remove(pyc_glob, recursive=True)
+
+    pycache_glob = "{}/**/__pycache__".format(dir_path)
+    _glob_remove(pycache_glob, recursive=True, directory=True)
+
+
+@task(default=True)
+def copy(ctx):
+    """
+    Copies the CPython archive and all installed modules from the cpython build
+    into the Faasm runtime root
+    """
+
+    include_root = join(FAASM_RUNTIME_ROOT, "include")
+    lib_root = join(FAASM_RUNTIME_ROOT, "lib")
+
+    if not exists(lib_root):
+        print("Creating {}".format(lib_root))
+        makedirs(lib_root)
+
+    if not exists(include_root):
+        print("Creating {}".format(include_root))
+        makedirs(include_root)
+
+    include_src_dir = join(CPYTHON_INSTALL_DIR, "include", "python3.8")
+    lib_src_dir = join(CPYTHON_INSTALL_DIR, "lib", "python3.8")
+    site_packages_src_dir = join(
+        CROSSENV_WASM_DIR, "lib", "python3.8", "site-packages"
+    )
+
+    include_dest_dir = join(include_root, "python3.8")
+    lib_dest_dir = join(lib_root, "python3.8")
+    site_packages_dest_dir = join(lib_dest_dir, "site-packages")
+
+    # Clear out pyc files
+    print("Clearing out pyc files")
+    _clear_pyc_files(lib_src_dir)
+    _clear_pyc_files(site_packages_src_dir)
+
+    # Remove dirs to be replaced by those we copy in
+    if exists(include_dest_dir):
+        print("Removing {}".format(include_dest_dir))
+        rmtree(include_dest_dir)
+
+    if exists(lib_dest_dir):
+        print("Removing {}".format(lib_dest_dir))
+        rmtree(lib_dest_dir)
+
+    # Copy CPython includes
+    print("Copying {} to {}".format(include_src_dir, include_dest_dir))
+    copytree(include_src_dir, include_dest_dir)
+
+    # Copy CPython libs
+    print("Copying {} to {}".format(lib_src_dir, lib_dest_dir))
+    copytree(lib_src_dir, lib_dest_dir)
+
+    # Copy cross-compiled modules
+    if not exists(site_packages_dest_dir):
+        makedirs(site_packages_dest_dir)
+
+    print(
+        "Copying {} to {}".format(
+            site_packages_src_dir, site_packages_dest_dir
+        )
+    )
+    run(
+        "cp -r {}/* {}/".format(site_packages_src_dir, site_packages_dest_dir),
+        shell=True,
+        check=True,
+    )