Merge pull request #1 from faasm/horovod-2

Add Horovod and MXNet

Author: Shillaker

.gitmodules

@@ -13,4 +13,14 @@
 [submodule "third-party/pyfaasm"]
 	path = third-party/pyfaasm
     url = https://github.com/faasm/pyfaasm.git
-
+[submodule "third-party/horovod"]
+	path = third-party/horovod
+    url = https://github.com/faasm/horovod
+    branch = faasm
+[submodule "third-party/mxnet"]
+	path = third-party/mxnet
+    url = https://github.com/faasm/incubator-mxnet
+    branch = faasm
+[submodule "third-party/libffi"]
+	path = third-party/libffi
+	url = https://github.com/faasm/libffi.git

README.md

@@ -3,8 +3,12 @@
 This build cross-compiles CPython and a number of Python modules to WebAssembly
 for use in [Faasm](https://github.com/faasm/faasm).
 
-You can see a list of the installed modules in the
-[`install_modules.sh`](scripts/install_modules.sh) script.
+You can try to install arbitrary packages, but the ones that definitely work 
+can be listed with:
+
+```bash
+inv libs.show
+```
 
 The build uses the [Faasm
 toolchain](https://github.com/faasm/faasm-toolchain) to cross-compile both 
@@ -44,15 +48,21 @@ compiling and the version on the build machine need to match _exactly_.
 To set up the relevant build machine python:
 
 ```
-cd ansible
-ansible-playbook python3_8.yml
+./bin/install_build_python.sh
 ```
 
 This will install Python at `/usr/local/faasm/python3.8`.
 
 When cross-compiling we _have_ to use this Python when running commands and
 scripts on the build machine (not any other Python that might be installed).
 
+### Upgrading Pip
+
+Do **not** upgrade Pip in the build machine copy of Python.
+
+The versions of Pip and Python for wasm and the build machine must match 
+exactly.
+
 ### Building CPython to WebAssembly
 
 You can build CPython by running (with optional `--clean`):
@@ -79,7 +89,7 @@ we use [crossenv](https://github.com/benfogle/crossenv).
 To set up crossenv:
 
 ```
-./scripts/crossenv_setup.sh
+./bin/crossenv_setup.sh
 ```
 
 You can then activate with:
@@ -91,7 +101,7 @@ You can then activate with:
 From inside the virtual environment, you can inspect the set-up with:
 
 ```
-python scripts/sanity_check.py | less
+python bin/sanity_check.py | less
 ```
 
 This will display the environment used to install Python modules (including the
@@ -105,18 +115,28 @@ environment:
 
 - Modify the CPython build (see `tasks.py`)
 - Rerun the CPython build (`inv cpython --clean`) 
-- Rebuild the crossenv (`./scripts/crossenv_setup.sh`) 
-- Enter the crossenv and inspect the environment with `scripts/sanity_check.py`
+- Rebuild the crossenv (`./bin/crossenv_setup.sh`) 
+- Enter the crossenv and inspect the environment with `bin/sanity_check.py`
 
 ## Modules
 
 With the crossenv activated, we can build modules with normal `pip`.
 
-To install all the modules, you can run:
+There is a wrapper script that will apply modifications if we know about them.
+To run this you must first have the cross-env activated as described above.
 
-```
-# NOTE: run this from a clean terminal, _not_ through the Faasm CLI
-./scripts/install_modules.sh
+```bash
+# Install all supported modules
+inv libs.install 
+
+# Install experimental modules
+inv libs.install --experimental
+
+# Install numpy
+inv libs.install --name numpy
+
+# (Attempt) to install arbitrary module
+inv libs.install --name <module_name>
 ```
 
 Libraries will then be installed to 
@@ -129,8 +149,52 @@ debugger.
 
 You can also set `DISTUTILS_DEBUG=1` to get distutils to print more info.
 
+## Experimental modules
+
+Some of the modules are experimental, these may require some extra set-up.
+
+### MXNet and Horovod
+
+To install the Python MXNet module we first need to cross-compile the MXNet
+shared library:
+
+```
+# Update all submodules
+cd third-party/mxnet
+git submodule update --init
+cd ../horovod
+git submodule update --init
+
+# Run our MXNet cross-compile (outside crossenv)
+cd ../..
+inv mxnet
+```
+
+Then we can install mxnet and horovod:
+
+```
+. ./cross_venv/bin/activate
+inv libs.install --name mxnet
+inv libs.install --name horovod
+```
+
+### Cleaning and uninstalling 
+
+```
+# Clean then rebuild
+inv mxnet --clean
+
+# Uninstall mxnet
+inv mxnet.uninstall
+```
+
 ## Running in Faasm
 
 See the [Faasm python
 docs](https://github.com/faasm/faasm/blob/master/docs/python.md).
 
+## BLAS and LAPACK
+
+Faasm's normal BLAS and LAPACK support using CLAPACK should be picked up by
+numpy due to the addition of the [site.cfg](../third-party/numpy/site.cfg).
+

ansible/ansible.cfg

@@ -6,8 +6,9 @@
 
 PROJ_ROOT = dirname(dirname(abspath(__file__)))
 
-WASM_CPYTHON = join(PROJ_ROOT, "third-party", "cpython", "install", "wasm",
-                    "bin", "python3.8")
+WASM_CPYTHON = join(
+    PROJ_ROOT, "third-party", "cpython", "install", "wasm", "bin", "python3.8"
+)
 
 CROSSENV_VENV_DIR = join(PROJ_ROOT, "cross_venv")
 CROSSENV_SYSROOT = "/usr/local/faasm/llvm-sysroot"
@@ -21,19 +22,18 @@ def main():
 
     # Set logging level
     level = logging.DEBUG
-    logging.basicConfig(level=level, format='%(levelname)s: %(message)s')
+    logging.basicConfig(level=level, format="%(levelname)s: %(message)s")
 
     # Invoke the cross-env builder
     print("Building crossenv at {}".format(CROSSENV_VENV_DIR))
     builder = CrossEnvBuilder(
-        with_cross_pip=True,
-        with_build_pip=True,
         host_python=WASM_CPYTHON,
         host_sysroot=CROSSENV_SYSROOT,
+        with_cross_pip=True,
+        with_build_pip=True,
     )
     builder.create(CROSSENV_VENV_DIR)
 
 
 if __name__ == "__main__":
     main()
-

ansible/python3_8.yml

@@ -11,20 +11,30 @@ BUILD_PYTHON_BIN=/usr/local/faasm/python3.8/bin
 BUILD_PYTHON=${BUILD_PYTHON_BIN}/python3.8
 BUILD_PIP=${BUILD_PYTHON_BIN}/pip3.8
 
+FAASM_DIR=/usr/local/code/faasm
+FAASMCLI=${FAASM_DIR}/faasmcli
+
 # Install the build machine python dependencies
 CROSSENV_SRC_DIR=${PROJ_ROOT}/third-party/crossenv
 echo "Installing crossenv from ${CROSSENV_SRC_DIR}"
 pushd ${CROSSENV_SRC_DIR} >> /dev/null
 ${BUILD_PIP} install -e .
+popd >> /dev/null
 
 echo "Installing cython"
 ${BUILD_PIP} install cython
-popd >> /dev/null
 
-pushd ${PROJ_ROOT} >> /dev/null
+echo "Installing invoke"
+${BUILD_PIP} install invoke
+
+echo "Installing Faasmcli"
+pushd ${FAASMCLI} >> /dev/null
+${BUILD_PIP} install .
+popd >> /dev/null
 
 # Run the set-up script
-${BUILD_PYTHON} scripts/crossenv_setup.py
+pushd ${PROJ_ROOT} >> /dev/null
+${BUILD_PYTHON} bin/crossenv_setup.py
 
 # Enter the env and print details
 source cross_venv/bin/activate
@@ -35,4 +45,3 @@ echo "python3.8 is: $(which python3.8)"
 
 popd >> /dev/null
 
-

scripts/crossenv_setup.py bin/crossenv_setup.pyscripts/crossenv_setup.py renamed to bin/crossenv_setup.py

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+PYTHON_VER=Python-3.8.2
+TAR_NAME=${PYTHON_VER}.tgz
+TAR_URL=https://www.python.org/ftp/python/3.8.2/${TAR_NAME}
+
+INSTALL_PREFIX=/usr/local/faasm/python3.8
+PYTHON_BIN=${INSTALL_PREFIX}/bin/python3.8
+PIP_BIN=${INSTALL_PREFIX}/bin/pip3.8
+
+N_PROC=$(nproc --ignore 1)
+
+pushd /tmp/ >> /dev/null
+
+# Download the TAR file and extract
+wget ${TAR_URL}
+tar -xf ${TAR_NAME}
+
+# Configure and install
+pushd ${PYTHON_VER} >> /dev/null
+./configure --prefix=${INSTALL_PREFIX}
+make -j ${N_PROC} altinstall
+popd >> /dev/null
+
+# Print version
+${PYTHON_BIN} --version
+${PIP_BIN} --version
+
+popd >> /dev/null
+

scripts/crossenv_setup.sh bin/crossenv_setup.shscripts/crossenv_setup.sh renamed to bin/crossenv_setup.sh

@@ -7,8 +7,11 @@ def main():
     # Sysconfig checks
     print("Platform: {}".format(sysconfig.get_platform()))
     print("Python version: {}".format(sysconfig.get_python_version()))
-    print("Current installation scheme: {}".format(
-        sysconfig._get_default_scheme()))
+    print(
+        "Current installation scheme: {}".format(
+            sysconfig._get_default_scheme()
+        )
+    )
     print("Paths")
     pprint(sysconfig.get_paths())
     print("Variables")

bin/install_build_python.sh

@@ -0,0 +1 @@
+ffi_call