Merge pull request #2 from faasm/toolchain-split

Self-contained container build

Author: Shillaker

.github/workflows/build.yml

@@ -0,0 +1,33 @@
+name: Build
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Get the latest code"
+        uses: actions/checkout@v2
+      - name: "Submodules"
+        run: "git submodule update --init --recursive"
+      - name: "Install requirements"
+        run: "pip3 install -r requirements.txt"
+      - name: "Run Black"
+        run: "python3 -m black --check $(git ls-files '*.py')"
+      - name: "Set up QEMU"
+        uses: docker/setup-qemu-action@v1
+      - name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      - name: "Build the sysroot container"
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: false
+          context: .
+          file: docker/cpython.dockerfile
+

.github/workflows/release.yml

@@ -0,0 +1,36 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Get the code"
+        uses: actions/checkout@v2
+      - name: "Submodules"
+        run: "git submodule update --init --recursive"
+      - name: "Get tag version"
+        run: echo "TAG_VERSION=${GITHUB_REF#refs/tags/v*}" >> $GITHUB_ENV
+      - name: "Print tag version"
+        run: echo ${{ env.TAG_VERSION }}
+      - name: "Set up QEMU"
+        uses: docker/setup-qemu-action@v1
+      - name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v1
+      - name: "Log in to DockerHub"
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: "Build and push sysroot container"
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          file: docker/cpython.dockerfile
+          context: .
+          tags: faasm/cpython:${{ env.TAG_VERSION }}

.gitmodules

@@ -21,6 +21,3 @@
 	path = third-party/mxnet
     url = https://github.com/faasm/incubator-mxnet
     branch = faasm
-[submodule "third-party/libffi"]
-	path = third-party/libffi
-	url = https://github.com/faasm/libffi.git

README.md

@@ -31,6 +31,15 @@ CPython is built statically, some notes on this process can be found
 Several of the code changes to CPython and numpy were borrowed from 
 [pyodide](https://github.com/iodide-project/pyodide).
 
+## Releasing
+
+This repo gets built as a container, `faasm/cpython`. If you want to release a
+new version, you can:
+
+- Update the version in `VERSION`
+- Run `git.tag`
+- Let the Github action do the rest
+
 ## Set-up
 
 ### Submodules

VERSION

@@ -0,0 +1 @@
+0.0.6

bin/crossenv_setup.sh

@@ -11,9 +11,6 @@ BUILD_PYTHON_BIN=/usr/local/faasm/python3.8/bin
 BUILD_PYTHON=${BUILD_PYTHON_BIN}/python3.8
 BUILD_PIP=${BUILD_PYTHON_BIN}/pip3.8
 
-FAASM_DIR=/usr/local/code/faasm
-FAASMCLI=${FAASM_DIR}/faasmcli
-
 # Install the build machine python dependencies
 CROSSENV_SRC_DIR=${PROJ_ROOT}/third-party/crossenv
 echo "Installing crossenv from ${CROSSENV_SRC_DIR}"
@@ -27,11 +24,6 @@ ${BUILD_PIP} install cython
 echo "Installing invoke"
 ${BUILD_PIP} install invoke
 
-echo "Installing Faasmcli"
-pushd ${FAASMCLI} >> /dev/null
-${BUILD_PIP} install .
-popd >> /dev/null
-
 # Run the set-up script
 pushd ${PROJ_ROOT} >> /dev/null
 ${BUILD_PYTHON} bin/crossenv_setup.py

docker/cpython.dockerfile

@@ -0,0 +1,35 @@
+FROM faasm/sysroot:0.0.10
+
+RUN apt install -y \
+    libssl-dev \
+    ninja-build
+
+WORKDIR /code/faasm-cpython
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt
+
+# Install build Python (careful with Docker cache here)
+COPY bin/install_build_python.sh bin/install_build_python.sh
+RUN ./bin/install_build_python.sh
+
+# Add the rest of the code
+COPY . .
+
+# Build CPython
+RUN inv cpython
+
+# Set up crossenv
+RUN ./bin/crossenv_setup.sh
+
+# Install cross-compiled python packages
+RUN . ./cross_venv/bin/activate && inv libs.install
+
+# TODO - enable these once the MXNet/ Horovod work is completed
+# Build mxnet
+# RUN inv mxnet
+
+# Install experimental pacakges
+# RUN . ./cross_venv/bin/activate && inv libs.install --experimental
+
+# Copy files into place
+RUN inv runtime

requirements.txt

@@ -0,0 +1,2 @@
+black
+invoke

tasks/__init__.py

@@ -1,15 +1,19 @@
 from invoke import Collection
 
 from . import (
+    container,
     cpython,
+    git,
     libs,
     mxnet,
-    ffi,
+    runtime,
 )
 
 ns = Collection(
+    container,
     cpython,
+    git,
     libs,
     mxnet,
-    ffi,
+    runtime,
 )

tasks/container.py

@@ -0,0 +1,38 @@
+from invoke import task
+from os.path import join
+
+from faasmtools.docker import (
+    build_container,
+    push_container,
+)
+
+from tasks.env import (
+    PROJ_ROOT,
+    get_version,
+)
+
+CONTAINER_IMAGE = "faasm/cpython"
+DOCKERFILE = join(PROJ_ROOT, "docker", "cpython.dockerfile")
+
+
+def _get_tag():
+    tag_name = "{}:{}".format(CONTAINER_IMAGE, get_version())
+    return tag_name
+
+
+@task(default=True)
+def build(ctx, nocache=False, push=False):
+    """
+    Build current version of the cpython container
+    """
+    build_container(
+        _get_tag(), DOCKERFILE, PROJ_ROOT, nocache=nocache, push=push
+    )
+
+
+@task
+def push(ctx):
+    """
+    Push the current version of the cpython container
+    """
+    push_container(_get_tag())