Fixes to CI/CD (#100)

* format: add task to format code

* gha: use new formatting task and actions checkout

* fix: remove hardcoded paths in workon file

* fix: update venv to remove hardcoded path

* gha: fix failing test

* set safe directory

* gha: add step to cancel previously running workflows for the same pr

Author: csegarragonz

.github/workflows/tests.yml

@@ -8,34 +8,27 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
 
 jobs:
+  # Cancel previous running actions for the same PR
+  cancel_previous:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Workflow Action
+        uses: styfle/[email protected]
+
   checks:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-20.04
     container:
       image: faasm/cpp-sysroot:0.1.8
-    defaults:
-      run:
-        working-directory: /code/cpp
     steps:
       # --- Update code ---
-      - name: "Fetch ref"
-        run: git fetch origin ${GITHUB_REF}:ci-branch
-      - name: "Check out branch"
-        run: git checkout --force ci-branch
-      - name: "Update Faabric submodule"
-        run: git submodule update --init -f third-party/faabric
-      - name: "Update FFI submodule"
-        run: git submodule update --init -f third-party/libffi
-      - name: "Update FFmpeg submodule"
-        run: git submodule update --init -f third-party/FFmpeg
-      - name: "Update ImageMagick submodule"
-        run: git submodule update --init -f third-party/ImageMagick
-      - name: "Update libpng submodule"
-        run: git submodule update --init -f third-party/libpng
-      - name: "Update zlib submodule"
-        run: git submodule update --init -f third-party/zlib
-      - name: "Update wasi-libc submodule"
-        run: git submodule update --init -f third-party/wasi-libc
+      - name: "Checkout code"
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      # See actions/checkout#766
+      - name: "Set the GH workspace as a safe git directory"
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
       # --- Build libraries to wasm ---
       - name: "Build libc"
         run: ./bin/inv_wrapper.sh libc
@@ -70,10 +63,5 @@ jobs:
       - name: "Build libfaasmpi native"
         run: ./bin/inv_wrapper.sh libfaasmpi --native --clean
       # --- Formatting ---
-      - name: "Check python"
-        run: source venv/bin/activate && ./bin/check_python.sh
-        shell: bash
-      - name: "Run C/C++ formatting"
-        run: ./bin/run_clang_format.sh
-      - name: "Check no formatting changes"
-        run: git diff --exit-code -- :^third-party
+      - name: "Format code"
+        run: ./bin/inv_wrapper.sh format-code --check

bin/create_venv.sh

@@ -11,7 +11,7 @@ VENV_PATH="undetected"
 if [[ -z "$CPP_DOCKER" ]]; then
     VENV_PATH="${PROJ_ROOT}/venv-bm"
 else
-    VENV_PATH="/code/cpp/venv"
+    VENV_PATH="${PROJ_ROOT}/venv"
 fi
 
 PIP=${VENV_PATH}/bin/pip3

bin/workon.sh

@@ -4,23 +4,17 @@
 # Container-specific settings
 # ----------------------------
 
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
+PROJ_ROOT="${THIS_DIR}/.."
 MODE="undetected"
 if [[ -z "$CPP_DOCKER" ]]; then
-
-    THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
-    PROJ_ROOT="${THIS_DIR}/.."
-    VENV_PATH="${PROJ_ROOT}/venv-bm"
-
     # Normal terminal
+    VENV_PATH="${PROJ_ROOT}/venv-bm"
     MODE="terminal"
 else
-    # Running inside the container, we know the project root
-    PROJ_ROOT="/code/cpp"
-    VENV_PATH="/code/cpp/venv"
-
+    VENV_PATH="${PROJ_ROOT}/venv"
     # Use containerised redis
     alias redis-cli="redis-cli -h redis"
-
     MODE="container"
 fi
 

tasks/__init__.py

@@ -3,6 +3,7 @@
 from . import clapack
 from . import docker
 from . import ffmpeg
+from . import format_code
 from . import func
 from . import git
 from . import imagemagick
@@ -20,6 +21,7 @@
 ns = Collection(
     clapack,
     docker,
+    format_code,
     ffmpeg,
     func,
     git,

tasks/format_code.py

@@ -0,0 +1,68 @@
+from invoke import task
+from tasks.util.env import PROJ_ROOT
+from subprocess import run
+
+
+@task(default=True)
+def format(ctx, check=False):
+    """
+    Format Python and C++ code
+    """
+    # ---- Python formatting ----
+
+    files_to_check = (
+        run(
+            'git ls-files -- "*.py"',
+            shell=True,
+            check=True,
+            cwd=PROJ_ROOT,
+            capture_output=True,
+        )
+        .stdout.decode("utf-8")
+        .split("\n")[:-1]
+    )
+    black_cmd = [
+        "python3 -m black",
+        "{}".format("--check" if check else ""),
+        " ".join(files_to_check),
+    ]
+    black_cmd = " ".join(black_cmd)
+    run(black_cmd, shell=True, check=True, cwd=PROJ_ROOT)
+
+    flake8_cmd = [
+        "python3 -m flake8",
+        "{}".format("--format" if not check else ""),
+        " ".join(files_to_check),
+    ]
+    flake8_cmd = " ".join(flake8_cmd)
+    run(flake8_cmd, shell=True, check=True, cwd=PROJ_ROOT)
+
+    # ---- C/C++ formatting ----
+
+    files_to_check = (
+        run(
+            'git ls-files -- "*.h" "*.cpp" "*.c"',
+            shell=True,
+            check=True,
+            cwd=PROJ_ROOT,
+            capture_output=True,
+        )
+        .stdout.decode("utf-8")
+        .split("\n")[:-1]
+    )
+
+    clang_cmd = "clang-format-10 -i {}".format(" ".join(files_to_check))
+    run(clang_cmd, shell=True, check=True, cwd=PROJ_ROOT)
+
+    # ---- Append newlines to C/C++ files if not there ----
+
+    for f in files_to_check:
+        # Append opens the file from the end, but there is no easy way to read
+        # just one character backwards unless you open the file as a byte
+        # stream, which then makes it very involved to compare against the
+        # newline character
+        with open(f, "a+") as fh:
+            fh.seek(0)
+            read = fh.read()
+            if read[-1] != "\n":
+                fh.write("\n")

tasks/util/env.py

@@ -0,0 +1,3 @@
+from os.path import dirname, realpath
+
+PROJ_ROOT = dirname(dirname(dirname(realpath(__file__))))