LLVM 18 (#33)

* gh: bump minor code version

* third-party: remove unsupported submodules

* tasks: bump to llvm 17 for clang formatting

* remove unnecessary references to apps

* cpython: use lllvm-17 for the native build

* cpp: point to llvm-17 branch

* cpython: fix func compilation

* cpp: bump

* cpython: set the target in the func compilation

* cpp: bump dep

* cpython: use llvm native version for native compilation

* nits

* nits: clang-format-17 checks passing

* cpp: bump to latest version

* cpp: bump after merge to main

Author: csegarragonz

.clang-format

@@ -10,4 +10,16 @@ AlwaysBreakAfterReturnType: None
 
 IndentWidth: 4
 DerivePointerAlignment: false
+
+# Precise control over braces alignment
+BreakBeforeBraces: Custom
+BraceWrapping:
+    AfterClass: true
+    AfterEnum: true
+    AfterExternBlock: true
+    AfterFunction: true
+    AfterStruct: true
+    AfterUnion: true
+    SplitEmptyFunction: false
+    SplitEmptyRecord: false
 ---

.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     container:
-      image: faasm.azurecr.io/cpython:0.3.1
+      image: faasm.azurecr.io/cpython:0.4.0
       credentials:
         username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
         password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}
@@ -44,7 +44,7 @@ jobs:
       REDIS_QUEUE_HOST: redis
       REDIS_STATE_HOST: redis
     container:
-      image: faasm.azurecr.io/cpython:0.3.1
+      image: faasm.azurecr.io/cpython:0.4.0
       credentials:
         username: ${{ secrets.ACR_SERVICE_PRINCIPAL_ID }}
         password: ${{ secrets.ACR_SERVICE_PRINCIPAL_PASSWORD }}

.gitmodules

@@ -2,22 +2,10 @@
     path = third-party/cpython
     url = https://github.com/faasm/cpython
     branch = faasm
-[submodule "third-party/numpy"]
-    path = third-party/numpy
-    url = https://github.com/faasm/numpy
-    branch = faasm
 [submodule "third-party/crossenv"]
     path = third-party/crossenv
     url = https://github.com/faasm/crossenv
     branch = faasm
-[submodule "third-party/horovod"]
-    path = third-party/horovod
-    url = https://github.com/faasm/horovod
-    branch = faasm
-[submodule "third-party/mxnet"]
-    path = third-party/mxnet
-    url = https://github.com/faasm/incubator-mxnet
-    branch = faasm
 [submodule "third-party/cpp"]
     path = third-party/cpp
     url = https://github.com/faasm/cpp

Dockerfile

@@ -1,4 +1,4 @@
-FROM faasm.azurecr.io/cpp-sysroot:0.2.4
+FROM faasm.azurecr.io/cpp-sysroot:0.4.0
 ARG FAASM_PYTHON_VERSION
 
 SHELL ["/bin/bash", "-c"]
@@ -41,11 +41,6 @@ RUN cd /code/python \
     && source ./venv/bin/activate \
     && inv modules.install
 
-
-# TODO: enable these once the MXNet/ Horovod work is completed
-# Build mxnet
-# RUN inv mxnet
-
 # TODO: Install experimental pacakges
 # RUN . ./cross_venv/bin/activate && inv libs.install --experimental
 

VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

crossenv/tasks/modules.py

@@ -25,6 +25,7 @@
 MODIFIED_LIBS_EXPERIMENTAL = {
     # 15/11/2022 - Move Numpy back to experimental as either the upgrades
     # to the CPP toolchain or LLVM 13 break the build
+    # (experimental <=> not yet supported)
     "numpy": {
         "env": {"NPY_NUM_BUILD_JOBS": USABLE_CPUS},
     },

func/cpp/CMakeLists.txt

@@ -3,27 +3,32 @@ project(faasm-func)
 
 set(CMAKE_CXX_STANDARD 17)
 
-if (CMAKE_SYSTEM_NAME STREQUAL "Wasm")
+if (CMAKE_SYSTEM_NAME STREQUAL "WASI")
+    # Set the right target so that we look in the right sysroot
+    set(CMAKE_C_FLAGS "--target=$ENV{FAASM_WASM_TRIPLE} ${CMAKE_C_FLAGS}")
+    set(CMAKE_CXX_FLAGS "--target=$ENV{FAASM_WASM_TRIPLE} ${CMAKE_CXX_FLAGS}")
+
     # For Faasm functions, we add `_faasm_zygote` as an exported symbol
     set(CMAKE_EXE_LINKER_FLAGS
         "${CMAKE_EXE_LINKER_FLAGS} -Xlinker --export=_faasm_zygote"
     )
+
     message(STATUS "Detected wasm build (sysroot=$ENV{FAASM_WASM_SYSROOT})")
     set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
 endif ()
 
-if (CMAKE_SYSTEM_NAME STREQUAL "Wasm")
+if (CMAKE_SYSTEM_NAME STREQUAL "WASI")
     set(FAASM_FUNC_LIBS faasm emscripten c-printscan-long-double)
 
-    set(PYTHON_LIBRARIES ${CMAKE_SYSROOT}/lib/wasm32-wasi/libpython3.8.a)
-    set(PYTHON_INCLUDE_DIRS ${CMAKE_SYSROOT}/include/python3.8)
+    set(PYTHON_LIBRARIES $ENV{FAASM_WASM_LIB_INSTALL_DIR}/libpython3.8.a)
+    set(PYTHON_INCLUDE_DIRS $ENV{FAASM_WASM_HEADER_INSTALL_DIR}/python3.8)
 else ()
     find_package(PythonLibs)
 endif()
 
 add_executable(py_func pyinit.c pyinit.h py_func.cpp)
 
-if (CMAKE_SYSTEM_NAME STREQUAL "Wasm")
+if (CMAKE_SYSTEM_NAME STREQUAL "WASI")
     target_link_libraries(py_func ${FAASM_FUNC_LIBS} ${PYTHON_LIBRARIES} ffi)
     target_include_directories(py_func PUBLIC ${PYTHON_INCLUDE_DIRS})
 

tasks/cpython.py

@@ -1,14 +1,8 @@
 from copy import copy as deep_copy
 from faasmctl.util.upload import upload_wasm
-from faasmtools.build import (
-    FAASM_BUILD_ENV_DICT,
-    WASM_HEADER_INSTALL,
-    WASM_LIB_INSTALL,
-    WASM_WASI_LIBC_LDFLAGS,
-    build_config_cmd,
-)
+from faasmtools.build import build_config_cmd, get_faasm_build_env_dict
 from faasmtools.compile_util import wasm_cmake, wasm_copy_upload
-from faasmtools.env import WASM_DIR
+from faasmtools.env import LLVM_NATIVE_VERSION, WASM_DIR
 from invoke import task
 from os import environ, makedirs
 from os.path import join, exists
@@ -54,16 +48,18 @@
         "PATH": PATH_ENV_VAR,
     }
 )
-ENV_VARS.update(FAASM_BUILD_ENV_DICT)
+ENV_VARS.update(get_faasm_build_env_dict(is_threads=True))
 
 LIB_SRC_DIR = join(CPYTHON_INSTALL_DIR, "lib")
 LIB_DEST_DIR = join(FAASM_RUNTIME_ROOT, "lib")
 
 LIBPYTHON_SRC_PATH = join(LIB_SRC_DIR, "libpython3.8.a")
-LIBPYTHON_DEST_PATH = join(WASM_LIB_INSTALL, "libpython3.8.a")
+LIBPYTHON_DEST_PATH = join(
+    ENV_VARS["FAASM_WASM_LIB_INSTALL_DIR"], "libpython3.8.a"
+)
 
 INCLUDE_SRC_DIR = join(CPYTHON_INSTALL_DIR, "include", "python3.8")
-INCLUDE_DEST_DIR = join(WASM_HEADER_INSTALL, "python3.8")
+INCLUDE_DEST_DIR = join(ENV_VARS["FAASM_WASM_HEADER_INSTALL_DIR"], "python3.8")
 
 # See the CPython docs for more info:
 # - General: https://devguide.python.org/setup/#compile-and-build
@@ -96,21 +92,26 @@ def wasm(ctx, clean=False, noconf=False, nobuild=False):
     # relevant in the module builds.
 
     # Link in extra wasi-libc long double support (see wasi-libc docs)
-    link_libs = ["-lfaasm"] + WASM_WASI_LIBC_LDFLAGS
-    link_libs = " ".join(link_libs)
+    link_libs = "-lfaasm " + ENV_VARS["FAASM_WASM_STATIC_LINKER_FLAGS"]
+    # link_libs = " ".join(link_libs)
 
     # Configure
     configure_cmd = build_config_cmd(
+        ENV_VARS,
         [
             "CONFIG_SITE=./config.site",
             "READELF=true",
             "./configure",
             'LIBS="{}"'.format(link_libs),
+            "--build=wasm32",
+            "--host={}".format(ENV_VARS["FAASM_WASM_TRIPLE"]),
             "--disable-ipv6",
             "--disable-shared",
             "--prefix={}".format(CPYTHON_INSTALL_DIR),
             "--with-system-ffi",
-        ]
+        ],
+        # Do not set the --host flag as we want to use the wasi-threads target
+        conf_args=False,
     )
 
     if not noconf:
@@ -133,8 +134,8 @@ def wasm(ctx, clean=False, noconf=False, nobuild=False):
     _run_cpython_cmd("bininstall", ["make", "bininstall"])
 
     # Prepare destinations
-    makedirs(WASM_HEADER_INSTALL, exist_ok=True)
-    makedirs(WASM_LIB_INSTALL, exist_ok=True)
+    makedirs(ENV_VARS["FAASM_WASM_HEADER_INSTALL_DIR"], exist_ok=True)
+    makedirs(ENV_VARS["FAASM_WASM_LIB_INSTALL_DIR"], exist_ok=True)
 
     rmtree(INCLUDE_DEST_DIR, ignore_errors=True)
 
@@ -187,12 +188,13 @@ def native(ctx, clean=False):
     run("wget {}".format(tar_url), shell=True, check=True, cwd=workdir)
     run("tar -xf {}".format(tar_name), shell=True, check=True, cwd=workdir)
 
+    llvm_native_version_major = LLVM_NATIVE_VERSION.split(".")[0]
     workdir = join(workdir, PYTHON_VERSION)
     native_configure_cmd = [
-        'CC="clang-13"',
-        'CXX="clang++-13"',
+        'CC="clang-{}"'.format(llvm_native_version_major),
+        'CXX="clang++-{}"'.format(llvm_native_version_major),
         'CFLAGS="-O3 -DANSI"',
-        'LD="clang-13"',
+        'LD="clang-{}"'.format(llvm_native_version_major),
         "./configure",
         "--prefix={}".format(PYTHON_INSTALL_DIR),
     ]
@@ -230,7 +232,14 @@ def func(ctx, clean=False, debug=False):
     wasm_file = join(func_build_dir, "{}.wasm".format(CPYTHON_FUNC_NAME))
 
     # Build and install the wasm
-    wasm_cmake(func_dir, func_build_dir, CPYTHON_FUNC_NAME, clean, debug)
+    wasm_cmake(
+        func_dir,
+        func_build_dir,
+        CPYTHON_FUNC_NAME,
+        clean,
+        debug,
+        is_threads=True,
+    )
     wasm_copy_upload(CPYTHON_FUNC_USER, CPYTHON_FUNC_NAME, wasm_file)
 
 

tasks/format_code.py

@@ -1,3 +1,4 @@
+from faasmtools.env import LLVM_NATIVE_VERSION
 from invoke import task
 from tasks.env import PROJ_ROOT
 from subprocess import run
@@ -51,8 +52,9 @@ def format(ctx, check=False):
     )
 
     clang_cmd = [
-        "clang-format-13",
+        "clang-format-{}".format(LLVM_NATIVE_VERSION.split(".")[0]),
         "--dry-run --Werror" if check else "-i",
+        "-style=file",
         " ".join(files_to_check),
     ]
     clang_cmd = " ".join(clang_cmd)