Skip to content

Commit 8f955ef

Browse files
csegarragonzcsegarragonz
authored
Single source-of-trust for build variables (#102)
* gha: fix failing test * set safe directory * build: make WasiToolchain and func/CMakeLists use env. variables defined in python script * format: remove compilartion warnings in func/demo and func/errors * faasmtools: go through all remaining targets * formant: undo erroneous formatting change * refactor: use the same env vars for CMake and make * make: populate Makefile.envs from env. variables * fix: right flags for executables that will do dynamic linking * format: move comment from cmakelists to build file * fix: add env. variables for clapack task * clapack: remove compilation warnings * undo unnecessary change * gh: bump code again for a new release after tflow branch * fix: for faasm functions add _faasm_zygote as an exported symbol * imagemagick: not track manual files * faasmtools: add dicts for runtime variables * python: add build parameter to get env vars * fix: use env. variables properly * cpp: stringify integer variables * faasmtools: add function to get a string of cmake variables * python: format code
1 parent 7c95c3d commit 8f955ef

File tree

17 files changed

+252
-204
lines changed

17 files changed

+252
-204
lines changed

‎.env‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
SYSROOT_VERSION=0.1.10
2-
SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.1.10
1+
SYSROOT_VERSION=0.1.11
2+
SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.1.11
33
COMPOSE_PROJECT_NAME=cpp-dev

‎.github/workflows/tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
if: github.event.pull_request.draft == false
2020
runs-on: ubuntu-20.04
2121
container:
22-
image: faasm/cpp-sysroot:0.1.10
22+
image: faasm/cpp-sysroot:0.1.11
2323
steps:
2424
# --- Update code ---
2525
- name: "Checkout code"

‎Makefile.envs‎

Lines changed: 23 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,33 @@
1-
#
2-
# IMPORTANT: this file is used by lots of submodules/
3-
# projects so be careful **NOT** to set any global variables
4-
# like CC/CXX. Keep it generic
5-
#
1+
# -----------------------------------------
2+
# IMPORTANT: this file is used by lots of submodules/ projects so be careful
3+
# **NOT** to set any global variables like CC/CXX. Keep it generic. The values
4+
# populating this file can be found in faasmtools/build.py
5+
# -----------------------------------------
66

7-
export WASM_SYSROOT=/usr/local/faasm/llvm-sysroot
8-
export WASM_TOOLCHAIN_BIN=/usr/local/faasm/toolchain/bin
7+
export WASM_SYSROOT=${FAASM_WASM_SYSROOT}
8+
export WASM_TOOLCHAIN_BIN=${FAASM_WASM_INSTALL_DIR}
99

10-
export WASM_CC=$(WASM_TOOLCHAIN_BIN)/clang
11-
export WASM_CXX=$(WASM_TOOLCHAIN_BIN)/clang++
12-
export WASM_CPP=$(WASM_TOOLCHAIN_BIN)/clang-cpp
13-
export WASM_AR=$(WASM_TOOLCHAIN_BIN)/llvm-ar
14-
export WASM_NM=$(WASM_TOOLCHAIN_BIN)/llvm.nm
15-
export WASM_RANLIB=$(WASM_TOOLCHAIN_BIN)/llvm-ranlib
16-
export WASM_LD=$(WASM_TOOLCHAIN_BIN)/wasm-ld
17-
export WASM_LDSHARED=$(WASM_TOOLCHAIN_BIN)/wasm-ld
18-
export WASM_LDFLAGS="-Xlinker --stack-first -Xlinker --no-check-features"
10+
export WASM_CC=${FAASM_WASM_CC}
11+
export WASM_CXX=${FAASM_WASM_CXX}
12+
export WASM_CPP=${FAASM_WASM_CXX}
13+
export WASM_AR=${FAASM_WASM_AR}
14+
export WASM_NM=${FAASM_WASM_NM}
15+
export WASM_RANLIB=${FAASM_WASM_RANLIB}
16+
export WASM_LD=${FAASM_WASM_CC}
17+
export WASM_LDSHARED=${FAASM_WASM_CC}
18+
export WASM_LDFLAGS="${FAASM_WASM_STATIC_LINKER_FLAGS}"
1919

2020
export WASM_BUILD=wasm32
21-
export WASM_HOST=wasm32-unknown-wasi
22-
export WASM_HOST_UNKNOWN=wasm32-unknown-unknown
21+
export WASM_HOST=${FAASM_WASM_HOST_SHARED}
22+
export WASM_HOST_UNKNOWN=${FAASM_WASM_HOST_UNKNOWN}
2323

24-
# Note potential performance gains from unimplemented-simd128 here.
25-
# May not be necessary in future.
26-
# 23/09/2020 - Remove SIMD
27-
# export WASM_CFLAGS_NO_QUOTES=-O3 --sysroot=$(WASM_SYSROOT) -msimd128 -munimplemented-simd128 -mno-atomics -D__faasm
28-
export WASM_CFLAGS_NO_QUOTES=-O3 --sysroot=$(WASM_SYSROOT) -mno-atomics -D__faasm
29-
export WASM_CXXFLAGS_NO_QUOTES=$(WASM_CFLAGS_NO_QUOTES)
24+
export WASM_CFLAGS_NO_QUOTES=${FAASM_WASM_CFLAGS}
25+
export WASM_CXXFLAGS_NO_QUOTES=$(FAASM_WASM_CFLAGS)
3026
export WASM_CFLAGS="$(WASM_CFLAGS_NO_QUOTES)"
3127
export WASM_CXXFLAGS="$(WASM_CXXFLAGS_NO_QUOTES)"
3228

33-
export WASM_SHARED_FLAGS_NO_QUOTES=-D__wasi__ -nostdlib -nostlib++ -fPIC --target=wasm32-unknown-emscripten -Xlinker --no-entry -Xlinker --shared -Xlinker --export-all -Xlinker --no-gc-sections
29+
export WASM_SHARED_FLAGS_NO_QUOTES=${FAASM_WASM_SHARED_LINKER_FLAGS}
3430
export WASM_SHARED_FLAGS="$(WASM_SHARED_FLAGS_NO_QUOTES)"
3531

36-
# Note - order very important here
37-
export WASM_BLAS_LIBS="lapack blas f2c"
32+
# Note - order very important here
33+
export WASM_BLAS_LIBS="${FAASM_WASM_BLAS_LIBS}"

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.1.10
1+
0.1.11

‎WasiToolchain.cmake‎

Lines changed: 16 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@ cmake_minimum_required(VERSION 3.4.0)
44
# A useful reference for this file is the wasi-sdk equivalent:
55
# https://github.com/WebAssembly/wasi-sdk/blob/main/wasi-sdk.cmake
66
#
7-
# BUT there are lots of crucial modifications in here and it's
8-
# used all over the place in this project, so be **very** careful
9-
# when changing it.
7+
# This toolchain file relies on the variable definitions in faasmtools/build.py
8+
# which we pass as environment variables to CMake, Make, or configure
109
# -----------------------------------------
1110

12-
set(INSTALL_DIR /usr/local/faasm/toolchain/bin)
13-
set(FAASM_SYSROOT /usr/local/faasm/llvm-sysroot)
11+
set(INSTALL_DIR $ENV{FAASM_WASM_INSTALL_DIR})
12+
set(FAASM_SYSROOT $ENV{FAASM_WASM_SYSROOT})
1413

1514
set(EXE_SUFFIX "")
1615

@@ -26,84 +25,41 @@ if(FAASM_BUILD_SHARED)
2625
message(STATUS "Faasm building SHARED libraries")
2726

2827
# See note in README about shared libraries targeting Emscripten
29-
set(WASM_TRIPLE wasm32-unknown-emscripten)
28+
set(WASM_TRIPLE $ENV{FAASM_WASM_HOST_SHARED})
3029
else()
3130
message(STATUS "Faasm building STATIC libraries")
3231

33-
set(WASM_TRIPLE wasm32-wasi)
32+
set(WASM_TRIPLE $ENV{FAASM_WASM_HOST_STATIC})
3433
endif()
3534

3635
set(CMAKE_C_COMPILER_TARGET ${WASM_TRIPLE} CACHE STRING "faasm build")
3736
set(CMAKE_CXX_COMPILER_TARGET ${WASM_TRIPLE} CACHE STRING "faasm build")
3837
message(STATUS "Faasm building target ${CMAKE_CXX_COMPILER_TARGET}")
3938

4039
# Specify LLVM toolchain
41-
set(CMAKE_C_COMPILER ${INSTALL_DIR}/clang)
42-
set(CMAKE_CXX_COMPILER ${INSTALL_DIR}/clang++)
43-
set(CMAKE_AR ${INSTALL_DIR}/llvm-ar CACHE STRING "faasm build")
44-
set(CMAKE_NM ${INSTALL_DIR}/llvm-nm CACHE STRING "faasm build")
45-
set(CMAKE_RANLIB ${INSTALL_DIR}/llvm-ranlib CACHE STRING "faasm build")
40+
set(CMAKE_C_COMPILER $ENV{FAASM_WASM_CC})
41+
set(CMAKE_CXX_COMPILER $ENV{FAASM_WASM_CXX})
42+
set(CMAKE_AR $ENV{FAASM_WASM_AR} CACHE STRING "faasm build")
43+
set(CMAKE_NM $ENV{FAASM_WASM_NM} CACHE STRING "faasm build")
44+
set(CMAKE_RANLIB $ENV{FAASM_WASM_RANLIB} CACHE STRING "faasm build")
4645

4746
# We define dynamic linking functions in Faasm
4847
unset(CMAKE_DL_LIBS CACHE)
4948

5049
# Add definition for flagging Faasm
5150
add_definitions(-D__faasm)
5251

53-
# Note: see Clang wasm-specific flags at
54-
# https://clang.llvm.org/docs/ClangCommandLineReference.html#webassembly Note
55-
# the optimisation level. We want to keep vectorization so stick with O3 Also
56-
# note that the optimisation is crucial to getting any kind of decent
57-
# performance.
58-
#
59-
# We must explicitly exclude atomics here just in case we've accidentally
60-
# introduced them upstream. Use of atomics means we can't link things together:
61-
# https://reviews.llvm.org/D59281
62-
#
63-
# The stack-first here is really important to help detect stack overflow
64-
# issues. Without it the stack will overflow into the global data.
65-
# stack-size is also crucial to bigger functions not messing up
66-
67-
# 23/09/2020 - Remove SIMD
68-
# set(FAASM_COMPILER_FLAGS "-O3 -msimd128 -mno-atomics --sysroot=${FAASM_SYSROOT}")
69-
set(FAASM_COMPILER_FLAGS " \
70-
-O3 -mno-atomics \
71-
--sysroot=${FAASM_SYSROOT} \
72-
-m32 \
73-
-DANSI \
74-
-Xlinker --stack-first \
75-
")
52+
set(FAASM_COMPILER_FLAGS $ENV{FAASM_WASM_CFLAGS})
7653

7754
if(FAASM_BUILD_SHARED)
78-
set(FAASM_COMPILER_FLAGS "${FAASM_COMPILER_FLAGS} \
79-
-fPIC \
80-
-nostdlib \
81-
-nostdlib++ \
82-
")
55+
set(FAASM_COMPILER_FLAGS "${FAASM_COMPILER_FLAGS} $ENV{FAASM_WASM_CFLAGS_SHARED}")
8356
endif()
8457

8558
set(CMAKE_SYSROOT ${FAASM_SYSROOT} CACHE STRING "faasm build")
8659
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FAASM_COMPILER_FLAGS}" CACHE STRING "faasm build")
8760
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FAASM_COMPILER_FLAGS}" CACHE STRING "faasm build")
88-
89-
# Shared library flags
90-
# See notes in README about WebAssembly and shared libraries
91-
SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} \
92-
-Xlinker --no-entry \
93-
-Xlinker --shared \
94-
-Xlinker --export-all \
95-
-Xlinker --no-gc-sections \
96-
" CACHE STRING "faasm build")
97-
98-
99-
# This needs to be included to support libcxx with atomics
100-
# -Xlinker --shared-memory
101-
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
102-
-Xlinker --stack-first \
103-
-Xlinker --no-check-features \
104-
-Xlinker --threads \
105-
-Xlinker --max-memory=4294901760 \
106-
" CACHE STRING "faasm build")
61+
set(CMAKE_SHARED_LINKER_FLAGS $ENV{FAASM_WASM_SHARED_LINKER_FLAGS} CACHE STRING "faasm build")
62+
set(CMAKE_EXE_LINKER_FLAGS $ENV{FAASM_WASM_EXE_LINKER_FLAGS} CACHE STRING "faasm build")
10763

10864
# This is important to ensure the right search path
10965
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
@@ -114,6 +70,5 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
11470
set(CMAKE_C_COMPILER_WORKS ON)
11571
set(CMAKE_CXX_COMPILER_WORKS ON)
11672

117-
# Note - order very important here
118-
SET(FAASM_BLAS_LIBS lapack blas f2c)
73+
set(FAASM_BLAS_LIBS $ENV{FAASM_WASM_BLAS_LIBS})
11974

‎faasmtools/build.py‎

Lines changed: 114 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,21 @@
33
from subprocess import run
44
from os import environ
55

6+
# -----------------------------------------
7+
# This file is the single source of truth for all build-related environment
8+
# variables for Faasm and Faasm's C/C++ support. There are several crucial
9+
# modifications, used all across the project, so be **very** careful when
10+
# changing it, and be ready to fix problems.
11+
# -----------------------------------------
12+
613
# Directories
714
FAASM_LOCAL_DIR = environ.get("FAASM_LOCAL_DIR", "/usr/local/faasm")
815
FAASM_NATIVE_DIR = join(FAASM_LOCAL_DIR, "native")
16+
FAASM_CMAKE_ROOT = "/usr/share/cmake-3.24"
917
WASM_SYSROOT = join(FAASM_LOCAL_DIR, "llvm-sysroot")
1018
WASM_HEADER_INSTALL = "{}/include".format(WASM_SYSROOT)
1119
WASM_LIB_INSTALL = "{}/lib/wasm32-wasi".format(WASM_SYSROOT)
12-
WASM_TOOLCHAIN_ROOT = "/usr/local/faasm/toolchain"
20+
WASM_TOOLCHAIN_ROOT = join(FAASM_LOCAL_DIR, "toolchain")
1321
WASM_TOOLCHAIN_TOOLS = join(WASM_TOOLCHAIN_ROOT, "tools")
1422
WASM_TOOLCHAIN_BIN = join(WASM_TOOLCHAIN_ROOT, "bin")
1523

@@ -31,10 +39,37 @@
3139
# Host triple
3240
WASM_BUILD = "wasm32"
3341
WASM_HOST = "wasm32-unknown-wasi"
42+
WASM_HOST_STATIC = "wasm32-wasi"
3443
WASM_HOST_SHARED = "wasm32-unknown-emscripten"
3544
WASM_HOST_UNKNOWN = "wasm32-unknown-unknown"
3645

37-
# CFLAGS
46+
# WASM Constants
47+
# NOTE: we have to set the max memory here but want it to be as close to the
48+
# max (4GB) as possible. If we set it to the max though, this input is
49+
# ignored and the function is set with no maximum memory size. Therefore we
50+
# set it to 4GB - 1 page, i.e. max-memory = (4*1024*1024*1024) - (64*1024)
51+
# = 4294901760.
52+
#
53+
# WARNING: the code may also set limits on the max memory size, so must be
54+
# changed there too (IRModuleCache.h)
55+
#
56+
# This max memory limit can stop benchmarks working so make sure it's big
57+
# and consider restricting in a more specific manner for certain functions.
58+
FAASM_WASM_MAX_MEMORY = 4 * 1024 * 1024 * 1024 - 64 * 1024
59+
FAASM_WASM_STACK_SIZE = 4 * 1024 * 1024
60+
FAASM_WASM_BYTES_PER_PAGE = 65536
61+
FAASM_WASM_INITIAL_MEMORY_SIZE = 4 * FAASM_WASM_STACK_SIZE
62+
63+
# C/C++ compilation flags
64+
# Note: see Clang wasm-specific flags at
65+
# https://clang.llvm.org/docs/ClangCommandLineReference.html#webassembly Note
66+
# the optimisation level. We want to keep vectorization so stick with O3 Also
67+
# note that the optimisation is crucial to getting any kind of decent
68+
# performance.
69+
#
70+
# We must explicitly exclude atomics here just in case we've accidentally
71+
# introduced them upstream. Use of atomics means we can't link things together:
72+
# https://reviews.llvm.org/D59281
3873
WASM_CFLAGS = [
3974
"-O3 -mno-atomics",
4075
"--sysroot={}".format(WASM_SYSROOT),
@@ -45,6 +80,7 @@
4580

4681
WASM_CXXFLAGS = WASM_CFLAGS
4782

83+
# Flags for shared libraries
4884
WASM_CFLAGS_SHARED = [
4985
"-D__wasi__",
5086
"-nostdlib",
@@ -58,7 +94,14 @@
5894
# LDFLAGS
5995
# ----------
6096

97+
# Important symbol names
98+
FAASM_WASM_CTORS_FUNC_NAME = "__wasm_call_ctors"
99+
FAASM_WASM_ZYGOTE_FUNC_NAME = "_faasm_zygote"
100+
61101
# Flags for static libraries
102+
# The stack-first here is really important to help detect stack overflow
103+
# issues. Without it the stack will overflow into the global data.
104+
# stack-size is also crucial to bigger functions not messing up
62105
WASM_LDFLAGS = [
63106
"-static",
64107
"-Xlinker --no-gc-sections",
@@ -71,12 +114,22 @@
71114
"-Xlinker --stack-first",
72115
"-Xlinker --export=__heap_base",
73116
"-Xlinker --export=__data_end",
74-
"-Xlinker --export=__wasm_call_ctors",
75-
"-Xlinker --max-memory=4294901760",
76-
"-Wl,-z,stack-size=4194304 -Wl",
117+
"-Xlinker --export={}".format(FAASM_WASM_CTORS_FUNC_NAME),
118+
"-Xlinker --max-memory={}".format(FAASM_WASM_MAX_MEMORY),
119+
"-Wl,-z,stack-size={} -Wl".format(FAASM_WASM_STACK_SIZE),
120+
"-Wl,--initial-memory={}".format(FAASM_WASM_INITIAL_MEMORY_SIZE),
121+
]
122+
123+
# These are the compiler and linker flags required for functions that will also
124+
# do dynamic linking. We need to export all symbols to make them available to
125+
# the dynamically loaded modules
126+
WASM_EXE_LDFLAGS_SHARED = [
127+
"-Xlinker --export-all",
128+
"-Xlinker --no-gc-sections",
77129
]
78130

79131
# Flags for shared libraries
132+
# See notes in README about WebAssembly and shared libraries
80133
WASM_LDFLAGS_SHARED = [
81134
"-nostdlib",
82135
"-nostdlib++",
@@ -91,6 +144,62 @@
91144
WASM_LDSHARED = " ".join([WASM_CC] + WASM_LDFLAGS_SHARED)
92145
WASM_LDXXSHARED = " ".join([WASM_CXX] + WASM_LDFLAGS_SHARED)
93146

147+
# Lib order very important here
148+
# TODO: remove this when BLAS support is moved to a task
149+
WASM_BLAS_LIBS = [
150+
"lapack",
151+
"blas",
152+
"f2c",
153+
]
154+
155+
# ----------
156+
# Env. variables for different environments
157+
# ----------
158+
159+
# Env. variables as a dictionary: prefix with FAASM_WASM or FAASM_NATIVE
160+
# depending on the build type variables target
161+
FAASM_BUILD_ENV_DICT = {
162+
"CMAKE_ROOT": FAASM_CMAKE_ROOT,
163+
"FAASM_NATIVE_INSTALL_DIR": FAASM_NATIVE_DIR,
164+
"FAASM_WASM_CC": WASM_CC,
165+
"FAASM_WASM_CXX": WASM_CXX,
166+
"FAASM_WASM_AR": WASM_AR,
167+
"FAASM_WASM_NM": WASM_NM,
168+
"FAASM_WASM_RANLIB": WASM_RANLIB,
169+
"FAASM_WASM_HOST_SHARED": WASM_HOST_SHARED,
170+
"FAASM_WASM_HOST_STATIC": WASM_HOST_STATIC,
171+
"FAASM_WASM_HOST_UNKNOWN": WASM_HOST_UNKNOWN,
172+
"FAASM_WASM_INSTALL_DIR": WASM_TOOLCHAIN_BIN,
173+
"FAASM_WASM_SYSROOT": WASM_SYSROOT,
174+
"FAASM_WASM_CFLAGS": " ".join(WASM_CFLAGS),
175+
"FAASM_WASM_CFLAGS_SHARED": " ".join(WASM_CFLAGS_SHARED),
176+
"FAASM_WASM_EXE_LINKER_FLAGS": " ".join(WASM_EXE_LDFLAGS),
177+
"FAASM_WASM_EXE_LINKER_FLAGS_SHARED": " ".join(WASM_EXE_LDFLAGS_SHARED),
178+
"FAASM_WASM_SHARED_LINKER_FLAGS": " ".join(WASM_LDFLAGS_SHARED),
179+
"FAASM_WASM_STATIC_LINKER_FLAGS": " ".join(WASM_LDFLAGS),
180+
"FAASM_WASM_BLAS_LIBS": " ".join(WASM_BLAS_LIBS),
181+
}
182+
183+
# Note that values in the environment dictionary can only be strings, bytes, or
184+
# Path-like objects, otherwise python's subprocess module complains
185+
FAASM_RUNTIME_ENV_DICT = {
186+
"FAASM_WASM_BYTES_PER_PAGE": str(FAASM_WASM_BYTES_PER_PAGE),
187+
"FAASM_WASM_CTORS_FUNC_NAME": FAASM_WASM_CTORS_FUNC_NAME,
188+
"FAASM_WASM_MAX_MEMORY": str(FAASM_WASM_MAX_MEMORY),
189+
"FAASM_WASM_STACK_SIZE": str(FAASM_WASM_STACK_SIZE),
190+
"FAASM_WASM_ZYGOTE_FUNC_NAME": FAASM_WASM_ZYGOTE_FUNC_NAME,
191+
}
192+
193+
194+
def get_dict_as_cmake_vars(env_dict):
195+
return " ".join(["-D{}={}".format(k, env_dict[k]) for k in env_dict])
196+
197+
198+
# ----------
199+
# Scripts for configure, automake, and autotools
200+
# ----------
201+
202+
# Variables for 'configure' scripts
94203
_BASE_CONFIG_CMD = [
95204
"CC={}".format(WASM_CC),
96205
"CXX={}".format(WASM_CXX),

0 commit comments

Comments
 (0)