Skip to content

Commit 956f84c

Browse files
csegarragonzcsegarragonz
authored
Homogenise git and docker interface (#95)
* chore(gh): bump code version * feat: add task to bump code version * fix(docker): refactor tasks from container to docker * fix(docker): new llvm dockerfile * feat(python): add create_venv script to use separate venv directories * fix(python): pin python dependencies to a version and include requests * fix: include venv-bm in gitignore * fix(python): refactor docker task to have the same signature than faasm's * fix(docker): fixes in llvm dockerfile * fix(docker): update cpp dockerfile * fix(docs): update code snippets in docs to create docker images * feat: add inv_wrapper script to call in the tests * fix: format docker task * fix(gha): update gha file to use venvs * fix(gha): activate venv to run python checks * fix(dockerfile): bump llvm dockerfile back to 20.04 * fix: add noninteractive flag to llvm dockerfile
1 parent a251869 commit 956f84c

File tree

15 files changed

+270
-208
lines changed

15 files changed

+270
-208
lines changed

‎.env‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
SYSROOT_VERSION=0.1.6
2-
SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.1.6
1+
SYSROOT_VERSION=0.1.7
2+
SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.1.7
33
COMPOSE_PROJECT_NAME=cpp-dev

‎.github/workflows/tests.yml‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
if: github.event.pull_request.draft == false
1313
runs-on: ubuntu-20.04
1414
container:
15-
image: faasm/cpp-sysroot:0.1.6
15+
image: faasm/cpp-sysroot:0.1.7
1616
defaults:
1717
run:
1818
working-directory: /code/cpp
@@ -30,38 +30,37 @@ jobs:
3030
run: git submodule update --init -f third-party/FFmpeg
3131
- name: "Update wasi-libc submodule"
3232
run: git submodule update --init -f third-party/wasi-libc
33-
- name: "Install requirements"
34-
run: pip3 install -r requirements.txt
3533
# --- Build libraries to wasm ---
3634
- name: "Build libc"
37-
run: inv libc
35+
run: ./bin/inv_wrapper.sh libc
3836
- name: "Build libfaasm"
39-
run: inv libfaasm
37+
run: ./bin/inv_wrapper.sh libfaasm
4038
- name: "Build libfaasmp"
41-
run: inv libfaasmp
39+
run: ./bin/inv_wrapper.sh libfaasmp
4240
- name: "Build libfaasmpi"
43-
run: inv libfaasmpi
41+
run: ./bin/inv_wrapper.sh libfaasmpi
4442
- name: "Build libffi"
45-
run: inv libffi
43+
run: ./bin/inv_wrapper.sh libffi
4644
- name: "Build libfake"
47-
run: inv libfake
45+
run: ./bin/inv_wrapper.sh libfake
4846
- name: "Build libemscripten"
49-
run: inv libemscripten
47+
run: ./bin/inv_wrapper.sh libemscripten
5048
- name: "Build FFmpeg"
51-
run: inv ffmpeg
49+
run: ./bin/inv_wrapper.sh ffmpeg
5250
# --- Build functions to wasm ---
5351
- name: "Build the functions"
54-
run: inv func.local --clean
52+
run: ./bin/inv_wrapper.sh func.local --clean
5553
# --- Build libraries natively ---
5654
- name: "Build libfaasm native"
57-
run: inv libfaasm --native --clean
55+
run: ./bin/inv_wrapper.sh libfaasm --native --clean
5856
- name: "Build libfaasmp native"
59-
run: inv libfaasmp --native --clean
57+
run: ./bin/inv_wrapper.sh libfaasmp --native --clean
6058
- name: "Build libfaasmpi native"
61-
run: inv libfaasmpi --native --clean
59+
run: ./bin/inv_wrapper.sh libfaasmpi --native --clean
6260
# --- Formatting ---
6361
- name: "Check python"
64-
run: ./bin/check_python.sh
62+
run: source venv/bin/activate && ./bin/check_python.sh
63+
shell: bash
6564
- name: "Run C/C++ formatting"
6665
run: ./bin/run_clang_format.sh
6766
- name: "Check no formatting changes"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ celerybeat.pid
115115
.venv
116116
env/
117117
venv/
118+
venv-bm
118119
ENV/
119120
env.bak/
120121
venv.bak/

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.1.6
1+
0.1.7

‎bin/create_venv.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
THIS_DIR=$(dirname $(readlink -f $0))
6+
PROJ_ROOT=${THIS_DIR}/..
7+
8+
# Set different virtual environment paths so that these don't clash when
9+
# mounting the code in a development container
10+
VENV_PATH="undetected"
11+
if [[ -z "$CPP_DOCKER" ]]; then
12+
VENV_PATH="${PROJ_ROOT}/venv-bm"
13+
else
14+
VENV_PATH="/code/cpp/venv"
15+
fi
16+
17+
PIP=${VENV_PATH}/bin/pip3
18+
19+
function pip_cmd {
20+
source ${VENV_PATH}/bin/activate && ${PIP} "$@"
21+
}
22+
23+
pushd ${PROJ_ROOT} >> /dev/null
24+
25+
if [ ! -d ${VENV_PATH} ]; then
26+
python3 -m venv ${VENV_PATH}
27+
fi
28+
29+
pip_cmd install -U pip
30+
pip_cmd install -U setuptools wheel
31+
pip_cmd install -r requirements.txt
32+
33+
popd >> /dev/null

‎bin/inv_wrapper.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Wrapper script for running invoke in virtual env
5+
6+
source bin/workon.sh && inv $@

‎bin/workon.sh‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,16 @@
77
MODE="undetected"
88
if [[ -z "$CPP_DOCKER" ]]; then
99

10-
THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
11-
if [ "$(ps -o comm= -p $$)" = "zsh" ]; then
12-
THIS_DIR="$( cd "$( dirname "${ZSH_ARGZERO}" )" >/dev/null 2>&1 && pwd )"
13-
fi
10+
THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )"
1411
PROJ_ROOT="${THIS_DIR}/.."
12+
VENV_PATH="${PROJ_ROOT}/venv-bm"
1513

1614
# Normal terminal
1715
MODE="terminal"
1816
else
1917
# Running inside the container, we know the project root
2018
PROJ_ROOT="/code/cpp"
19+
VENV_PATH="/code/cpp/venv"
2120

2221
# Use containerised redis
2322
alias redis-cli="redis-cli -h redis"
@@ -31,12 +30,12 @@ pushd ${PROJ_ROOT}>>/dev/null
3130
# Virtualenv
3231
# ----------------------------
3332

34-
if [ ! -d "venv" ]; then
35-
python3 -m venv venv
33+
if [ ! -d ${VENV_PATH} ]; then
34+
./bin/create_venv.sh
3635
fi
3736

3837
export VIRTUAL_ENV_DISABLE_PROMPT=1
39-
source venv/bin/activate
38+
source ${VENV_PATH}/bin/activate
4039

4140
# ----------------------------
4241
# Invoke tab-completion

‎docker/cpp-sysroot.dockerfile‎

Lines changed: 51 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,67 @@
11
FROM faasm/llvm:10.0.1 as llvm
22

33
# faabric-base image is not re-built often, so tag may be behind
4-
FROM faasm/faabric-base:0.1.0
5-
ARG SYSROOT_VERSION
4+
FROM faasm/faabric-base:0.3.5
5+
SHELL ["/bin/bash", "-c"]
6+
ENV CPP_DOCKER="on"
67

78
# Copy the toolchain in from the LLVM container
89
COPY --from=llvm /usr/local/faasm /usr/local/faasm
910

10-
RUN apt update
11-
RUN apt install -y autotools-dev
12-
13-
# Get the code
14-
WORKDIR /code
15-
RUN git clone -b v${SYSROOT_VERSION} https://github.com/faasm/cpp
16-
WORKDIR /code/cpp
11+
# Update APT dependencies
12+
RUN apt update && apt install -y autotools-dev
1713

18-
# Update submodules (not LLVM)
19-
RUN git submodule update --init -f third-party/faabric
20-
RUN git submodule update --init -f third-party/faasm-clapack
21-
RUN git submodule update --init -f third-party/libffi
22-
RUN git submodule update --init -f third-party/wasi-libc
23-
RUN git submodule update --init -f third-party/FFmpeg
14+
# Get the code and submodules
15+
ARG SYSROOT_VERSION
16+
RUN mkdir -p /code \
17+
&& git clone -b v${SYSROOT_VERSION} \
18+
https://github.com/faasm/cpp \
19+
/code/cpp \
20+
&& cd /code/cpp \
21+
# Update submodules (not LLVM)
22+
&& git submodule update --init -f third-party/faabric \
23+
&& git submodule update --init -f third-party/faasm-clapack \
24+
&& git submodule update --init -f third-party/libffi \
25+
&& git submodule update --init -f third-party/wasi-libc \
26+
&& git submodule update --init -f third-party/FFmpeg
2427

2528
# Install the faasmtools Python lib
26-
RUN pip3 install -r requirements.txt
27-
RUN pip3 install .
28-
29-
# ---------------------------------
30-
# NATIVE
31-
# ---------------------------------
32-
33-
# Native static libraries
34-
RUN inv libfaasm --native
35-
RUN inv libfaasmp --native
36-
RUN inv libfaasmpi --native
37-
38-
# Native shared libraries
39-
RUN inv libfaasm --native --shared
40-
RUN inv libfaasmp --native --shared
41-
RUN inv libfaasmpi --native --shared
42-
43-
# ---------------------------------
44-
# WASM
45-
# ---------------------------------
46-
47-
# Install toolchain files
48-
RUN inv install
49-
50-
# Libraries
51-
RUN inv libc
52-
RUN inv libffi
53-
RUN inv ffmpeg
54-
55-
# Both static and shared clapack
56-
RUN inv clapack
57-
RUN inv clapack --clean --shared
58-
59-
# Faasm libraries to wasm
60-
RUN inv libfaasm
61-
RUN inv libemscripten
62-
RUN inv libfaasmp
63-
RUN inv libfaasmpi
29+
RUN cd /code/cpp \
30+
&& ./bin/create_venv.sh \
31+
&& source venv/bin/activate \
32+
&& pip3 install -r requirements.txt \
33+
&& pip3 install .
34+
35+
# Build all the targets
36+
RUN cd /code/cpp \
37+
&& source venv/bin/activate \
38+
# Build native Faasm libraries (static and shared)
39+
&& inv \
40+
libfaasm --native \
41+
libfaasmp --native \
42+
libfaasmpi --native \
43+
libfaasm --native --shared \
44+
libfaasmp --native --shared \
45+
libfaasmpi --native --shared \
46+
# Install toolchain files
47+
&& inv install \
48+
# Build ported third-pary WASM libraries
49+
&& inv \
50+
clapack \
51+
clapack --clean --shared \
52+
ffmpeg \
53+
libc \
54+
libffi \
55+
# Build Faasm WASM libraries
56+
&& inv \
57+
libemscripten \
58+
libfaasm \
59+
libfaasmp \
60+
libfaasmpi
6461

6562
# CLI setup
63+
WORKDIR /code/cpp
6664
ENV TERM xterm-256color
67-
SHELL ["/bin/bash", "-c"]
6865

6966
RUN echo ". /code/cpp/bin/workon.sh" >> ~/.bashrc
7067
CMD ["/bin/bash", "-l"]

‎docker/llvm.dockerfile‎

Lines changed: 38 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,40 @@
11
FROM ubuntu:20.04
22

3-
RUN apt update
4-
RUN apt install -y software-properties-common
5-
RUN apt update
6-
RUN apt upgrade -y
7-
8-
RUN apt install -y \
9-
autoconf \
10-
clang-10 \
11-
build-essential \
12-
git \
13-
ninja-build \
14-
pkg-config \
15-
wget
16-
17-
# Up-to-date CMake
18-
RUN apt remove --purge --auto-remove cmake
19-
WORKDIR /setup
20-
RUN wget -q -O cmake-linux.sh https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.sh
21-
RUN sh cmake-linux.sh -- --skip-license --prefix=/usr/local
22-
23-
# Tidy up
24-
RUN apt-get clean autoclean
25-
RUN apt-get autoremove
26-
27-
# Copy the code in
28-
WORKDIR /code
29-
COPY . .
30-
31-
# Run the main make
32-
RUN make
33-
34-
# Print the clang version
35-
RUN /usr/local/faasm/toolchain/bin/clang --version
36-
37-
# Remove the code
38-
WORKDIR /
39-
RUN rm -r /code
40-
41-
CMD /bin/bash
42-
3+
# Install APT dependencies
4+
ARG DEBIAN_FRONTEND=noninteractive
5+
RUN apt update \
6+
&& apt upgrade -y \
7+
&& apt install -y \
8+
autoconf \
9+
clang-10 \
10+
curl \
11+
build-essential \
12+
git \
13+
gpg \
14+
ninja-build \
15+
pkg-config \
16+
software-properties-common \
17+
wget
18+
19+
# Install up-to-date CMake
20+
RUN apt remove --purge --auto-remove cmake \
21+
&& mkdir -p /setup \
22+
&& cd /setup \
23+
&& wget -q -O cmake-linux.sh \
24+
https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \
25+
&& sh cmake-linux.sh -- --skip-license --prefix=/usr/local \
26+
&& apt clean autoclean -y \
27+
&& apt autoremove -y
28+
29+
# Get the code, build the main targets, and remove the code
30+
ARG SYSROOT_VERSION
31+
RUN mkdir -p /code \
32+
&& git clone -b v${SYSROOT_VERSION} \
33+
https://github.com/faasm/cpp \
34+
/code/cpp \
35+
&& cd /code/cpp \
36+
&& git submodule update --init -f third-party/llvm-project \
37+
&& git submodule update --init -f third-party/wasi-libc \
38+
&& make \
39+
&& /usr/local/faasm/toolchain/bin/clang --version \
40+
&& rm -rf /code

0 commit comments

Comments
 (0)