Skip to content

Commit 78fd86d

Browse files
csegarragonzcsegarragonz
authored
LLVM 13 (#21)
* gh: bump code version * tasks: homogeneise docker and git tasks * python: pin requirements.txt version * format: add task to format code and remove ad-hoc scripts * bump cpp version * gh: bump code for llvm release * docker: fixes to dockerfile * docker: fix * docker: build working without cross-compiled python libraries * crossenv: fixes * crossenv: more fixes * docker: failing again * move numpy to experimental support * minor fixes * gh: add flake8 to requirements.txt * crossenv: some fixes * gha: fix formatting * gha: more fixes * cpp: bump submodule * tasks: re-factor cpython tasks * docs: update readme * gha: apply refactor * cpp: bump submodule to include simd * cpp: bump submodule again * reqs: bump invoke version * docker: correct typos * docs: update readme * cpp: bump submodule * re-factor libs and runtime to modules * comment out numpy * cpython: python hello function working by building ctypes as a static library * cpp: bump submodule to include latest libffi imports * format * docs: update readme * cpython: mark string as regex so that flake8 is happy * actually install the cross-compiled modules * cpython: fix hashing problem in faasm tests by building the hashing modules statically * docs: update readme * cpp: bump submodule after merge
1 parent 5d1661b commit 78fd86d

28 files changed

+617
-365
lines changed

‎.github/workflows/tests.yml‎

Lines changed: 26 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,31 +8,30 @@ on:
88
types: [opened, synchronize, reopened, ready_for_review]
99

1010
jobs:
11+
# Cancel previous running actions for the same PR
12+
cancel_previous:
13+
if: github.event.pull_request.draft == false
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Cancel Workflow Action
17+
uses: styfle/[email protected]
18+
1119
formatting:
1220
if: github.event.pull_request.draft == false
1321
runs-on: ubuntu-20.04
1422
container:
15-
image: faasm/cpython:0.1.1
16-
defaults:
17-
run:
18-
working-directory: /code/python
23+
image: faasm/cpython:0.2.0
1924
steps:
20-
- name: "Fetch all"
21-
run: git fetch --all
22-
- name: "Fetch ref"
23-
run: git fetch origin ${GITHUB_REF}:ci-branch
24-
- name: "Check out branch"
25-
run: git checkout --force ci-branch
26-
- name: "Update cpp submodule"
27-
run: git submodule update third-party/cpp
28-
- name: "Install requirements"
29-
run: pip3 install -r requirements.txt
30-
- name: "Python formatting check"
31-
run: ./bin/check_python.sh
32-
- name: "Check C/C++ formatting"
33-
run: ./bin/run_clang_format.sh
34-
- name: "Check no formatting changes"
35-
run: git diff --exit-code
25+
- name: "Fetch code"
26+
uses: actions/checkout@v3
27+
with:
28+
submodules: true
29+
# We need to set the safe git directory as formatting relies on git-ls
30+
# See actions/checkout#766
31+
- name: "Set the GH workspace as a safe git directory"
32+
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
33+
- name: "Check formatting"
34+
run: ./bin/inv_wrapper.sh format-code --check
3635

3736
build:
3837
if: github.event.pull_request.draft == false
@@ -42,23 +41,16 @@ jobs:
4241
REDIS_QUEUE_HOST: redis
4342
REDIS_STATE_HOST: redis
4443
container:
45-
image: faasm/cpython:0.1.1
46-
defaults:
47-
run:
48-
working-directory: /code/python
44+
image: faasm/cpython:0.2.0
4945
services:
5046
redis:
5147
image: redis
5248
steps:
53-
- name: "Fetch all"
54-
run: git fetch --all
55-
- name: "Fetch ref"
56-
run: git fetch origin ${GITHUB_REF}:ci-branch
57-
- name: "Check out branch"
58-
run: git checkout --force ci-branch
59-
- name: "Install requirements"
60-
run: pip3 install -r requirements.txt
49+
- name: "Fetch code"
50+
uses: actions/checkout@v3
51+
with:
52+
submodules: true
6153
- name: "Build Python function"
62-
run: inv func
54+
run: ./bin/inv_wrapper.sh cpython.func
6355
- name: "Check copying Python functions locally"
64-
run: inv func.upload-all --local
56+
run: ./bin/inv_wrapper.sh func.upload-all --local

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Proj-specific
22
build/
33
cross_venv/
4+
venv/
5+
venv-bm/
46

57
# Byte-compiled / optimized / DLL files
68
__pycache__/

‎Dockerfile‎

Lines changed: 48 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,55 @@
1-
FROM faasm/cpp-sysroot:0.1.6
1+
FROM faasm/cpp-sysroot:0.2.0
22
ARG FAASM_PYTHON_VERSION
33

4-
RUN apt install -y \
5-
libssl-dev \
6-
ninja-build
7-
8-
# Hack to avoid rebuilding build CPython every time the code changes
9-
WORKDIR /tmp
10-
COPY bin/install_build_python.sh .
11-
RUN ./install_build_python.sh
12-
13-
# Hack to avoid reinstalling Python libs every time
14-
COPY requirements.txt .
15-
RUN pip3 install -r requirements.txt
16-
17-
# Clone current tag
18-
WORKDIR /code
19-
RUN git clone \
20-
-b v${FAASM_PYTHON_VERSION} \
21-
https://github.com/faasm/python
22-
23-
# Submodules
24-
WORKDIR /code/python
25-
RUN git submodule update --init
26-
27-
# Build CPython to wasm
28-
RUN inv cpython
29-
30-
# Set up crossenv
31-
RUN ./bin/crossenv_setup.sh
32-
33-
# Install cross-compiled python packages
34-
RUN . ./cross_venv/bin/activate && inv libs.install
35-
36-
# Build Faasm function
37-
RUN inv func
38-
39-
# TODO - enable these once the MXNet/ Horovod work is completed
4+
SHELL ["/bin/bash", "-c"]
5+
ENV PYTHON_DOCKER="on"
6+
7+
RUN apt update && apt install -y libssl-dev
8+
9+
# Clone code and submodules
10+
RUN mkdir -p /code \
11+
&& git clone \
12+
-b v${FAASM_PYTHON_VERSION} \
13+
https://github.com/faasm/python \
14+
/code/python \
15+
&& cd /code/python \
16+
&& git submodule update --init -f third-party/cpp \
17+
&& git submodule update --init -f third-party/cpython \
18+
&& git submodule update --init -f third-party/crossenv \
19+
&& git config --global --add safe.directory /code/python
20+
21+
# Cross-compile CPython to WASM and the python libraries
22+
RUN cd /code/python \
23+
&& ./bin/create_venv.sh \
24+
&& source ./venv/bin/activate \
25+
# Buld and install a native CPython and a cross-compiled CPython with the
26+
# same version. Also cross-compile the entrypoint function for CPython
27+
&& inv \
28+
cpython.native \
29+
cpython.wasm \
30+
cpython.func
31+
32+
# Build cross-compiled python modules, including `pyfaasm`
33+
RUN cd /code/python \
34+
&& ./bin/crossenv_setup.sh \
35+
&& source ./cross_venv/bin/activate \
36+
&& pip3 install -r crossenv/requirements.txt \
37+
&& inv -r crossenv modules.build
38+
39+
# Finally, install the cross-compiled Python modules
40+
RUN cd /code/python \
41+
&& source ./venv/bin/activate \
42+
&& inv modules.install
43+
44+
45+
# TODO: enable these once the MXNet/ Horovod work is completed
4046
# Build mxnet
4147
# RUN inv mxnet
4248

43-
# Install experimental pacakges
49+
# TODO: Install experimental pacakges
4450
# RUN . ./cross_venv/bin/activate && inv libs.install --experimental
4551

46-
# Copy files into place
47-
RUN inv runtime
52+
WORKDIR /code/python
53+
ENV TERM xterm-256color
54+
RUN sed -i 's/\/code\/cpp\/bin/\/code\/python\/bin/g' ~/.bashrc
55+
CMD ["/bin/bash", "-l"]

0 commit comments

Comments
 (0)