Skip to content

Commit 760edc4

Browse files
AndreWeinerAndreWeiner
authored
Merge pull request #24 from FoamScience/ci
[ci] build matrix on ubuntu, openfoam, database, and smartsim versions
2 parents 5d39ca7 + 152d379 commit 760edc4

File tree

5 files changed

+184
-47
lines changed

5 files changed

+184
-47
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
name: smartredis
1+
name: tests_ubuntu
22
defaults:
33
run:
44
shell: bash -o pipefail -i {0}
55
on:
66
push:
77
branches:
88
- main
9+
- ci
910
- 'bugfix*'
1011
- 'releases/**'
1112
pull_request:
12-
types: [opened, reopened]
13+
types: [opened, reopened, synchronize]
1314
workflow_dispatch:
1415
inputs:
1516
debug_enabled:
@@ -20,23 +21,32 @@ on:
2021

2122
jobs:
2223
build:
24+
strategy:
25+
fail-fast: false
26+
matrix:
27+
openfoam_version: ['2312', '2212']
28+
smartsim_version: ['0.7.0', '0.6.2']
29+
database_backend: ['redis']
30+
ubuntu_version: ['22.04', '20.04'] ## SmartSim not available yet for Python 3.12 on Ubuntu 24
31+
exclude:
32+
- ubuntu_version: '20.04' ## SmartSim 0.7.0 not released on ubuntu 20.04
33+
smartsim_version: '0.7.0'
2334
runs-on: ubuntu-22.04
2435

2536
container:
26-
image: ghcr.io/foamscience/openfoam-smartsim:cpu-2312
27-
options: --user openfoam
37+
image: ghcr.io/ofdatacommittee/openfoam-smartsim:of-${{ matrix.openfoam_version }}-smart-${{ matrix.smartsim_version }}-${{ matrix.database_backend }}-ubuntu-${{ matrix.ubuntu_version }}
2838
env:
2939
CATCH_TIMEOUT: 20
3040
FOAM_FOAMUT: "/tmp/foamUT"
31-
SSDB: "redis:6379"
41+
SSDB: "smartdb:6379"
3242

3343
services:
34-
redis:
35-
image: redislabs/redisai
44+
smartdb:
45+
image: ${{ matrix.database_backend == 'redis' && 'redislabs/redisai' || 'eqalpha/keydb' }}
3646

3747
steps:
3848
- name: Checkout repository
39-
uses: actions/checkout@v3
49+
uses: actions/checkout@v4
4050

4151
- name: Setup tmate session
4252
uses: mxschmitt/action-tmate@v3
@@ -46,15 +56,15 @@ jobs:
4656

4757
- name: Compile
4858
run: |
49-
source /usr/lib/openfoam/openfoam2312/etc/bashrc
59+
source /usr/lib/openfoam/openfoam${{ matrix.openfoam_version }}/etc/bashrc
5060
mkdir -p $FOAM_USER_LIBBIN
5161
./Allwmake
5262
5363
- name: Compile Tests and Run
5464
run: |
55-
source /usr/lib/openfoam/openfoam2312/etc/bashrc
65+
source /usr/lib/openfoam/openfoam${{ matrix.openfoam_version }}/etc/bashrc
5666
./Allwmake
57-
export SSDB="redis:6379"
67+
export SSDB="smartdb:6379"
5868
git clone https://github.com/FoamScience/foamUT $FOAM_FOAMUT
5969
sed -i 's/mpirun/mpirun --oversubscribe/g' $FOAM_FOAMUT/Alltest
6070
ln -s "$PWD"/tests "$FOAM_FOAMUT"/tests/smartSimFOTests

‎.github/workflows/todos.yaml‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎docker/Dockerfile‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
ARG UBUNTU_VERSION # Eg. 22.04
2+
FROM ubuntu:${UBUNTU_VERSION}
3+
# To build this image from this folder with OpenFOAM v2206:
4+
# docker build --build-arg OPENFOAM_VERSION=2206 --build-tag SMARTSIM_VERSION=0.7.0 -t openfoam-smartsim:latest .
5+
6+
ARG SMARTSIM_VERSION # Eg. 0.6.2
7+
ARG OPENFOAM_VERSION # Eg. 2206
8+
ARG DATABASE_BACKEND=redis # redis or keydb
9+
LABEL maintainer="Mohammed Elwardi Fadeli <[email protected]>"
10+
LABEL source="https://github.com/OFDataCommittee/openfoam-smartsim"
11+
12+
# Install and setup
13+
ENV DEBIAN_FRONTEND=noninteractive
14+
ENV SHELL=/usr/bin/bash
15+
RUN apt update -y && \
16+
apt install -y --no-install-recommends git-lfs libpython3-dev pip cmake \
17+
sudo apt-utils vim curl wget software-properties-common \
18+
git-core cmake openssh-server gcc g++ \
19+
binutils libopenmpi-dev openmpi-bin openmpi-common openmpi-doc
20+
21+
# Set up OpenFOAM repositories
22+
RUN sh -c "curl https://dl.openfoam.com/add-debian-repo.sh | bash"
23+
RUN apt update -y && \
24+
apt install -y --no-install-recommends openfoam${OPENFOAM_VERSION}-default
25+
RUN apt clean && apt purge && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
26+
# Set up smartsim
27+
RUN pip install smartsim==${SMARTSIM_VERSION} && \
28+
if [ "${DATABASE_BACKEND}" = "keydb" ]; then smart build --device cpu --no_tf --no_pt; else smart build --device cpu --no_tf --no_pt --keydb; fi
29+
ENV USER openfoam
30+
ENV HOME=/home/${USER}
31+
RUN adduser --disabled-password --gecos "" ${USER} -u 1001 && \
32+
echo "ALL ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
33+
USER openfoam
34+
RUN echo "source /usr/lib/openfoam/openfoam${OPENFOAM_VERSION}/etc/bashrc" >> ${HOME}/.bashrc
35+
RUN . "/usr/lib/openfoam/openfoam${OPENFOAM_VERSION}/etc/bashrc" && mkdir -p $FOAM_USER_LIBBIN && mkdir -p $FOAM_USER_APPBIN
36+
37+
38+
#COPY bin/uid_entrypoint /usr/local/bin/uid_entrypoint
39+
#ENTRYPOINT ["uid_entrypoint"]

‎docker/README.md‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Continuous Integration
2+
3+
## Overview
4+
5+
We continuously test against recent versions of the following software pieces:
6+
- Ubuntu
7+
- OpenFOAM
8+
- SmartSim
9+
- Database backend (redis, or keydb)
10+
11+
In particular, by testing on multiple versions of Ubuntu distribution, we make sure
12+
things work on the default toolchain versions (Compilers, system OpenMPI, ..., etc)
13+
14+
We use [Github Container Registry](https://ghcr.io/) to store the Docker images necessary for
15+
CI workflows.
16+
17+
## Instructions for automated image builds
18+
19+
> [!NOTE]
20+
> You will want to install ansible (locally) to build the images easily: `pip install ansible`.
21+
> Each image is a little than 2GB so make sure you have enough disk space.
22+
23+
1. In [docker/build.yml](docker/build.yml) file, add the new version in the relevant fact task.
24+
- For example, to add a new version of OpenFOAM, add the version to `openfoam_versions`
25+
- Then run `ansible-playbook build.yml --extra-vars "username=github_user token=$GITHUB_TOKEN`
26+
to build the corresponding image and push it.
27+
- When building the images locally, you can speed the process by removing other software versions
28+
and keeping only the new one
29+
2. In [.github/workflows/ci.yaml](.github/workflows/ci.yaml) file, add the new version in the relevant
30+
`strategy` section.
31+
- For example, to add a new version of OpenFOAM, add the version to `jobs.build.strategy.matrix.openfoam_version`
32+
- Then commit the changes to the repository.
33+
34+
## Instructions for manual image builds
35+
36+
```sh
37+
cd docker
38+
docker build \
39+
--build-arg OPENFOAM_VERSION=2312 \
40+
--build-arg UBUNTU_VERSION=22.04 \
41+
--build-arg DATABASE_BACKEND=redis \
42+
--build-arg SMARTSIM_VERSION=0.7.0 \
43+
-t ghcr.io/<github_user>/openfoam-smartsim:of-2312-smartsim-0.7.0-redis-ubuntu-22.04 .
44+
```

‎docker/build.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# ansible-playbook build.yml --extra-vars "username=user token=token"
2+
---
3+
- name: Build openfoam-smartsim CI infrastructure
4+
hosts: localhost
5+
gather_facts: false
6+
vars:
7+
ansible_python_interpreter: /usr/bin/python3
8+
9+
tasks:
10+
- name: Install docker python support for ansible
11+
pip:
12+
name: docker
13+
state: present
14+
15+
- name: Define list of OpenFOAM versions
16+
set_fact:
17+
openfoam_versions:
18+
- "2312"
19+
- "2212"
20+
21+
- name: Define list of SmartSim versions
22+
set_fact:
23+
smartsim_versions:
24+
- "0.7.0"
25+
- "0.6.2"
26+
27+
- name: Define list of DB backends
28+
set_fact:
29+
database_backends:
30+
- "redis"
31+
32+
- name: Define list of Ubuntu versions
33+
set_fact:
34+
ubuntu_versions:
35+
#- "24.04" ## SmartSim not available for Python 3.12
36+
- "22.04"
37+
- "20.04"
38+
39+
- name: Login to GitHub Container Registry
40+
docker_login:
41+
username: "{{ username }}"
42+
password: "{{ token }}"
43+
registry_url: "https://ghcr.io"
44+
45+
- name: Build Docker images
46+
docker_image:
47+
name: "ghcr.io/{{ username }}/openfoam-smartsim:of-{{ item.0 }}-smart-{{ item.1 }}-{{ item.2 }}-ubuntu-{{ item.3 }}"
48+
source: "build"
49+
build:
50+
path: "."
51+
dockerfile: "Dockerfile"
52+
pull: yes
53+
args:
54+
OPENFOAM_VERSION: "{{ item.0 }}"
55+
SMARTSIM_VERSION: "{{ item.1 }}"
56+
DATABASE_BACKEND: "{{ item.2 }}"
57+
UBUNTU_VERSION: "{{ item.3 }}"
58+
loop: "{{ openfoam_versions|product(smartsim_versions, database_backends, ubuntu_versions)|list }}"
59+
when: "not (item.1 == '0.7.0' and item.3 == '20.04')"
60+
async: 1200
61+
poll: 0
62+
register: async_result
63+
64+
- name: Wait for all build tasks to complete
65+
async_status:
66+
jid: "{{ item.ansible_job_id }}"
67+
loop: "{{ async_result.results }}"
68+
when: item.changed
69+
register: async_status_result
70+
until: async_status_result.finished
71+
retries: 20
72+
delay: 60
73+
74+
- name: Push Docker images to GitHub Container Registry
75+
docker_image:
76+
name: "ghcr.io/ofdatacommittee/openfoam-smartsim:of-{{ item.0 }}-smart-{{ item.1 }}-{{ item.2 }}-ubuntu-{{ item.3 }}"
77+
source: local
78+
push: yes
79+
loop: "{{ openfoam_versions|product(smartsim_versions, database_backends, ubuntu_versions)|list }}"
80+
when: "not (item.1 == '0.7.0' and item.3 == '20.04')"

0 commit comments

Comments
 (0)