Skip to content

Commit aba5ffd

Browse files
cailmdaleycailmdaley
authored
Merge pull request #695 from shapepipe:docker
Automatic Docker Builds
2 parents 65671c5 + f20d115 commit aba5ffd

File tree

6 files changed

+147
-115
lines changed

6 files changed

+147
-115
lines changed

‎.github/workflows/ci-dev.yml‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎.github/workflows/deploy-image.yml‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,40 +2,51 @@ name: Create and publish a Docker image
22

33
on:
44
push:
5-
branches: ['master', 'develop']
65

76
env:
87
REGISTRY: ghcr.io
98
IMAGE_NAME: ${{ github.repository }}
9+
BRANCH: ${{ github.ref }}
1010

1111
jobs:
1212
build-and-push-image:
13-
runs-on: ubuntu-latest
13+
runs-on:
14+
- ubuntu-latest
15+
# - macos-latest
1416
permissions:
1517
contents: read
1618
packages: write
1719

1820
steps:
19-
- name: Checkout repository
20-
uses: actions/checkout@v3
21-
2221
- name: Log in to the Container registry
23-
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
22+
uses: docker/login-action@v3
2423
with:
2524
registry: ${{ env.REGISTRY }}
2625
username: ${{ github.actor }}
2726
password: ${{ secrets.GITHUB_TOKEN }}
2827

28+
- name: Set up Docker Buildx
29+
uses: docker/setup-buildx-action@v3
30+
2931
- name: Extract metadata (tags, labels) for Docker
3032
id: meta
31-
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
33+
uses: docker/metadata-action@v5
3234
with:
3335
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
3436

35-
- name: Build and push Docker image
36-
uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
37+
- name: Build and export to Docker
38+
uses: docker/build-push-action@v6
3739
with:
38-
context: .
39-
push: true
40+
load: true
4041
tags: ${{ steps.meta.outputs.tags }}
4142
labels: ${{ steps.meta.outputs.labels }}
43+
44+
- name: Test
45+
run: docker run --rm ${{ steps.meta.outputs.tags }} shapepipe_run -c /app/example/config.ini
46+
47+
- name: Push
48+
uses: docker/build-push-action@v6
49+
with:
50+
push: true
51+
tags: ${{ steps.meta.outputs.tags }}
52+
labels: ${{ steps.meta.outputs.labels }}

‎Dockerfile‎

Lines changed: 96 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,103 @@
1-
FROM continuumio/miniconda3
1+
FROM python:3.11-bookworm
22

3-
LABEL Description="ShapePipe Docker Image"
4-
ENV SHELL /bin/bash
3+
LABEL Description="Conda-Free ShapePipe Docker Image"
4+
ENV SHELL=/bin/bash
55

6-
ARG CC=gcc-9
7-
ARG CXX=g++-9
6+
# Install system dependencies
7+
RUN apt-get update -y --quiet --fix-missing && \
8+
apt-get dist-upgrade -y --quiet --fix-missing && \
9+
apt-cache policy autconf && \
10+
apt-get install -y --quiet \
11+
apt-utils \
12+
autoconf \
13+
automake \
14+
build-essential \
15+
cmake \
16+
curl \
17+
ffmpeg \
18+
g++ \
19+
gcc \
20+
gfortran \
21+
git-lfs \
22+
libatlas-base-dev \
23+
libblas-dev \
24+
liblapack-dev \
25+
libcfitsio-dev \
26+
libfftw3-bin \
27+
libfftw3-dev \
28+
libgl1-mesa-glx \
29+
libtool \
30+
libtool-bin \
31+
libtool-doc \
32+
locales \
33+
locate \
34+
make \
35+
openmpi-bin \
36+
libopenmpi-dev \
37+
pkg-config \
38+
protobuf-compiler \
39+
psfex=3.21.1-1 \
40+
source-extractor=2.25.0+ds-3 \
41+
weightwatcher=1.12+dfsg-3 \
42+
vim \
43+
xterm && \
44+
apt-get clean -y && \
45+
apt-get autoremove --purge --quiet -y && \
46+
rm -rf /var/lib/apt/lists/* /var/tmp/*
847

9-
# gcc < 10 is required to compile ww
10-
ENV CC=gcc-9
11-
ENV CXX=g++-9
48+
# Install CDS client by hand
49+
RUN cd /tmp && \
50+
curl -O http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz && \
51+
tar xvfz cdsclient.tar.gz \
52+
&& cd cdsclient-* \
53+
&& ./configure && make && make install \
54+
&& rm -rf /tmp/*
1255

13-
RUN apt-get update --allow-releaseinfo-change && \
14-
apt-get update && \
15-
apt-get upgrade -y && \
16-
apt-get install apt-utils -y && \
17-
apt-get install make -y && \
18-
apt-get install automake -y && \
19-
apt-get install autoconf -y && \
20-
apt-get install gcc-9 g++-9 -y && \
21-
apt-get install gfortran -y && \
22-
apt-get install locales -y && \
23-
apt-get install libgl1-mesa-glx -y && \
24-
apt-get install xterm -y && \
25-
apt-get install cmake protobuf-compiler -y && \
26-
apt-get install libtool libtool-bin libtool-doc -y && \
27-
apt-get install libfftw3-bin libfftw3-dev -y && \
28-
apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
29-
apt-get install vim -y && \
30-
apt-get install locate -y && \
31-
apt-get install curl -y && \
32-
apt-get install acl -y && \
33-
apt-get install sssd -y && \
34-
apt-get clean
56+
# Install python dependencies
57+
RUN pip install --no-cache-dir --upgrade pip && \
58+
pip install --no-cache-dir \
59+
astropy==5.2 \
60+
cs_util==0.1.0 \
61+
galsim==2.2.6 \
62+
ipython==8.18.1 \
63+
joblib==1.1.0 \
64+
jupyterlab==4.3.1 \
65+
matplotlib==3.8.4 \
66+
mccd==1.2.4 \
67+
modopt==1.6.1 \
68+
mpi4py==4.0.3 \
69+
numba==0.58.1 \
70+
numpy==1.26.4 \
71+
numpydoc==1.2 \
72+
pandas==2.2 \
73+
PyQt5==5.15.6 \
74+
pyqtgraph==0.12.4 \
75+
pytest==8.3.3 \
76+
pytest-cov==5.0.0 \
77+
pytest-pycodestyle==2.4.1 \
78+
pytest-pydocstyle==2.4.0 \
79+
reproject==0.14.1 \
80+
sf_tools==2.0.4 \
81+
sip_tpv==1.1 \
82+
skaha==1.4.3 \
83+
sqlitedict==2.0.0 \
84+
termcolor==1.1.0 \
85+
tqdm==4.63.0 \
86+
treecorr==5.1.1 \
87+
vos==3.6.1.1 \
88+
git+https://github.com/aguinot/ngmix@stable_version \
89+
git+https://github.com/tobias-liaudat/[email protected] \
90+
git+https://github.com/CEA-COSMIC/pysap@develop
3591

36-
ADD nsswitch.conf /etc/
3792

38-
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
39-
locale-gen
40-
ENV LANG en_US.UTF-8
41-
ENV LANGUAGE en_US:en
42-
ENV LC_ALL en_US.UTF-8
93+
WORKDIR /app
94+
COPY . /app/.
4395

44-
SHELL ["/bin/bash", "--login", "-c"]
45-
46-
COPY ./environment.yml ./
47-
COPY install_shapepipe README.rst setup.py setup.cfg ./
48-
RUN touch ./README.md
49-
50-
RUN conda update -n base -c defaults conda -c defaults
51-
RUN conda env create --file environment.yml
52-
53-
COPY shapepipe ./shapepipe
54-
COPY scripts ./scripts
55-
56-
RUN source activate shapepipe
57-
#RUN pip install jupyter
96+
# Install shapepipe and symlink scripts
97+
RUN pip install --no-cache-dir -e . && \
98+
for ext in .py .sh .bash; do \
99+
for script in /app/scripts/*/*$ext; do \
100+
link_name=`basename $script $ext`; \
101+
ln -s $script /usr/local/bin/$link_name; \
102+
done; \
103+
done

‎docs/source/installation.md‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,33 @@ are expected to install the full ShapePipe environment on the system(s) where
66
data should be processed.
77
```
88

9-
## Standard Installation
9+
## Container Installation (Recommended)
10+
11+
The easiest way to install ShapePipe is via a container. Docker images are automatically built and pushed to the [Github Container Registry (GHCR)](ghcr.io/cosmostat/shapepipe) for each release. This images can be installed and run on most systems (including clusters) with just a few lines of code.
12+
13+
We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command:
14+
15+
```bash
16+
# build writeable "sandbox" container in the current directory
17+
# ./shapepipe will be a directory that functions like a vm
18+
apptainer build --sandbox shapepipe docker://ghcr.io/cosmostat/shapepipe:develop
19+
20+
# open a shell in the container and run the example
21+
apptainer shell --writable shapepipe
22+
cd /app && shapepipe_run -c /app/example/config.ini
23+
```
24+
25+
You can also run the image with **Docker**:
26+
27+
```bash
28+
docker run --rm -it ghcr.io/cosmostat/shapepipe:develop shapepipe_run -c /app/example/config.ini
29+
```
30+
31+
```{attention}
32+
We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance.
33+
```
34+
35+
## Conda Installation (Deprecated)
1036

1137
```{tip}
1238
:class: margin

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "shapepipe"
3-
version = "1.0.2"
3+
version = "1.1.0"
44
description = "Galaxy shape measurement pipeline."
55
readme = "README.rst"
66
authors = [

‎src/shapepipe/run.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from modopt.interface.errors import catch_error
1515
from modopt.interface.log import close_log, set_up_log
1616

17-
from shapepipe.info import line, shapepipe_logo
17+
from shapepipe.info import line, shapepipe_logo, __installs__
1818
from shapepipe.pipeline.args import create_arg_parser
1919
from shapepipe.pipeline.config import create_config_parser
2020
from shapepipe.pipeline.dependency_handler import DependencyHandler

0 commit comments

Comments
 (0)