Skip to content

Commit 9c96e1c

Browse files
alexanderschmiBaunsgaard
alexanderschmi
authored and
Baunsgaard
committed
[SYSTEMDS-3877] Docker image size reduction and safety improvement
This commit modifies our docker images to follow current standard procedures to make safe and small docker images. In specific the size of our release image is reduced from 1.5 GB to 450MB, and similarly our testing image is reduced. Closes #2274
1 parent 012a843 commit 9c96e1c

File tree

4 files changed

+108
-60
lines changed

4 files changed

+108
-60
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,3 +150,7 @@ venv/*
150150
# resource optimization
151151
scripts/resource/output
152152
*.pem
153+
154+
# docker tests
155+
docker/mountFolder/*.bin
156+
docker/mountFolder/*.bin.mtd

‎docker/build.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@
2323
# Build the docker containers
2424

2525
# The first build is for running systemds through docker.
26-
# docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest .
26+
docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest .
2727

2828
# The second build is for testing systemds. This image installs the R dependencies needed to run the tests.
29-
docker image build -f docker/testsysds.Dockerfile -t apache/systemds:testing-latest .
29+
# docker image build -f docker/testsysds.Dockerfile -t apache/systemds:testing-latest .
3030

3131
# The third build is python docker for systemds.
3232
# docker image build -f docker/pythonsysds.Dockerfile -t apache/systemds:python-nightly .

‎docker/sysds.Dockerfile‎

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,16 @@
1919
#
2020
#-------------------------------------------------------------
2121

22-
FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233
22+
FROM alpine:3.20@sha256:de4fe7064d8f98419ea6b49190df1abbf43450c1702eeb864fe9ced453c1cc5f AS compile-image
2323

2424
WORKDIR /usr/src/
2525

2626
# Do basic updates on the image
27-
RUN apt-get update -qq \
28-
&& apt-get upgrade -y \
29-
&& apt-get install -y --no-install-recommends \
27+
RUN apk add --no-cache \
3028
wget \
3129
git \
3230
ca-certificates \
33-
&& apt-get clean
31+
bash
3432

3533
# Set environment variables
3634
# Maven
@@ -43,11 +41,11 @@ ENV SYSTEMDS_ROOT=/usr/src/systemds
4341
ENV PATH=$SYSTEMDS_ROOT/bin:$PATH
4442
ENV SYSDS_QUIET=1
4543

46-
# Download Java and Mvn
44+
# Download Mvn and JDK
4745
RUN mkdir -p /usr/lib/jvm \
4846
&& wget -qO- \
49-
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_linux_hotspot_17.0.15_6.tar.gz | tar xzf - \
50-
&& mv jdk-17.0.15+6 /usr/lib/jvm/jdk-17.0.15+6 \
47+
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_alpine-linux_hotspot_17.0.15_6.tar.gz | tar xzf - \
48+
&& mv jdk-17.0.15+6 $JAVA_HOME \
5149
&& wget -qO- \
5250
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - \
5351
&& mv apache-maven-$MAVEN_VERSION /usr/lib/mvn
@@ -57,8 +55,11 @@ RUN git clone --depth 1 https://github.com/apache/systemds.git systemds && \
5755
cd /usr/src/systemds/ && \
5856
mvn --no-transfer-progress clean package -P distribution
5957

58+
COPY docker/mountFolder/main.dml /input/main.dml
59+
6060
# Remove all unnecessary files from the Image
61-
RUN rm -rf .git && \
61+
RUN cd /usr/src/systemds/ && \
62+
rm -rf .git && \
6263
rm -rf .github && \
6364
rm -rf target/javadoc** && \
6465
rm -rf target/apidocs** && \
@@ -71,9 +72,55 @@ RUN rm -rf .git && \
7172
rm -rf /usr/lib/mvn && \
7273
rm -rf CONTRIBUTING.md && \
7374
rm -rf pom.xml && \
74-
rm -rf ~/.m2
75+
rm -rf ~/.m2 && \
76+
rm -rf docker && \
77+
rm -rf .mvn
7578

79+
FROM alpine:3.20@sha256:de4fe7064d8f98419ea6b49190df1abbf43450c1702eeb864fe9ced453c1cc5f
7680

77-
COPY docker/mountFolder/main.dml /input/main.dml
81+
RUN apk add --no-cache bash \
82+
snappy \
83+
lz4 \
84+
zlib
85+
86+
ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
87+
ENV PATH=$JAVA_HOME/bin:$PATH
88+
ENV SYSTEMDS_ROOT=/systemds
89+
ENV PATH=$SYSTEMDS_ROOT/bin:$PATH
90+
ENV SYSDS_QUIET=1
91+
92+
ENV HADOOP_VERSION=3.3.6
93+
ENV HADOOP_HOME=/opt/hadoop
94+
ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native
95+
ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
96+
ENV GLIBC_VERSION=2.35-r1
97+
98+
RUN wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
99+
&& wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \
100+
&& apk add glibc-${GLIBC_VERSION}.apk \
101+
&& rm glibc-${GLIBC_VERSION}.apk
102+
103+
RUN mkdir -p /usr/lib/jvm \
104+
&& wget -qO- \
105+
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jre_x64_alpine-linux_hotspot_17.0.15_6.tar.gz | tar xzf - \
106+
&& mv jdk-17.0.15+6-jre $JAVA_HOME
107+
108+
RUN mkdir -p $HADOOP_HOME/lib/native \
109+
&& wget -q https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \
110+
tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \
111+
hadoop-${HADOOP_VERSION}/lib/native && \
112+
mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \
113+
mv native/libhadoop.so /opt/hadoop/lib/native && \
114+
rm hadoop-${HADOOP_VERSION}.tar.gz && \
115+
rm -rf native
116+
117+
COPY --from=compile-image /usr/src/systemds /systemds
118+
COPY --from=compile-image /input/main.dml /input/main.dml
119+
120+
WORKDIR /input
121+
122+
RUN addgroup -S default && adduser -S systemds -G default
123+
USER systemds
78124

79-
CMD ["systemds", "/input/main.dml"]
125+
ENTRYPOINT ["systemds"]
126+
CMD ["main.dml"]

‎docker/testsysds.Dockerfile‎

Lines changed: 43 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,29 @@
1818
# under the License.
1919
#
2020
#-------------------------------------------------------------
21+
# Stage 1: Build SEAL
22+
FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973 AS seal-build
2123

22-
FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233
24+
RUN apt-get update && apt-get install -y --no-install-recommends \
25+
build-essential \
26+
cmake \
27+
wget \
28+
tar \
29+
git \
30+
ca-certificates \
31+
&& rm -rf /var/lib/apt/lists/*
32+
33+
WORKDIR /seal
34+
35+
# Install SEAL
36+
RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \
37+
&& cd SEAL-3.7.0 \
38+
&& cmake -S . -B build -DBUILD_SHARED_LIBS=ON \
39+
&& cmake --build build \
40+
&& cmake --install build --prefix /seal-install
41+
42+
# Stage 2: Final image with R, JDK, Maven, SEAL
43+
FROM debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
2344

2445
WORKDIR /usr/src/
2546
ENV MAVEN_VERSION=3.9.9
@@ -28,61 +49,37 @@ ENV MAVEN_HOME=/usr/lib/mvn
2849
ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
2950
ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH
3051

31-
ENV LANGUAGE=en_US:en
32-
ENV LC_ALL=en_US.UTF-8
33-
ENV LANG=en_US.UTF-8
34-
ENV LD_LIBRARY_PATH=/usr/local/lib/
35-
36-
RUN apt-get update -qq \
37-
&& apt-get upgrade -y \
38-
&& apt-get install -y --no-install-recommends \
39-
libcurl4-openssl-dev \
40-
libxml2-dev \
41-
locales \
42-
software-properties-common \
43-
dirmngr \
44-
gnupg \
45-
apt-transport-https \
46-
wget \
47-
ca-certificates \
48-
git \
49-
cmake \
50-
patchelf \
51-
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
52-
&& add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" \
53-
&& apt-get update -qq \
54-
&& apt-get upgrade -y \
55-
&& echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
56-
&& locale-gen en_US.utf8 \
57-
&& /usr/sbin/update-locale LANG=en_US.UTF-8 \
52+
RUN apt-get update && apt-get install -y --no-install-recommends \
53+
r-base \
54+
wget \
55+
cmake \
56+
r-base-dev \
57+
libcurl4-openssl-dev \
58+
libssl-dev \
59+
libxml2-dev \
60+
ca-certificates \
61+
patchelf \
62+
git \
63+
libssl-dev \
64+
r-base-dev \
65+
r-base-core \
66+
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
5867
&& mkdir -p /usr/lib/jvm \
5968
&& wget -qO- \
6069
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_linux_hotspot_17.0.15_6.tar.gz | tar xzf - \
61-
&& mv jdk-17.0.15+6 /usr/lib/jvm/jdk-17.0.15+6 \
70+
&& mv jdk-17.0.15+6 $JAVA_HOME \
6271
&& wget -qO- \
63-
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - \
72+
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar xzf - \
6473
&& mv apache-maven-$MAVEN_VERSION /usr/lib/mvn
6574

66-
# Install R Base
67-
RUN apt-get install -y --no-install-recommends \
68-
libssl-dev \
69-
r-base \
70-
r-base-dev \
71-
r-base-core
72-
7375

7476
# Install R packages
75-
COPY ./src/test/scripts/installDependencies.R installDependencies.R
77+
COPY ./src/test/scripts/installDependencies.R installDependencies.R
7678
RUN Rscript installDependencies.R \
77-
&& rm -rf installDependencies.R \
78-
&& rm -rf /var/lib/apt/lists/*
79+
&& rm -f installDependencies.R
7980

80-
# Install SEAL
81-
RUN wget -qO- https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \
82-
&& cd SEAL-3.7.0 \
83-
&& cmake -S . -B build -DBUILD_SHARED_LIBS=ON \
84-
&& cmake --build build \
85-
&& cmake --install build
81+
# Copy SEAL
82+
COPY --from=seal-build /seal-install /usr/local
8683

8784
# Finally copy the entrypoint script
8885
# This is last to enable quick updates to the script after initial local build.

0 commit comments

Comments
 (0)