Skip to content

DEV: separate builders #882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

DEV: separate builders #882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
00a3088
DEV: separate builders
featheredtoast Oct 16, 2024
d0ed8de
FIX: add font for letterbox avatar generation
featheredtoast Nov 4, 2024
0b722b2
no-install-recommends, install font only
featheredtoast Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 97 additions & 84 deletions image/base/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,60 @@
# VERSION: release

ARG DEBIAN_RELEASE=bookworm
FROM discourse/ruby:3.3.4-${DEBIAN_RELEASE}-slim AS builder
ADD install-imagemagick /tmp/install-imagemagick
# From https://nginx.org/en/pgp_keys.html
ADD nginx_public_keys.key /tmp/nginx_public_keys.key
ADD install-nginx /tmp/install-nginx
RUN apt update && \
DEBIAN_FRONTEND=noninteractive apt-get -y install wget \
autoconf build-essential \
git \
cmake \
gnupg \
libpcre3-dev \
libbrotli-dev
RUN /tmp/install-imagemagick
RUN gpg --import /tmp/nginx_public_keys.key &&\
rm /tmp/nginx_public_keys.key &&\
/tmp/install-nginx

ADD thpoff.c /src/thpoff.c
RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c

FROM discourse/ruby:3.3.4-${DEBIAN_RELEASE}-slim AS discourse_dependencies

ARG DEBIAN_RELEASE
ENV PG_MAJOR=13 \
RUBY_ALLOCATOR=/usr/lib/libjemalloc.so \
LEFTHOOK=0 \
DEBIAN_RELEASE=${DEBIAN_RELEASE}
DEBIAN_RELEASE=${DEBIAN_RELEASE} \
LC_ALL=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LANGUAGE=en_US.UTF-8

#LABEL maintainer="Sam Saffron \"https://twitter.com/samsaffron\""

ADD install-oxipng install-jemalloc /tmp/

# Ensures that the gid and uid of the following users are consistent to avoid permission issues on directories in the
# mounted volumes.
RUN groupadd --gid 104 postgres &&\
useradd --uid 101 --gid 104 --home /var/lib/postgresql --shell /bin/bash -c "PostgreSQL administrator,,," postgres &&\
groupadd --gid 106 redis &&\
useradd --uid 103 --gid 106 --home /var/lib/redis --shell /usr/sbin/nologin redis &&\
groupadd --gid 1000 discourse &&\
useradd --uid 1000 --gid 1000 -m --shell /bin/bash discourse

RUN echo 2.0.`date +%Y%m%d` > /VERSION
RUN echo "deb http://deb.debian.org/debian ${DEBIAN_RELEASE}-backports main" > "/etc/apt/sources.list.d/${DEBIAN_RELEASE}-backports.list"

RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/cache/debconf,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
--mount=type=tmpfs,target=/var/log \
echo "debconf debconf/frontend select Teletype" | debconf-set-selections; \
apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y install gnupg sudo curl fping locales \
--mount=type=tmpfs,target=/root/.npm \
groupadd --gid 104 postgres &&\
useradd --uid 101 --gid 104 --home /var/lib/postgresql --shell /bin/bash -c "PostgreSQL administrator,,," postgres &&\
groupadd --gid 106 redis &&\
useradd --uid 103 --gid 106 --home /var/lib/redis --shell /usr/sbin/nologin redis &&\
groupadd --gid 1000 discourse &&\
useradd --uid 1000 --gid 1000 -m --shell /bin/bash discourse &&\
# add version
echo 2.0.`date +%Y%m%d` > /VERSION &&\
echo "deb http://deb.debian.org/debian ${DEBIAN_RELEASE}-backports main" > "/etc/apt/sources.list.d/${DEBIAN_RELEASE}-backports.list" &&\
echo "debconf debconf/frontend select Teletype" | debconf-set-selections &&\
apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y install gnupg sudo curl fping locales \
ca-certificates rsync \
cmake g++ pkg-config patch \
libxslt-dev libcurl4-openssl-dev \
Expand All @@ -38,94 +64,72 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
libxml2-dev gawk parallel \
libreadline-dev anacron wget \
psmisc whois brotli libunwind-dev \
libtcmalloc-minimal4 cmake \
pngcrush pngquant ripgrep poppler-utils; \
libtcmalloc-minimal4 \
pngquant ripgrep poppler-utils \
# imagemagick runtime dependencies
libheif1 libjbig0 libtiff6 libpng16-16 libfontconfig1 \
libwebpdemux2 libwebpmux3 libxext6 librsvg2-2 libgomp1 \
# nginx runtime dependencies
nginx-common &&\
# install these without recommends to avoid pulling in e.g.
# X11 libraries, mailutils
DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends git rsyslog logrotate cron ssh-client less; \
DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends git rsyslog logrotate cron ssh-client less fonts-urw-base35 &&\
# postgres packages
install -d /usr/share/postgresql-common/pgdg &&\
curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc &&\
echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${DEBIAN_RELEASE}-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \
echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${DEBIAN_RELEASE}-pgdg main" > /etc/apt/sources.list.d/pgdg.list &&\
# yarn packages
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -; \
echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list; \
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - &&\
echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list &&\
# node packages
curl --silent --location https://deb.nodesource.com/setup_18.x | sudo bash -; \
curl --silent --location https://deb.nodesource.com/setup_18.x | sudo bash - &&\
# setup anacron, rsyslog, initctl
sed -i -e 's/start -q anacron/anacron -s/' /etc/cron.d/anacron; \
sed -i.bak 's/$ModLoad imklog/#$ModLoad imklog/' /etc/rsyslog.conf; \
sed -i.bak 's/module(load="imklog")/#module(load="imklog")/' /etc/rsyslog.conf; \
dpkg-divert --local --rename --add /sbin/initctl; \
sh -c "test -f /sbin/initctl || ln -s /bin/true /sbin/initctl"; \
sed -i -e 's/start -q anacron/anacron -s/' /etc/cron.d/anacron &&\
sed -i.bak 's/$ModLoad imklog/#$ModLoad imklog/' /etc/rsyslog.conf &&\
sed -i.bak 's/module(load="imklog")/#module(load="imklog")/' /etc/rsyslog.conf &&\
dpkg-divert --local --rename --add /sbin/initctl &&\
sh -c "test -f /sbin/initctl || ln -s /bin/true /sbin/initctl" &&\
apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y install runit socat \
libpq-dev postgresql-client \
postgresql-${PG_MAJOR} postgresql-contrib-${PG_MAJOR} postgresql-${PG_MAJOR}-pgvector \
nodejs yarn &&\
mkdir -p /etc/runit/1.d

ENV LC_ALL=en_US.UTF-8
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US.UTF-8
RUN sed -i "s/^# $LANG/$LANG/" /etc/locale.gen; \
locale-gen

RUN --mount=type=tmpfs,target=/root/.npm \
npm install -g terser uglify-js pnpm

ADD install-imagemagick /tmp/install-imagemagick
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/cache/debconf,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
/tmp/install-imagemagick

ADD install-jemalloc /tmp/install-jemalloc
RUN /tmp/install-jemalloc

# From https://nginx.org/en/pgp_keys.html
ADD nginx_public_keys.key /tmp/nginx_public_keys.key
ADD install-nginx /tmp/install-nginx

RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/cache/debconf,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
gpg --import /tmp/nginx_public_keys.key &&\
rm /tmp/nginx_public_keys.key &&\
/tmp/install-nginx

ADD install-redis /tmp/install-redis
RUN /tmp/install-redis
mkdir -p /etc/runit/1.d &&\
sed -i "s/^# $LANG/$LANG/" /etc/locale.gen && locale-gen &&\
npm install -g terser uglify-js pnpm &&\
/tmp/install-oxipng &&\
/tmp/install-jemalloc

ADD install-oxipng /tmp/install-oxipng
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/cache/debconf,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
/tmp/install-oxipng

RUN echo 'gem: --no-document' >> /usr/local/etc/gemrc &&\
gem update --system

RUN gem install pups --force &&\
mkdir -p /pups/bin/ &&\
ln -s /usr/local/bin/pups /pups/bin/pups
# Copy binary and configuration files for magick
COPY --from=builder /usr/local/bin/magick /usr/local/bin/magick
COPY --from=builder /usr/local/etc/ImageMagick-7 /usr/local/etc/ImageMagick-7
COPY --from=builder /usr/sbin/nginx /usr/sbin/nginx

# This tool allows us to disable huge page support for our current process
# since the flag is preserved through forks and execs it can be used on any
# process
ADD thpoff.c /src/thpoff.c
RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c

# clean up for docker squash
RUN rm -fr /usr/local/share/doc &&\
rm -fr /usr/local/share/ri &&\
rm -fr /var/lib/apt/lists/* &&\
rm -fr /root/.gem &&\
rm -fr /root/.npm &&\
rm -fr /tmp/*

COPY --from=builder /usr/local/sbin/thpoff /usr/local/sbin/thpoff

# Create symlinks to imagemagick tools
RUN ln -s /usr/local/bin/magick /usr/local/bin/animate &&\
ln -s /usr/local/bin/magick /usr/local/bin/compare &&\
ln -s /usr/local/bin/magick /usr/local/bin/composite &&\
ln -s /usr/local/bin/magick /usr/local/bin/conjure &&\
ln -s /usr/local/bin/magick /usr/local/bin/convert &&\
ln -s /usr/local/bin/magick /usr/local/bin/display &&\
ln -s /usr/local/bin/magick /usr/local/bin/identify &&\
ln -s /usr/local/bin/magick /usr/local/bin/import &&\
ln -s /usr/local/bin/magick /usr/local/bin/magick-script &&\
ln -s /usr/local/bin/magick /usr/local/bin/mogrify &&\
ln -s /usr/local/bin/magick /usr/local/bin/montage &&\
ln -s /usr/local/bin/magick /usr/local/bin/stream &&\
test $(magick -version | grep -o -e png -e tiff -e jpeg -e freetype -e heic -e webp | wc -l) -eq 6 &&\
echo 'gem: --no-document' >> /usr/local/etc/gemrc &&\
gem update --system &&\
gem install pups --force &&\
mkdir -p /pups/bin/ &&\
ln -s /usr/local/bin/pups /pups/bin/pups &&\
# this is required for aarch64 which uses buildx
# see https://github.com/docker/buildx/issues/150
RUN rm -f /etc/service
rm -f /etc/service

COPY etc/ /etc
COPY sbin/ /sbin
Expand All @@ -138,7 +142,7 @@ RUN install -dm 0755 -o discourse -g discourse /var/www/discourse &&\
sudo -u discourse git clone --branch $DISCOURSE_BRANCH --filter=tree:0 https://github.com/discourse/discourse.git /var/www/discourse &&\
gem install bundler --conservative -v $(awk '/BUNDLED WITH/ { getline; gsub(/ /,""); print $0 }' /var/www/discourse/Gemfile.lock)

FROM discourse_slim AS discourse_release
FROM discourse_slim AS discourse_web
ENV RAILS_ENV=production

RUN cd /var/www/discourse &&\
Expand All @@ -151,3 +155,12 @@ RUN cd /var/www/discourse &&\

RUN cd /var/www/discourse &&\
sudo -u discourse /bin/bash -c 'if [ -f yarn.lock ]; then yarn install --frozen-lockfile && yarn cache clean; else pnpm install --frozen-lockfile; fi'

FROM discourse_web AS discourse_release
ADD install-redis /tmp/install-redis
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/cache/debconf,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
--mount=type=tmpfs,target=/var/log \
apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get -y install postgresql-${PG_MAJOR} postgresql-contrib-${PG_MAJOR} postgresql-${PG_MAJOR}-pgvector &&\
/tmp/install-redis
2 changes: 2 additions & 0 deletions image/base/install-imagemagick
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ tar zxf $WDIR/ImageMagick.tar.gz -C $WDIR
cd $IMDIR
PKG_CONF_LIBDIR=$PREFIX/lib LDFLAGS=-L$PREFIX/lib CFLAGS='-O2 -I$PREFIX/include' ./configure \
--prefix=$PREFIX \
--disable-shared \
--enable-delegate-build \
--enable-static \
--enable-bounds-checking \
--enable-hdri \
Expand Down
6 changes: 3 additions & 3 deletions image/base/install-jemalloc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ if uname -m | grep -qi 'aarch64'; then
sha256sum jemalloc-5.3.0.tar.bz2
echo "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa jemalloc-5.3.0.tar.bz2" | sha256sum -c
tar --strip-components=1 -xjf jemalloc-5.3.0.tar.bz2
./configure --prefix=/usr --with-lg-page=16 && make build_lib -j"$(nproc)" && make install_lib
./configure --prefix=/usr --with-lg-page=16 && make build_lib -j"$(nproc)" && make install_lib_shared
cd / && rm -rf /jemalloc-new
else
# jemalloc stable
Expand All @@ -27,7 +27,7 @@ else
sha256sum jemalloc-3.6.0.tar.bz2
echo "e16c2159dd3c81ca2dc3b5c9ef0d43e1f2f45b04548f42db12e7c12d7bdf84fe jemalloc-3.6.0.tar.bz2" | sha256sum -c
tar --strip-components=1 -xjf jemalloc-3.6.0.tar.bz2
./configure --prefix=/usr $EXTRA_CONF && make -j"$(nproc)" && make install
./configure --prefix=/usr $EXTRA_CONF && make -j"$(nproc)" && make install_lib_shared
cd / && rm -rf /jemalloc-stable

# jemalloc new
Expand All @@ -38,6 +38,6 @@ else
sha256sum jemalloc-5.3.0.tar.bz2
echo "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa jemalloc-5.3.0.tar.bz2" | sha256sum -c
tar --strip-components=1 -xjf jemalloc-5.3.0.tar.bz2
./configure --prefix=/usr --with-install-suffix=5.3.0 && make build_lib -j"$(nproc)" && make install_lib
./configure --prefix=/usr --with-install-suffix=5.3.0 && make build_lib -j"$(nproc)" && make install_lib_shared
cd / && rm -rf /jemalloc-new
fi
8 changes: 6 additions & 2 deletions image/base/install-oxipng
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ case "${dpkgArch##*-}" in
esac

# Install other deps
apt -y -q install advancecomp jhead jpegoptim libjpeg-turbo-progs optipng
apt -y -q install advancecomp jpegoptim libjpeg-turbo-progs

git clone --depth 1 --branch "3.08" https://github.com/Matthias-Wandel/jhead.git /tmp/jhead
cd /tmp/jhead && make && cp /tmp/jhead/jhead /usr/local/bin/jhead
cd / && rm -rf /tmp/jhead

mkdir /oxipng-install
cd /oxipng-install
Expand All @@ -22,4 +26,4 @@ sha256sum ${OXIPNG_FILE}
echo "${OXIPNG_HASH} ${OXIPNG_FILE}" | sha256sum -c
tar --strip-components=1 -xzf $OXIPNG_FILE
cp -v ./oxipng /usr/local/bin
cd / && rm -fr /oxipng-install
cd / && rm -fr /oxipng-install
Loading