Skip to content

Commit 92ddd74

Browse files
samrosesamrose
authored
feat: docker base psql15.6 img and release (#1072)
* feat: docker base psql15.6 img and release on merge to develop. nix creates a base version of the image which will be pushed to docker.io on each PR push, ovewriting the same tag. Dockerfile-156 will then use that tag as it's base to build the finale base supabase/postgres image for 15.6 (built from nix) * feat: our nix set up uses skopeo under the hood so we'll try to feed skopeo what it needs to push to docekr hub * chore: update paths * chore: propagate vars in Dockerfile used by gha * chore: use conventional env var * chore: better handling for arm64 and amd64 * chore: fix sha, rules about when wf runs * chore: a different approach to trigger on success * chore: cleaning up to test triggering * test: run in this PR context on any success * chore: just combine docker operations * chore: remove completion rule * chore: account for both scripts * chore: yml formatting * chore: reformat and cleaning up * fix: typo on filename * chore: need some way to run this on push to PR for now * chore: adjusting tmp rules * chore: naild down var names * fix: no need for s3 access here * chore: reorder * chore: not all systems for flake check * chore: use tmp img tag for now * chore: bring in values for base image * fix: version * chore: prefix already passed in * chore: isolate cache to this commit * chore: dockerfile should be file * chore: use root at this layer * chore: rm base image from workflow * chore: db dir * chore: make sure dirs and perms exist * chore: include timezonesets * chore: create symlink * chore: create more missing dirs * chore: more symlinks for the psql package * test: try removing caching of docker image in ci * test: force different tag each time * chore: rm git sha from docker image tag * chore: cleanup for merge * chore: re-intro docker caching --------- Co-authored-by: Sam Rose <[email protected]>
1 parent 40b884c commit 92ddd74

File tree

6 files changed

+381
-20
lines changed

6 files changed

+381
-20
lines changed

‎.github/workflows/dockerhub-release-15-6.yml

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
name: Release 15.6 on Dockerhub
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
paths:
8+
- ".github/workflows/dockerhub-release-15-6.yml"
9+
- "common-nix.vars*"
10+
11+
jobs:
12+
settings:
13+
runs-on: ubuntu-latest
14+
outputs:
15+
docker_version: ${{ steps.settings.outputs.postgres-version }}
16+
image_tag: supabase/postgres:${{ steps.settings.outputs.postgres-version }}
17+
build_args: ${{ steps.args.outputs.result }}
18+
steps:
19+
- uses: actions/checkout@v3
20+
- id: settings
21+
# Remove spaces and quotes to get the raw version string
22+
run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT
23+
- id: args
24+
uses: mikefarah/yq@master
25+
with:
26+
cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
27+
28+
build_release_image:
29+
needs: [settings]
30+
strategy:
31+
matrix:
32+
include:
33+
- runner: [self-hosted, X64]
34+
arch: amd64
35+
- runner: arm-runner
36+
arch: arm64
37+
runs-on: ${{ matrix.runner }}
38+
timeout-minutes: 180
39+
outputs:
40+
image_digest: ${{ steps.build.outputs.digest }}
41+
steps:
42+
- run: docker context create builders
43+
- uses: docker/setup-buildx-action@v3
44+
with:
45+
endpoint: builders
46+
- uses: docker/login-action@v2
47+
with:
48+
username: ${{ secrets.DOCKER_USERNAME }}
49+
password: ${{ secrets.DOCKER_PASSWORD }}
50+
- id: build
51+
uses: docker/build-push-action@v5
52+
with:
53+
push: true
54+
build-args: |
55+
${{ needs.settings.outputs.build_args }}
56+
target: production
57+
tags: ${{ needs.settings.outputs.image_tag }}_${{ matrix.arch }}
58+
platforms: linux/${{ matrix.arch }}
59+
cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
60+
cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
61+
file: "Dockerfile-156"
62+
- name: Slack Notification
63+
if: ${{ failure() }}
64+
uses: rtCamp/action-slack-notify@v2
65+
env:
66+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
67+
SLACK_USERNAME: "gha-failures-notifier"
68+
SLACK_COLOR: "danger"
69+
SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed"
70+
SLACK_FOOTER: ""
71+
72+
merge_manifest:
73+
needs: [settings, build_release_image]
74+
runs-on: ubuntu-latest
75+
steps:
76+
- uses: docker/setup-buildx-action@v3
77+
- uses: docker/login-action@v2
78+
with:
79+
username: ${{ secrets.DOCKER_USERNAME }}
80+
password: ${{ secrets.DOCKER_PASSWORD }}
81+
- name: Merge multi-arch manifests
82+
run: |
83+
docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \
84+
${{ needs.settings.outputs.image_tag }}_amd64 \
85+
${{ needs.settings.outputs.image_tag }}_arm64
86+
- name: Slack Notification
87+
if: ${{ failure() }}
88+
uses: rtCamp/action-slack-notify@v2
89+
env:
90+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
91+
SLACK_USERNAME: "gha-failures-notifier"
92+
SLACK_COLOR: "danger"
93+
SLACK_MESSAGE: "Building Postgres image failed"
94+
SLACK_FOOTER: ""
95+
96+
publish:
97+
needs: [settings, merge_manifest]
98+
# Call workflow explicitly because events from actions cannot trigger more actions
99+
uses: ./.github/workflows/mirror.yml
100+
with:
101+
version: ${{ needs.settings.outputs.docker_version }}
102+
secrets: inherit

‎.github/workflows/nix-build.yml

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,18 @@ jobs:
4242
python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()"
4343
env:
4444
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
45-
46-
- name: Build docker images
45+
- name: Log in to Docker Hub
46+
uses: docker/login-action@v2
47+
with:
48+
username: ${{ secrets.DOCKER_USERNAME }}
49+
password: ${{ secrets.DOCKER_PASSWORD }}
50+
- name: Build psql bundle with nix
4751
run: docker build -t base_nix -f docker/nix/Dockerfile .
4852
- name: Run build psql bundle
49-
run: docker run -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} base_nix bash -c "./workspace/docker/nix/build.sh"
53+
run: |
54+
docker run -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
55+
-e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
56+
-e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
57+
base_nix bash -c "./workspace/docker/nix/build_nix.sh"
5058
name: build psql bundle on ${{ matrix.arch }}
59+

‎Dockerfile-156

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
# syntax=docker/dockerfile:1.6
2+
ARG postgresql_major=15
3+
ARG postgresql_release=${postgresql_major}.1
4+
5+
# Bump default build arg to build a package from source
6+
# Bump vars.yml to specify runtime package version
7+
ARG sfcgal_release=1.3.10
8+
ARG postgis_release=3.3.2
9+
ARG pgrouting_release=3.4.1
10+
ARG pgtap_release=1.2.0
11+
ARG pg_cron_release=1.6.2
12+
ARG pgaudit_release=1.7.0
13+
ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9
14+
ARG pgsql_http_release=1.5.0
15+
ARG plpgsql_check_release=2.2.5
16+
ARG pg_safeupdate_release=1.4
17+
ARG timescaledb_release=2.9.1
18+
ARG wal2json_release=2_5
19+
ARG pljava_release=1.6.4
20+
ARG plv8_release=3.1.5
21+
ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492
22+
ARG pg_net_release=0.7.1
23+
ARG rum_release=1.3.13
24+
ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
25+
ARG libsodium_release=1.0.18
26+
ARG pgsodium_release=3.1.6
27+
ARG pg_graphql_release=1.5.1
28+
ARG pg_stat_monitor_release=1.1.1
29+
ARG pg_jsonschema_release=0.1.4
30+
ARG pg_repack_release=1.4.8
31+
ARG vault_release=0.2.8
32+
ARG groonga_release=12.0.8
33+
ARG pgroonga_release=2.4.0
34+
ARG wrappers_release=0.3.0
35+
ARG hypopg_release=1.3.1
36+
ARG pgvector_release=0.4.0
37+
ARG pg_tle_release=1.3.2
38+
ARG index_advisor_release=0.2.0
39+
ARG supautils_release=2.2.0
40+
ARG wal_g_release=2.0.1
41+
42+
FROM ubuntu:focal as base
43+
44+
RUN apt update -y && apt install -y \
45+
curl \
46+
gnupg \
47+
lsb-release \
48+
software-properties-common \
49+
wget \
50+
sudo \
51+
&& apt clean
52+
53+
54+
RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres
55+
RUN adduser --system --no-create-home --shell /bin/bash --group wal-g
56+
RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \
57+
--init none \
58+
--no-confirm \
59+
--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \
60+
--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
61+
62+
ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin"
63+
64+
COPY . /nixpg
65+
66+
WORKDIR /nixpg
67+
68+
RUN nix profile install .#psql_15/bin
69+
70+
71+
72+
WORKDIR /
73+
74+
75+
RUN mkdir -p /usr/lib/postgresql/bin \
76+
/usr/lib/postgresql/share/postgresql \
77+
/usr/share/postgresql \
78+
# /usr/lib/postgresql/share/postgresql/contrib \
79+
#/usr/lib/postgresql/share/postgresql/timezonesets \
80+
#/usr/lib/postgresql/share/postgresql/tsearch_data \
81+
# /usr/lib/postgresql/share/postgresql/extension \
82+
&& chown -R postgres:postgres /usr/lib/postgresql \
83+
&& chown -R postgres:postgres /usr/share/postgresql
84+
85+
# Create symbolic links
86+
RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \
87+
&& ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \
88+
&& chown -R postgres:postgres /usr/bin
89+
90+
# Create symbolic links for PostgreSQL shares
91+
RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/
92+
RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/
93+
RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/
94+
RUN chown -R postgres:postgres /usr/share/postgresql/
95+
# Create symbolic links for contrib directory
96+
RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \
97+
&& find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \
98+
&& chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/
99+
100+
RUN chown -R postgres:postgres /usr/lib/postgresql
101+
102+
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
103+
104+
105+
RUN apt-get update && \
106+
apt-get install -y --no-install-recommends tzdata
107+
108+
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
109+
dpkg-reconfigure --frontend noninteractive tzdata
110+
111+
RUN apt-get update && \
112+
apt-get install -y --no-install-recommends \
113+
build-essential \
114+
checkinstall \
115+
cmake
116+
117+
ENV PGDATA=/var/lib/postgresql/data
118+
119+
####################
120+
# setup-wal-g.yml
121+
####################
122+
FROM base as walg
123+
ARG wal_g_release
124+
# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz
125+
RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \
126+
apt-get update && apt-get install -y --no-install-recommends curl && \
127+
curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \
128+
tar -xvf /tmp/wal-g.tar.gz -C /tmp && \
129+
rm -rf /tmp/wal-g.tar.gz && \
130+
mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g
131+
132+
# ####################
133+
# # Download gosu for easy step-down from root
134+
# ####################
135+
FROM base as gosu
136+
ARG TARGETARCH
137+
# Install dependencies
138+
RUN apt-get update && apt-get install -y --no-install-recommends \
139+
gnupg \
140+
ca-certificates \
141+
&& rm -rf /var/lib/apt/lists/*
142+
# Download binary
143+
ARG GOSU_VERSION=1.16
144+
ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4
145+
ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \
146+
/usr/local/bin/gosu
147+
ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \
148+
/usr/local/bin/gosu.asc
149+
# Verify checksum
150+
RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \
151+
gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \
152+
gpgconf --kill all && \
153+
chmod +x /usr/local/bin/gosu
154+
155+
# ####################
156+
# # Build final image
157+
# ####################
158+
FROM gosu as production
159+
RUN id postgres || (echo "postgres user does not exist" && exit 1)
160+
# # Setup extensions
161+
COPY --from=walg /tmp/wal-g /usr/local/bin/
162+
163+
# # Initialise configs
164+
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
165+
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
166+
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
167+
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
168+
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
169+
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
170+
COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
171+
COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf
172+
COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf
173+
COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
174+
COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh
175+
176+
RUN sed -i \
177+
-e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
178+
-e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
179+
-e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
180+
-e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
181+
echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
182+
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
183+
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
184+
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
185+
usermod -aG postgres wal-g && \
186+
mkdir -p /etc/postgresql-custom && \
187+
chown postgres:postgres /etc/postgresql-custom
188+
189+
# # Include schema migrations
190+
COPY migrations/db /docker-entrypoint-initdb.d/
191+
COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql
192+
COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql
193+
194+
# # Add upstream entrypoint script
195+
COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu
196+
ADD --chmod=0755 \
197+
https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \
198+
/usr/local/bin/
199+
200+
RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql
201+
202+
ENTRYPOINT ["docker-entrypoint.sh"]
203+
204+
HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost
205+
STOPSIGNAL SIGINT
206+
EXPOSE 5432
207+
208+
ENV POSTGRES_HOST=/var/run/postgresql
209+
RUN apt-get update && apt-get install -y --no-install-recommends \
210+
locales \
211+
&& rm -rf /var/lib/apt/lists/* && \
212+
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
213+
&& localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8
214+
RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen
215+
ENV LANG en_US.UTF-8
216+
ENV LANGUAGE en_US:en
217+
ENV LC_ALL en_US.UTF-8
218+
ENV LC_CTYPE=C.UTF-8
219+
ENV LC_COLLATE=C.UTF-8
220+
ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive
221+
CMD ["postgres", "-D", "/etc/postgresql"]
222+

‎docker/nix/Dockerfile

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ RUN mkdir -p /workspace
1111

1212
COPY ./ /workspace
1313

14-
RUN chmod +x /workspace/docker/nix/build.sh
15-
16-
RUN chown -R $USER:$USER /workspace
17-
14+
RUN chmod +x /workspace/docker/nix/build_nix.sh
1815

16+
RUN chown -R $USER:$USER /workspace

‎docker/nix/build.sh renamed to ‎docker/nix/build_nix.sh

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,5 @@ nix --version
55
cd /workspace
66
nix build .#psql_15/bin -o psql_15
77
nix build .#psql_15/docker
8-
nix flake check -L --all-systems
8+
nix flake check -L
99
nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
10-
11-
#a future step nix run .#psql_15/docker.copyToRegistry

0 commit comments

Comments
 (0)