Skip to content

Commit 6b12363

Browse files
samrosesamrose
committed
chore: introduce tooling for pg 17.2 non-orioledb
1 parent 45e80ac commit 6b12363

23 files changed

+6947
-176
lines changed

‎Dockerfile-17

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
# syntax=docker/dockerfile:1.6
2+
ARG postgresql_major=17-orioledb
3+
ARG postgresql_release=${postgresql_major}.1
4+
5+
# Bump default build arg to build a package from source
6+
# Bump vars.yml to specify runtime package version
7+
ARG sfcgal_release=1.3.10
8+
ARG postgis_release=3.3.2
9+
ARG pgrouting_release=3.4.1
10+
ARG pgtap_release=1.2.0
11+
ARG pg_cron_release=1.6.2
12+
ARG pgaudit_release=1.7.0
13+
ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9
14+
ARG pgsql_http_release=1.5.0
15+
ARG plpgsql_check_release=2.2.5
16+
ARG pg_safeupdate_release=1.4
17+
ARG timescaledb_release=2.9.1
18+
ARG wal2json_release=2_5
19+
ARG pljava_release=1.6.4
20+
ARG plv8_release=3.1.5
21+
ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492
22+
ARG pg_net_release=0.7.1
23+
ARG rum_release=1.3.13
24+
ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
25+
ARG libsodium_release=1.0.18
26+
ARG pgsodium_release=3.1.6
27+
ARG pg_graphql_release=1.5.1
28+
ARG pg_stat_monitor_release=1.1.1
29+
ARG pg_jsonschema_release=0.1.4
30+
ARG pg_repack_release=1.4.8
31+
ARG vault_release=0.2.8
32+
ARG groonga_release=12.0.8
33+
ARG pgroonga_release=2.4.0
34+
ARG wrappers_release=0.3.0
35+
ARG hypopg_release=1.3.1
36+
ARG pgvector_release=0.4.0
37+
ARG pg_tle_release=1.3.2
38+
ARG index_advisor_release=0.2.0
39+
ARG supautils_release=2.2.0
40+
ARG wal_g_release=2.0.1
41+
42+
FROM ubuntu:focal as base
43+
44+
RUN apt update -y && apt install -y \
45+
curl \
46+
gnupg \
47+
lsb-release \
48+
software-properties-common \
49+
wget \
50+
sudo \
51+
tree \
52+
&& apt clean
53+
54+
55+
RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres
56+
RUN adduser --system --no-create-home --shell /bin/bash --group wal-g
57+
RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \
58+
--init none \
59+
--no-confirm \
60+
--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \
61+
--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
62+
63+
ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin"
64+
65+
COPY . /nixpg
66+
67+
WORKDIR /nixpg
68+
69+
RUN nix profile install .#psql_orioledb-17/bin
70+
71+
72+
73+
WORKDIR /
74+
75+
76+
RUN mkdir -p /usr/lib/postgresql/bin \
77+
/usr/lib/postgresql/share/postgresql \
78+
/usr/share/postgresql \
79+
/var/lib/postgresql \
80+
&& chown -R postgres:postgres /usr/lib/postgresql \
81+
&& chown -R postgres:postgres /var/lib/postgresql \
82+
&& chown -R postgres:postgres /usr/share/postgresql
83+
84+
# Create symbolic links
85+
RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \
86+
&& ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \
87+
&& chown -R postgres:postgres /usr/bin
88+
89+
# Create symbolic links for PostgreSQL shares
90+
RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/
91+
RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/
92+
RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/
93+
RUN chown -R postgres:postgres /usr/share/postgresql/
94+
# Create symbolic links for contrib directory
95+
RUN tree /nix > /tmp/tree.txt && cat /tmp/tree.txt && cat /tmp/tree.txt >&2
96+
97+
RUN chown -R postgres:postgres /usr/lib/postgresql
98+
99+
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
100+
101+
102+
RUN apt-get update && \
103+
apt-get install -y --no-install-recommends tzdata
104+
105+
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
106+
dpkg-reconfigure --frontend noninteractive tzdata
107+
108+
RUN apt-get update && \
109+
apt-get install -y --no-install-recommends \
110+
build-essential \
111+
checkinstall \
112+
cmake
113+
114+
ENV PGDATA=/var/lib/postgresql/data
115+
116+
####################
117+
# setup-wal-g.yml
118+
####################
119+
FROM base as walg
120+
ARG wal_g_release
121+
# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz
122+
RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \
123+
apt-get update && apt-get install -y --no-install-recommends curl && \
124+
curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \
125+
tar -xvf /tmp/wal-g.tar.gz -C /tmp && \
126+
rm -rf /tmp/wal-g.tar.gz && \
127+
mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g
128+
129+
# ####################
130+
# # Download gosu for easy step-down from root
131+
# ####################
132+
FROM base as gosu
133+
ARG TARGETARCH
134+
# Install dependencies
135+
RUN apt-get update && apt-get install -y --no-install-recommends \
136+
gnupg \
137+
ca-certificates \
138+
&& rm -rf /var/lib/apt/lists/*
139+
# Download binary
140+
ARG GOSU_VERSION=1.16
141+
ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4
142+
ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \
143+
/usr/local/bin/gosu
144+
ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \
145+
/usr/local/bin/gosu.asc
146+
# Verify checksum
147+
RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \
148+
gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \
149+
gpgconf --kill all && \
150+
chmod +x /usr/local/bin/gosu
151+
152+
# ####################
153+
# # Build final image
154+
# ####################
155+
FROM gosu as production
156+
RUN id postgres || (echo "postgres user does not exist" && exit 1)
157+
# # Setup extensions
158+
COPY --from=walg /tmp/wal-g /usr/local/bin/
159+
160+
# # Initialise configs
161+
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
162+
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
163+
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
164+
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
165+
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
166+
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
167+
COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
168+
COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf
169+
COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf
170+
COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
171+
COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh
172+
173+
RUN sed -i \
174+
-e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
175+
-e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
176+
-e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
177+
-e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
178+
echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
179+
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
180+
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
181+
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
182+
usermod -aG postgres wal-g && \
183+
mkdir -p /etc/postgresql-custom && \
184+
chown postgres:postgres /etc/postgresql-custom
185+
186+
# Remove items from postgresql.conf
187+
RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf"
188+
#as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present
189+
RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf"
190+
RUN sed -i 's/ timescaledb,//g; s/ plv8,//g' "/etc/postgresql-custom/supautils.conf"
191+
192+
193+
194+
# # Include schema migrations
195+
COPY migrations/db /docker-entrypoint-initdb.d/
196+
COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql
197+
COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql
198+
199+
# # Add upstream entrypoint script
200+
COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu
201+
ADD --chmod=0755 \
202+
https://github.com/docker-library/postgres/raw/master/17/bullseye/docker-entrypoint.sh \
203+
/usr/local/bin/
204+
205+
RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql
206+
207+
ENTRYPOINT ["docker-entrypoint.sh"]
208+
209+
HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost
210+
STOPSIGNAL SIGINT
211+
EXPOSE 5432
212+
213+
ENV POSTGRES_HOST=/var/run/postgresql
214+
ENV POSTGRES_USER=supabase_admin
215+
ENV POSTGRES_DB=postgres
216+
ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8"
217+
RUN apt-get update && apt-get install -y --no-install-recommends \
218+
locales \
219+
&& rm -rf /var/lib/apt/lists/* && \
220+
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
221+
&& localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8
222+
RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen
223+
ENV LANG en_US.UTF-8
224+
ENV LANGUAGE en_US:en
225+
ENV LC_ALL en_US.UTF-8
226+
ENV LC_CTYPE=C.UTF-8
227+
ENV LC_COLLATE=C.UTF-8
228+
ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive
229+
CMD ["postgres", "-D", "/etc/postgresql"]

‎ansible/tasks/setup-postgres.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,8 @@
216216
block:
217217
- name: Check if psql_version is psql_orioledb
218218
set_fact:
219-
is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
219+
is_psql_oriole: "{{ psql_version in ['psql_orioledb-17'] }}"
220+
is_psql_17: "{{ psql_version in ['psql_17'] }}"
220221

221222
- name: Initialize the database stage2_nix (non-orioledb)
222223
become: yes
@@ -232,7 +233,7 @@
232233
LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
233234
vars:
234235
ansible_command_timeout: 60
235-
when: stage2_nix and not is_psql_oriole
236+
when: stage2_nix and not is_psql_oriole and not is_psql_17
236237

237238
- name: Initialize the database stage2_nix (orioledb)
238239
become: yes
@@ -254,7 +255,7 @@
254255
LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
255256
vars:
256257
ansible_command_timeout: 60
257-
when: stage2_nix and is_psql_oriole
258+
when: stage2_nix and is_psql_oriole or is_psql_17
258259

259260
- name: copy PG systemd unit
260261
template:

‎ansible/tasks/stage2-setup-postgres.yml

Lines changed: 13 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -6,32 +6,36 @@
66

77
- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary
88
block:
9-
- name: Check if psql_version is psql_orioledb-16
9+
- name: Check if psql_version is psql_orioledb-17
1010
set_fact:
11-
is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
11+
is_psql_oriole: "{{ psql_version in ['psql_orioledb-17'] }}"
1212

13-
- name: Remove specified extensions from postgresql.conf if oriole-16 build
13+
- name: Check if psql_version is psql_17
14+
set_fact:
15+
is_psql_17: "{{ psql_version in ['psql_17'] }}"
16+
17+
- name: Remove specified extensions from postgresql.conf if orioledb-17 or 17 build
1418
ansible.builtin.command:
1519
cmd: >
1620
sed -i 's/ timescaledb,//g'
1721
/etc/postgresql/postgresql.conf
18-
when: is_psql_oriole and stage2_nix
22+
when: is_psql_oriole or is_psql_17 and stage2_nix
1923
become: yes
2024

21-
- name: Remove specified extensions from supautils.conf if oriole-16 build
25+
- name: Remove specified extensions from supautils.conf if orioledb-17 or 17 build
2226
ansible.builtin.command:
2327
cmd: >
24-
sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g'
28+
sed -i 's/ timescaledb,//g; s/ plv8,//g'
2529
/etc/postgresql-custom/supautils.conf
26-
when: is_psql_oriole and stage2_nix
30+
when: is_psql_oriole or is_psql_17 and stage2_nix
2731
become: yes
2832

29-
- name: Remove db_user_namespace from postgresql.conf if oriole-xx build
33+
- name: Remove db_user_namespace from postgresql.conf if orioledb-17 or 17 build
3034
ansible.builtin.command:
3135
cmd: >
3236
sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;'
3337
/etc/postgresql/postgresql.conf
34-
when: is_psql_oriole and stage2_nix
38+
when: is_psql_oriole or is_psql_17 and stage2_nix
3539
become: yes
3640

3741
- name: Append orioledb to shared_preload_libraries append within closing quote
@@ -153,15 +157,6 @@
153157
group: postgres
154158
when: stage2_nix
155159

156-
# - name: Ensure /usr/lib/postgresql/share/postgresql/pljava directory exists
157-
# file:
158-
# path: /usr/lib/postgresql/share/postgresql/pljava
159-
# state: directory
160-
# owner: postgres
161-
# group: postgres
162-
# when: stage2_nix
163-
# It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task
164-
165160
- name: import pgsodium_getkey script
166161
template:
167162
src: files/pgsodium_getkey_readonly.sh.j2
@@ -199,16 +194,6 @@
199194
become: yes
200195
when: stage2_nix
201196

202-
# - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/pljava to /usr/lib/postgresql/share/postgresql/pljava
203-
# file:
204-
# src: "{{ item }}"
205-
# dest: "/usr/lib/postgresql/share/postgresql/pljava/{{ item | basename }}"
206-
# state: link
207-
# with_fileglob:
208-
# - "/var/lib/postgresql/.nix-profile/share/pljava/*"
209-
# become: yes
210-
# It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task
211-
212197
- name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql to /usr/lib/postgresql/share/postgresql
213198
shell: >-
214199
find /var/lib/postgresql/.nix-profile/share/postgresql/ -maxdepth 1 -type f,l -exec sh -c 'ln -s "$0" "/usr/lib/postgresql/share/postgresql/$(basename $0)"' {} \;

‎ansible/tasks/test-image.yml

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,14 @@
6262
block:
6363
- name: Check if psql_version is psql_orioledb-xx
6464
set_fact:
65-
is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
65+
is_psql_oriole: "{{ psql_version in ['psql_orioledb-17'] }}"
66+
is_psql_17: "{{ psql_version in ['psql_17'] }}"
6667

6768
- name: Remove specified extensions from SQL file
6869
ansible.builtin.command:
6970
cmd: >
7071
sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql
71-
when: is_psql_oriole
72+
when: is_psql_oriole or is_psql_17
7273
become: yes
7374

7475
- name: Remove specified extension files from extensions directory
@@ -77,17 +78,15 @@
7778
patterns:
7879
- '*timescaledb*.sql'
7980
- '*plv8*.sql'
80-
- '*postgis*.sql'
81-
- '*pgrouting*.sql'
8281
register: files_to_remove
83-
when: is_psql_oriole
82+
when: is_psql_oriole or is_psql_17
8483

8584
- name: Delete matched extension files
8685
ansible.builtin.file:
8786
path: "{{ item.path }}"
8887
state: absent
8988
loop: "{{ files_to_remove.files }}"
90-
when: is_psql_oriole
89+
when: is_psql_oriole or is_psql_17
9190
become: yes
9291

9392
- name: Run Unit tests (with filename unit-test-*) on Postgres Database

‎ansible/vars.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ async_mode: true
44

55
postgres_major:
66
- "15"
7+
- "17"
78
- "orioledb-17"
89

910
# Full version strings for each major version
1011
postgres_release:
11-
postgresorioledb-17: "17.0.1.34-orioledb"
12+
postgresorioledb-17: "17.0.1.34-orioledb-pg17"
1213
postgres15: "15.8.1.038"
1314

1415
# Non Postgres Extensions

0 commit comments

Comments
 (0)