Skip to content

Commit b4f37a0

Browse files
hunleydhunleyd
authored
Merge branch 'develop' into INDATA-255
2 parents a9c19fd + c53a4f8 commit b4f37a0

File tree

10 files changed

+1613
-15
lines changed

10 files changed

+1613
-15
lines changed

‎.github/workflows/base-image-nightly.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Build Base Image Nightly
2+
3+
on:
4+
#schedule:
5+
# - cron: '0 2 * * *' # 2 AM UTC daily
6+
workflow_dispatch:
7+
inputs:
8+
branch:
9+
description: 'Branch to build from'
10+
required: false
11+
default: 'develop'
12+
type: string
13+
14+
permissions:
15+
contents: read
16+
id-token: write
17+
18+
jobs:
19+
build-base-image:
20+
runs-on: blacksmith-4vcpu-ubuntu-2404-arm
21+
timeout-minutes: 150
22+
23+
steps:
24+
- name: Checkout Repo
25+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
26+
with:
27+
ref: ${{ github.event.inputs.branch || 'develop' }}
28+
29+
- name: Configure AWS credentials
30+
uses: aws-actions/configure-aws-credentials@v4
31+
with:
32+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
33+
aws-region: "us-east-1"
34+
output-credentials: true
35+
role-duration-seconds: 7200
36+
37+
- name: Install nix
38+
uses: cachix/install-nix-action@v27
39+
with:
40+
install_url: https://releases.nixos.org/nix/nix-2.29.1/install
41+
extra_nix_config: |
42+
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
43+
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
44+
45+
- name: Set execution ID and timestamp
46+
run: |
47+
echo "EXECUTION_ID=${{ github.run_id }}-base-nightly" >> $GITHUB_ENV
48+
echo "BUILD_TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
49+
50+
- name: Build base stage 1 AMI
51+
env:
52+
AWS_MAX_ATTEMPTS: 10
53+
AWS_RETRY_MODE: adaptive
54+
run: |
55+
GIT_SHA=${{ github.sha }}
56+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
57+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build \
58+
-var "git-head-version=${GIT_SHA}" \
59+
-var "packer-execution-id=${EXECUTION_ID}" \
60+
-var-file="development-arm.vars.pkr.hcl" \
61+
-var "base-image-nightly=true" \
62+
-var "build-timestamp=${BUILD_TIMESTAMP}" \
63+
-var "region=us-east-1" \
64+
-var 'ami_regions=["us-east-1","ap-southeast-1"]' \
65+
amazon-arm64-nix.pkr.hcl
66+
67+
- name: Slack Notification on Failure
68+
if: ${{ failure() }}
69+
uses: rtCamp/action-slack-notify@v2
70+
env:
71+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
72+
SLACK_USERNAME: 'gha-failures-notifier'
73+
SLACK_COLOR: 'danger'
74+
SLACK_MESSAGE: 'Building base image nightly failed'
75+
SLACK_FOOTER: ''
76+
77+
- name: Cleanup resources after build
78+
if: ${{ always() }}
79+
run: |
80+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
81+
82+
- name: Cleanup resources on build cancellation
83+
if: ${{ cancelled() }}
84+
run: |
85+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids

‎Dockerfile-15‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,56 @@ ARG wal_g_release=2.0.1
4040

4141
FROM ubuntu:noble as base
4242

43-
RUN apt update -y && apt install -y \
43+
# Create reusable apt mirror fallback function
44+
RUN echo '#!/bin/bash\n\
45+
apt_update_with_fallback() {\n\
46+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
47+
local max_attempts=2\n\
48+
local attempt=1\n\
49+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
50+
\n\
51+
for mirror in $mirrors; do\n\
52+
echo "========================================="\n\
53+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
54+
echo "Attempt ${attempt} of ${max_attempts}"\n\
55+
echo "========================================="\n\
56+
\n\
57+
if [ -f "${sources_file}" ]; then\n\
58+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
59+
fi\n\
60+
\n\
61+
if timeout 300 apt-get update 2>&1; then\n\
62+
echo "========================================="\n\
63+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
64+
echo "========================================="\n\
65+
return 0\n\
66+
else\n\
67+
local exit_code=$?\n\
68+
echo "========================================="\n\
69+
echo "✗ Failed to update using mirror: ${mirror}"\n\
70+
echo "Exit code: ${exit_code}"\n\
71+
echo "========================================="\n\
72+
\n\
73+
apt-get clean\n\
74+
rm -rf /var/lib/apt/lists/*\n\
75+
\n\
76+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
77+
local sleep_time=$((attempt * 5))\n\
78+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
79+
sleep ${sleep_time}\n\
80+
fi\n\
81+
fi\n\
82+
\n\
83+
attempt=$((attempt + 1))\n\
84+
done\n\
85+
\n\
86+
echo "========================================="\n\
87+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
88+
echo "========================================="\n\
89+
return 1\n\
90+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
91+
92+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4493
curl \
4594
gnupg \
4695
lsb-release \
@@ -96,13 +145,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
96145
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
97146

98147

99-
RUN apt-get update && \
148+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
100149
apt-get install -y --no-install-recommends tzdata
101150

102151
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
103152
dpkg-reconfigure --frontend noninteractive tzdata
104153

105-
RUN apt-get update && \
154+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
106155
apt-get install -y --no-install-recommends \
107156
build-essential \
108157
checkinstall \
@@ -143,7 +192,7 @@ WORKDIR /
143192
FROM base as gosu
144193
ARG TARGETARCH
145194
# Install dependencies
146-
RUN apt-get update && apt-get install -y --no-install-recommends \
195+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
147196
gnupg \
148197
ca-certificates \
149198
&& rm -rf /var/lib/apt/lists/*
@@ -218,7 +267,7 @@ EXPOSE 5432
218267
ENV POSTGRES_HOST=/var/run/postgresql
219268
ENV POSTGRES_USER=supabase_admin
220269
ENV POSTGRES_DB=postgres
221-
RUN apt-get update && apt-get install -y --no-install-recommends \
270+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
222271
locales \
223272
&& rm -rf /var/lib/apt/lists/* && \
224273
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

‎Dockerfile-17‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,56 @@ ARG wal_g_release=3.0.5
4141

4242
FROM ubuntu:noble as base
4343

44-
RUN apt update -y && apt install -y \
44+
# Create reusable apt mirror fallback function
45+
RUN echo '#!/bin/bash\n\
46+
apt_update_with_fallback() {\n\
47+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
48+
local max_attempts=2\n\
49+
local attempt=1\n\
50+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
51+
\n\
52+
for mirror in $mirrors; do\n\
53+
echo "========================================="\n\
54+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
55+
echo "Attempt ${attempt} of ${max_attempts}"\n\
56+
echo "========================================="\n\
57+
\n\
58+
if [ -f "${sources_file}" ]; then\n\
59+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
60+
fi\n\
61+
\n\
62+
if timeout 300 apt-get update 2>&1; then\n\
63+
echo "========================================="\n\
64+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
65+
echo "========================================="\n\
66+
return 0\n\
67+
else\n\
68+
local exit_code=$?\n\
69+
echo "========================================="\n\
70+
echo "✗ Failed to update using mirror: ${mirror}"\n\
71+
echo "Exit code: ${exit_code}"\n\
72+
echo "========================================="\n\
73+
\n\
74+
apt-get clean\n\
75+
rm -rf /var/lib/apt/lists/*\n\
76+
\n\
77+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
78+
local sleep_time=$((attempt * 5))\n\
79+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
80+
sleep ${sleep_time}\n\
81+
fi\n\
82+
fi\n\
83+
\n\
84+
attempt=$((attempt + 1))\n\
85+
done\n\
86+
\n\
87+
echo "========================================="\n\
88+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
89+
echo "========================================="\n\
90+
return 1\n\
91+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
92+
93+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4594
curl \
4695
gnupg \
4796
lsb-release \
@@ -100,13 +149,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
100149
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
101150

102151

103-
RUN apt-get update && \
152+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
104153
apt-get install -y --no-install-recommends tzdata
105154

106155
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
107156
dpkg-reconfigure --frontend noninteractive tzdata
108157

109-
RUN apt-get update && \
158+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
110159
apt-get install -y --no-install-recommends \
111160
build-essential \
112161
checkinstall \
@@ -148,7 +197,7 @@ WORKDIR /
148197
FROM base as gosu
149198
ARG TARGETARCH
150199
# Install dependencies
151-
RUN apt-get update && apt-get install -y --no-install-recommends \
200+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
152201
gnupg \
153202
ca-certificates \
154203
&& rm -rf /var/lib/apt/lists/*
@@ -232,7 +281,7 @@ ENV POSTGRES_HOST=/var/run/postgresql
232281
ENV POSTGRES_USER=supabase_admin
233282
ENV POSTGRES_DB=postgres
234283
ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8"
235-
RUN apt-get update && apt-get install -y --no-install-recommends \
284+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
236285
locales \
237286
&& rm -rf /var/lib/apt/lists/* && \
238287
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

‎Dockerfile-orioledb-17‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,56 @@ ARG wal_g_release=3.0.5
4141

4242
FROM ubuntu:noble as base
4343

44-
RUN apt update -y && apt install -y \
44+
# Create reusable apt mirror fallback function
45+
RUN echo '#!/bin/bash\n\
46+
apt_update_with_fallback() {\n\
47+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
48+
local max_attempts=2\n\
49+
local attempt=1\n\
50+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
51+
\n\
52+
for mirror in $mirrors; do\n\
53+
echo "========================================="\n\
54+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
55+
echo "Attempt ${attempt} of ${max_attempts}"\n\
56+
echo "========================================="\n\
57+
\n\
58+
if [ -f "${sources_file}" ]; then\n\
59+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
60+
fi\n\
61+
\n\
62+
if timeout 300 apt-get update 2>&1; then\n\
63+
echo "========================================="\n\
64+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
65+
echo "========================================="\n\
66+
return 0\n\
67+
else\n\
68+
local exit_code=$?\n\
69+
echo "========================================="\n\
70+
echo "✗ Failed to update using mirror: ${mirror}"\n\
71+
echo "Exit code: ${exit_code}"\n\
72+
echo "========================================="\n\
73+
\n\
74+
apt-get clean\n\
75+
rm -rf /var/lib/apt/lists/*\n\
76+
\n\
77+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
78+
local sleep_time=$((attempt * 5))\n\
79+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
80+
sleep ${sleep_time}\n\
81+
fi\n\
82+
fi\n\
83+
\n\
84+
attempt=$((attempt + 1))\n\
85+
done\n\
86+
\n\
87+
echo "========================================="\n\
88+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
89+
echo "========================================="\n\
90+
return 1\n\
91+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
92+
93+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4594
curl \
4695
gnupg \
4796
lsb-release \
@@ -100,13 +149,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
100149
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
101150

102151

103-
RUN apt-get update && \
152+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
104153
apt-get install -y --no-install-recommends tzdata
105154

106155
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
107156
dpkg-reconfigure --frontend noninteractive tzdata
108157

109-
RUN apt-get update && \
158+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
110159
apt-get install -y --no-install-recommends \
111160
build-essential \
112161
checkinstall \
@@ -148,7 +197,7 @@ WORKDIR /
148197
FROM base as gosu
149198
ARG TARGETARCH
150199
# Install dependencies
151-
RUN apt-get update && apt-get install -y --no-install-recommends \
200+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
152201
gnupg \
153202
ca-certificates \
154203
&& rm -rf /var/lib/apt/lists/*
@@ -243,7 +292,7 @@ ENV POSTGRES_HOST=/var/run/postgresql
243292
ENV POSTGRES_USER=supabase_admin
244293
ENV POSTGRES_DB=postgres
245294
ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8"
246-
RUN apt-get update && apt-get install -y --no-install-recommends \
295+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
247296
locales \
248297
&& rm -rf /var/lib/apt/lists/* && \
249298
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

‎nix/docs/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md).
1111
## Development
1212

1313
- **[Nix tree structure](./nix-directory-structure.md)** - Overview of the Nix directory structure
14+
- **[Flake-Parts Architecture](./flake-parts-architecture.md)** - Deep dive into the flake-parts module system
15+
- **[Flake-Parts and nixpkgs lib](./flake-parts-nixpkgs-lib.md)** - How flake-parts uses nixpkgs lib foundations
1416
- **[Development Workflow](./development-workflow.md)** - Complete development and testing workflow
1517
- **[Build PostgreSQL](./build-postgres.md)** - Building PostgreSQL from source
1618
- **[Receipt Files](./receipt-files.md)** - Understanding build receipts

‎nix/docs/adding-new-package.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Adding a new extension package
22

3+
!!! tip "Understanding the Module System"
4+
To better understand how packages are organized and how `ourExtensions` works with flake-parts, see:
5+
6+
- **[Flake-Parts Architecture](./flake-parts-architecture.md)** - Module structure overview
7+
- **[Flake-Parts and nixpkgs lib](./flake-parts-nixpkgs-lib.md)** - Extension composition patterns
38

49
## Pre-packaging steps
510
1. Make sure you have nix installed [Nix installer](https://github.com/DeterminateSystems/nix-installer)

0 commit comments

Comments
 (0)