Skip to content

Commit ea10189

Browse files
soedirgosoedirgo
committed
Merge branch 'develop' into or/pgmq-upgrade-seqs
2 parents b57a412 + 9904831 commit ea10189

File tree

247 files changed

+19689
-4165
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

247 files changed

+19689
-4165
lines changed

‎.envrc.recommended

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
watch_file nix/devShells.nix
2+
3+
use flake

‎.github/CODEOWNERS

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
* @supabase/backend
2-
migrations/ @supabase/cli @supabase/backend
1+
* @supabase/backend @supabase/postgres
2+
migrations/ @supabase/dev-workflows @supabase/postgres @supabase/backend
33
docker/orioledb @supabase/postgres @supabase/backend
44
common.vars.pkr.hcl @supabase/postgres @supabase/backend

‎.github/actions/shared-checkout/action.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name: Checkout
2+
description: Checkout repository for pull requests and branches
3+
runs:
4+
using: "composite"
5+
steps:
6+
- name: Checkout repository
7+
uses: actions/checkout@v4
8+
with:
9+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
10+
ref: ${{ github.event.pull_request.head.sha || github.sha }}
11+
fetch-depth: 0
12+
fetch-tags: true

‎.github/workflows/ami-release-nix-single.yml

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
name: Release Single AMI Nix
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
postgres_version:
7+
description: 'PostgreSQL major version to build (e.g. 15)'
8+
required: true
9+
type: string
10+
branch:
11+
description: 'Branch to run the workflow from'
12+
required: true
13+
type: string
14+
default: 'main'
15+
16+
permissions:
17+
contents: write
18+
id-token: write
19+
20+
jobs:
21+
build:
22+
runs-on: large-linux-arm
23+
timeout-minutes: 150
24+
25+
steps:
26+
- name: Checkout Repo
27+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
28+
with:
29+
ref: ${{ github.event.inputs.branch }}
30+
- name: aws-creds
31+
uses: aws-actions/configure-aws-credentials@v4
32+
with:
33+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
34+
aws-region: "us-east-1"
35+
output-credentials: true
36+
role-duration-seconds: 7200
37+
38+
- name: Get current branch SHA
39+
id: get_sha
40+
run: |
41+
echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
42+
43+
- uses: DeterminateSystems/nix-installer-action@main
44+
45+
- name: Set PostgreSQL version environment variable
46+
run: echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV
47+
48+
- name: Generate common-nix.vars.pkr.hcl
49+
run: |
50+
PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml)
51+
PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes
52+
echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
53+
# Ensure there's a newline at the end of the file
54+
echo "" >> common-nix.vars.pkr.hcl
55+
56+
- name: Build AMI stage 1
57+
env:
58+
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
59+
run: |
60+
packer init amazon-arm64-nix.pkr.hcl
61+
GIT_SHA=${{ steps.get_sha.outputs.sha }}
62+
packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
63+
64+
- name: Build AMI stage 2
65+
env:
66+
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
67+
run: |
68+
packer init stage2-nix-psql.pkr.hcl
69+
GIT_SHA=${{ steps.get_sha.outputs.sha }}
70+
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
71+
packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
72+
73+
- name: Grab release version
74+
id: process_release_version
75+
run: |
76+
VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
77+
echo "version=$VERSION" >> $GITHUB_OUTPUT
78+
79+
- name: Create nix flake revision tarball
80+
run: |
81+
GIT_SHA=${{ steps.get_sha.outputs.sha }}
82+
MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
83+
84+
mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
85+
echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
86+
tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" .
87+
88+
- name: configure aws credentials - staging
89+
uses: aws-actions/configure-aws-credentials@v4
90+
with:
91+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
92+
aws-region: "us-east-1"
93+
94+
- name: Upload software manifest to s3 staging
95+
run: |
96+
cd ansible
97+
ansible-playbook -i localhost \
98+
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
99+
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
100+
-e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
101+
manifest-playbook.yml
102+
103+
- name: Upload nix flake revision to s3 staging
104+
run: |
105+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
106+
107+
- name: configure aws credentials - prod
108+
uses: aws-actions/configure-aws-credentials@v4
109+
with:
110+
role-to-assume: ${{ secrets.PROD_AWS_ROLE }}
111+
aws-region: "us-east-1"
112+
113+
- name: Upload software manifest to s3 prod
114+
run: |
115+
cd ansible
116+
ansible-playbook -i localhost \
117+
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
118+
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
119+
-e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
120+
manifest-playbook.yml
121+
122+
- name: Upload nix flake revision to s3 prod
123+
run: |
124+
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
125+
126+
- name: Create release
127+
uses: softprops/action-gh-release@v2
128+
with:
129+
name: ${{ steps.process_release_version.outputs.version }}
130+
tag_name: ${{ steps.process_release_version.outputs.version }}
131+
target_commitish: ${{ steps.get_sha.outputs.sha }}
132+
133+
- name: Slack Notification on Failure
134+
if: ${{ failure() }}
135+
uses: rtCamp/action-slack-notify@v2
136+
env:
137+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
138+
SLACK_USERNAME: 'gha-failures-notifier'
139+
SLACK_COLOR: 'danger'
140+
SLACK_MESSAGE: 'Building Postgres AMI failed'
141+
SLACK_FOOTER: ''
142+
143+
- name: Cleanup resources after build
144+
if: ${{ always() }}
145+
run: |
146+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
147+
148+
- name: Cleanup resources on build cancellation
149+
if: ${{ cancelled() }}
150+
run: |
151+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
152+

‎.github/workflows/ami-release-nix.yml

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,19 @@ on:
1111
- 'ansible/vars.yml'
1212
workflow_dispatch:
1313

14+
permissions:
15+
contents: write
16+
id-token: write
17+
1418
jobs:
1519
prepare:
16-
runs-on: ubuntu-latest
20+
runs-on: large-linux-x86
1721
outputs:
1822
postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
1923
steps:
2024
- name: Checkout Repo
21-
uses: actions/checkout@v3
22-
25+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
26+
2327
- uses: DeterminateSystems/nix-installer-action@main
2428

2529
- name: Set PostgreSQL versions
@@ -34,29 +38,27 @@ jobs:
3438
matrix:
3539
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
3640
include:
37-
- runner: arm-runner
38-
arch: arm64
39-
ubuntu_release: focal
40-
ubuntu_version: 20.04
41-
mcpu: neoverse-n1
41+
- runner: large-linux-arm
4242
runs-on: ${{ matrix.runner }}
4343
timeout-minutes: 150
44-
permissions:
45-
contents: write
46-
packages: write
47-
id-token: write
4844

4945
steps:
5046
- name: Checkout Repo
51-
uses: actions/checkout@v3
52-
47+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
48+
- name: aws-creds
49+
uses: aws-actions/configure-aws-credentials@v4
50+
with:
51+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
52+
aws-region: "us-east-1"
53+
output-credentials: true
54+
role-duration-seconds: 7200
5355
- uses: DeterminateSystems/nix-installer-action@main
5456

5557
- name: Run checks if triggered manually
5658
if: ${{ github.event_name == 'workflow_dispatch' }}
5759
run: |
58-
SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/')
59-
if [[ -z $SUFFIX ]] ; then
60+
SUFFIX=$(nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/')
61+
if [[ -z "$SUFFIX" ]] ; then
6062
echo "Version must include non-numeric characters if built manually."
6163
exit 1
6264
fi
@@ -66,8 +68,8 @@ jobs:
6668

6769
- name: Generate common-nix.vars.pkr.hcl
6870
run: |
69-
PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
70-
PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
71+
PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
72+
PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes
7173
echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
7274
# Ensure there's a newline at the end of the file
7375
echo "" >> common-nix.vars.pkr.hcl

‎.github/workflows/check-shellscripts.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,15 @@ on:
77
pull_request:
88
workflow_dispatch:
99

10+
permissions:
11+
contents: read
12+
1013
jobs:
1114
build:
1215
runs-on: ubuntu-latest
1316
steps:
14-
- uses: actions/checkout@v3
17+
- name: Checkout Repo
18+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
1519
- name: Run ShellCheck
1620
uses: ludeeus/action-shellcheck@master
1721
env:

‎.github/workflows/ci.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@ name: Check merge requirements
33
on:
44
pull_request:
55

6+
permissions:
7+
contents: read
8+
69
jobs:
710
check-release-version:
811
timeout-minutes: 5
912
runs-on: ubuntu-latest
1013
steps:
1114
- name: Checkout Repo
12-
uses: actions/checkout@v3
15+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
1316

1417
- name: Load postgres_release values
1518
id: load_postgres_release

‎.github/workflows/dockerhub-release-matrix.yml

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ on:
99
- ".github/workflows/dockerhub-release-matrix.yml"
1010
- "ansible/vars.yml"
1111
workflow_dispatch:
12-
12+
13+
permissions:
14+
contents: read
15+
id-token: write
16+
1317
jobs:
1418
prepare:
15-
runs-on: ubuntu-latest
19+
runs-on: large-linux-x86
1620
outputs:
1721
matrix_config: ${{ steps.set-matrix.outputs.matrix_config }}
1822
steps:
1923
- uses: DeterminateSystems/nix-installer-action@main
2024
- name: Checkout Repo
21-
uses: actions/checkout@v3
25+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
2226
- name: Generate build matrix
2327
id: set-matrix
2428
run: |
@@ -45,11 +49,12 @@ jobs:
4549
needs: prepare
4650
strategy:
4751
matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }}
48-
runs-on: ubuntu-latest
52+
runs-on: large-linux-x86
4953
outputs:
5054
build_args: ${{ steps.args.outputs.result }}
5155
steps:
52-
- uses: actions/checkout@v3
56+
- name: Checkout Repo
57+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
5358
- uses: DeterminateSystems/nix-installer-action@main
5459
- name: Set PostgreSQL version environment variable
5560
run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV
@@ -70,10 +75,11 @@ jobs:
7075
matrix:
7176
postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
7277
arch: [amd64, arm64]
73-
runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }}
78+
runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }}
7479
timeout-minutes: 180
7580
steps:
76-
- uses: actions/checkout@v3
81+
- name: Checkout Repo
82+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
7783
- uses: DeterminateSystems/nix-installer-action@main
7884
- run: docker context create builders
7985
- uses: docker/setup-buildx-action@v3
@@ -87,7 +93,7 @@ jobs:
8793
id: image
8894
run: |
8995
if [[ "${{ matrix.arch }}" == "arm64" ]]; then
90-
pg_version=$(sudo nix run nixpkgs#nushell -- -c '
96+
pg_version=$(nix run nixpkgs#nushell -- -c '
9197
let version = "${{ matrix.postgres.version }}"
9298
let release_key = if ($version | str contains "orioledb") {
9399
$"postgresorioledb-17"
@@ -126,9 +132,10 @@ jobs:
126132
strategy:
127133
matrix:
128134
include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
129-
runs-on: ubuntu-latest
135+
runs-on: large-linux-x86
130136
steps:
131-
- uses: actions/checkout@v3
137+
- name: Checkout Repo
138+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
132139
- uses: DeterminateSystems/nix-installer-action@main
133140
- uses: docker/setup-buildx-action@v3
134141
- uses: docker/login-action@v2
@@ -169,9 +176,10 @@ jobs:
169176
${{ steps.get_version.outputs.pg_version }}_arm64
170177
combine_results:
171178
needs: [prepare, merge_manifest]
172-
runs-on: ubuntu-latest
179+
runs-on: large-linux-x86
173180
steps:
174-
- uses: actions/checkout@v3
181+
- name: Checkout Repo
182+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
175183
- uses: DeterminateSystems/nix-installer-action@main
176184

177185
- name: Debug Input from Prepare
@@ -233,6 +241,10 @@ jobs:
233241
matrix: ${{ steps.combine.outputs.matrix }}
234242
publish:
235243
needs: combine_results
244+
permissions:
245+
contents: read
246+
packages: write
247+
id-token: write
236248
strategy:
237249
matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }}
238250
uses: ./.github/workflows/mirror.yml

0 commit comments

Comments
 (0)