Skip to content

Commit 8910ea0

Browse files
pcncpcnc
committed
Merge branch 'develop' into pcnc/release156-develop
2 parents 4b0e661 + 4a1d472 commit 8910ea0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+4250
-388
lines changed

‎.github/workflows/ami-release-nix.yml‎

Lines changed: 47 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,31 @@ on:
88
paths:
99
- '.github/workflows/ami-release-nix.yml'
1010
- 'common-nix.vars.pkr.hcl'
11+
- 'ansible/vars.yml'
1112
workflow_dispatch:
1213

1314
jobs:
15+
prepare:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
19+
steps:
20+
- name: Checkout Repo
21+
uses: actions/checkout@v3
22+
23+
- uses: DeterminateSystems/nix-installer-action@main
24+
25+
- name: Set PostgreSQL versions
26+
id: set-versions
27+
run: |
28+
VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
29+
echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
30+
1431
build:
32+
needs: prepare
1533
strategy:
1634
matrix:
35+
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
1736
include:
1837
- runner: arm-runner
1938
arch: arm64
@@ -31,42 +50,55 @@ jobs:
3150
- name: Checkout Repo
3251
uses: actions/checkout@v3
3352

53+
- uses: DeterminateSystems/nix-installer-action@main
54+
3455
- name: Run checks if triggered manually
3556
if: ${{ github.event_name == 'workflow_dispatch' }}
36-
# Update `ci.yaml` too if changing constraints.
3757
run: |
38-
SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl)
58+
SUFFIX=$(sudo nix run nixpkgs#yq -- ".postgres_release[\"postgres${{ matrix.postgres_version }}\"]" ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/')
3959
if [[ -z $SUFFIX ]] ; then
4060
echo "Version must include non-numeric characters if built manually."
4161
exit 1
4262
fi
4363
44-
# extensions are build in nix prior to this step
45-
# so we can just use the binaries from the nix store
46-
# for postgres, extensions and wrappers
64+
- name: Set PostgreSQL version environment variable
65+
run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
66+
67+
- name: Generate common-nix.vars.pkr.hcl
68+
run: |
69+
PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
70+
PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
71+
echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
72+
# Ensure there's a newline at the end of the file
73+
echo "" >> common-nix.vars.pkr.hcl
4774
4875
- name: Build AMI stage 1
76+
env:
77+
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
4978
run: |
5079
packer init amazon-arm64-nix.pkr.hcl
5180
GIT_SHA=${{github.sha}}
52-
packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl
81+
packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
5382
5483
- name: Build AMI stage 2
84+
env:
85+
POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
5586
run: |
5687
packer init stage2-nix-psql.pkr.hcl
5788
GIT_SHA=${{github.sha}}
58-
packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
89+
POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
90+
packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
5991
6092
- name: Grab release version
6193
id: process_release_version
6294
run: |
63-
VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl)
64-
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
95+
VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
96+
echo "version=$VERSION" >> $GITHUB_OUTPUT
6597
6698
- name: Create nix flake revision tarball
6799
run: |
68100
GIT_SHA=${{github.sha}}
69-
MAJOR_VERSION=$(echo "${{ steps.process_release_version.outputs.version }}" | cut -d. -f1)
101+
MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
70102
71103
mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
72104
echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -84,17 +116,13 @@ jobs:
84116
ansible-playbook -i localhost \
85117
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
86118
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
119+
-e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
87120
manifest-playbook.yml
88121
89122
- name: Upload nix flake revision to s3 staging
90123
run: |
91124
aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
92125
93-
#Our self hosted github runner already has permissions to publish images
94-
#but they're limited to only that;
95-
#so if we want s3 access we'll need to config credentials with the below steps
96-
# (which overwrites existing perms) after the ami build
97-
98126
- name: configure aws credentials - prod
99127
uses: aws-actions/configure-aws-credentials@v4
100128
with:
@@ -107,6 +135,7 @@ jobs:
107135
ansible-playbook -i localhost \
108136
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
109137
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
138+
-e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
110139
manifest-playbook.yml
111140
112141
- name: Upload nix flake revision to s3 prod
@@ -130,12 +159,12 @@ jobs:
130159
SLACK_MESSAGE: 'Building Postgres AMI failed'
131160
SLACK_FOOTER: ''
132161

133-
- name: Cleanup resources on build cancellation
162+
- name: Cleanup resources after build
134163
if: ${{ always() }}
135164
run: |
136-
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {}
165+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
137166
138167
- name: Cleanup resources on build cancellation
139168
if: ${{ cancelled() }}
140169
run: |
141-
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {}
170+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids

‎.github/workflows/nix-build.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ jobs:
2424
arch: arm64
2525
- runner: macos-latest
2626
arch: arm64
27+
- runner: macos-13
28+
arch: amd64
2729
runs-on: ${{ matrix.runner }}
28-
30+
timeout-minutes: 180
2931
steps:
3032

3133
- name: Check out code
@@ -40,30 +42,31 @@ jobs:
4042
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
4143
aws-region: "us-east-1"
4244
output-credentials: true
45+
role-duration-seconds: 7200
4346
- name: write secret key
4447
# use python so we don't interpolate the secret into the workflow logs, in case of bugs
4548
run: |
4649
python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()"
4750
env:
4851
NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
4952
- name: Log in to Docker Hub
50-
if: matrix.runner != 'macos-latest'
53+
if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13'
5154
uses: docker/login-action@v2
5255
with:
5356
username: ${{ secrets.DOCKER_USERNAME }}
5457
password: ${{ secrets.DOCKER_PASSWORD }}
5558
- name: Build psql bundle with nix
56-
if: matrix.runner != 'macos-latest'
59+
if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13'
5760
run: docker build -t base_nix -f docker/nix/Dockerfile .
5861
- name: Run build psql bundle
59-
if: matrix.runner != 'macos-latest'
62+
if: matrix.runner != 'macos-latest' && matrix.runner != 'macos-13'
6063
run: |
6164
docker run -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
6265
-e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
6366
-e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
6467
base_nix bash -c "./workspace/docker/nix/build_nix.sh"
6568
- name: Build psql bundle on macos
66-
if: matrix.runner == 'macos-latest'
69+
if: matrix.runner == 'macos-latest' || matrix.runner == 'macos-13'
6770
run: |
6871
curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \
6972
--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \

‎.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,28 @@ permissions:
1111
id-token: write
1212

1313
jobs:
14+
prepare:
15+
runs-on: ubuntu-latest
16+
outputs:
17+
postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
18+
steps:
19+
- name: Checkout Repo
20+
uses: actions/checkout@v3
21+
22+
- uses: DeterminateSystems/nix-installer-action@main
23+
24+
- name: Set PostgreSQL versions
25+
id: set-versions
26+
run: |
27+
VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
28+
echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
29+
1430
publish-staging:
31+
needs: prepare
1532
runs-on: ubuntu-latest
33+
strategy:
34+
matrix:
35+
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
1636

1737
steps:
1838
- name: Checkout Repo
@@ -21,7 +41,8 @@ jobs:
2141
- name: Grab release version
2242
id: process_release_version
2343
run: |
24-
VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
44+
VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
45+
VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
2546
if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
2647
VERSION=${{ inputs.postgresVersion }}
2748
fi
@@ -58,6 +79,10 @@ jobs:
5879
publish-prod:
5980
runs-on: ubuntu-latest
6081
if: github.ref_name == 'develop' || contains( github.ref, 'release' )
82+
needs: prepare
83+
strategy:
84+
matrix:
85+
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
6186

6287
steps:
6388
- name: Checkout Repo
@@ -66,10 +91,8 @@ jobs:
6691
- name: Grab release version
6792
id: process_release_version
6893
run: |
69-
VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
70-
if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
71-
VERSION=${{ inputs.postgresVersion }}
72-
fi
94+
VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
95+
VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
7396
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
7497
echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT"
7598

‎.github/workflows/publish-nix-pgupgrade-scripts.yml‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ on:
77
- release/*
88
paths:
99
- '.github/workflows/publish-nix-pgupgrade-scripts.yml'
10-
- 'common-nix.vars.pkr.hcl'
1110
workflow_dispatch:
1211
inputs:
1312
postgresVersion:
@@ -18,17 +17,40 @@ permissions:
1817
id-token: write
1918

2019
jobs:
20+
prepare:
21+
runs-on: ubuntu-latest
22+
outputs:
23+
postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
24+
steps:
25+
- name: Checkout Repo
26+
uses: actions/checkout@v3
27+
28+
- uses: DeterminateSystems/nix-installer-action@main
29+
30+
- name: Set PostgreSQL versions
31+
id: set-versions
32+
run: |
33+
VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
34+
echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
35+
2136
publish-staging:
37+
needs: prepare
2238
runs-on: ubuntu-latest
39+
strategy:
40+
matrix:
41+
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
2342

2443
steps:
2544
- name: Checkout Repo
2645
uses: actions/checkout@v3
2746

47+
- uses: DeterminateSystems/nix-installer-action@main
48+
2849
- name: Grab release version
2950
id: process_release_version
3051
run: |
31-
VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
52+
VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
53+
VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
3254
if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
3355
VERSION=${{ inputs.postgresVersion }}
3456
fi
@@ -61,17 +83,24 @@ jobs:
6183
SLACK_FOOTER: ''
6284

6385
publish-prod:
86+
needs: prepare
6487
runs-on: ubuntu-latest
6588
if: github.ref_name == 'develop' || contains( github.ref, 'release' )
6689

90+
strategy:
91+
matrix:
92+
postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
93+
94+
6795
steps:
6896
- name: Checkout Repo
6997
uses: actions/checkout@v3
7098

7199
- name: Grab release version
72100
id: process_release_version
73101
run: |
74-
VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
102+
VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
103+
VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes
75104
if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
76105
VERSION=${{ inputs.postgresVersion }}
77106
fi

0 commit comments

Comments
 (0)