Skip to content

Commit c15b960

Browse files
samrosesamrose
committed
feat: clean up and attach stages in packer
1 parent 5c46f97 commit c15b960

File tree

77 files changed

+4396
-9
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+4396
-9
lines changed

‎.github/workflows/ami-release-nix.yml

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
name: Release AMI
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
paths:
8+
- '.github/workflows/ami-release-nix.yml'
9+
- 'common-nix.vars.pkr.hcl'
10+
workflow_dispatch:
11+
12+
jobs:
13+
build:
14+
strategy:
15+
matrix:
16+
include:
17+
- runner: arm-runner
18+
arch: arm64
19+
ubuntu_release: focal
20+
ubuntu_version: 20.04
21+
mcpu: neoverse-n1
22+
runs-on: ${{ matrix.runner }}
23+
timeout-minutes: 150
24+
permissions:
25+
contents: write
26+
packages: write
27+
id-token: write
28+
29+
steps:
30+
- name: Checkout Repo
31+
uses: actions/checkout@v3
32+
33+
- name: Run checks if triggered manually
34+
if: ${{ github.event_name == 'workflow_dispatch' }}
35+
# Update `ci.yaml` too if changing constraints.
36+
run: |
37+
SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl)
38+
if [[ -z $SUFFIX ]] ; then
39+
echo "Version must include non-numeric characters if built manually."
40+
exit 1
41+
fi
42+
43+
# - id: args
44+
# uses: mikefarah/yq@master
45+
# with:
46+
# cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
47+
# - run: docker context create builders
48+
# - uses: docker/setup-buildx-action@v3
49+
# with:
50+
# endpoint: builders
51+
# - uses: docker/build-push-action@v5
52+
# with:
53+
# build-args: |
54+
# ${{ steps.args.outputs.result }}
55+
# target: extensions
56+
# tags: supabase/postgres:extensions
57+
# platforms: linux/${{ matrix.arch }}
58+
# outputs: type=tar,dest=/tmp/extensions.tar
59+
# cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
60+
# # No need to export extensions cache because latest depends on it
61+
62+
# - name: Extract built packages
63+
# run: |
64+
# mkdir -p ansible/files/extensions
65+
# tar xvf /tmp/extensions.tar -C ansible/files/extensions --strip-components 1
66+
# TODO remove this block as extensions are build in nix prior to this step
67+
68+
- id: version
69+
run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT"
70+
- name: Build Postgres deb
71+
uses: docker/build-push-action@v5
72+
with:
73+
file: docker/Dockerfile
74+
target: pg-deb
75+
build-args: |
76+
ubuntu_release=${{ matrix.ubuntu_release }}
77+
ubuntu_release_no=${{ matrix.ubuntu_version }}
78+
postgresql_major=${{ steps.version.outputs.postgresql_major }}
79+
postgresql_release=${{ steps.version.outputs.postgresql_release }}
80+
CPPFLAGS=-mcpu=${{ matrix.mcpu }}
81+
tags: supabase/postgres:deb
82+
platforms: linux/${{ matrix.arch }}
83+
outputs: type=tar,dest=/tmp/pg-deb.tar
84+
cache-from: type=gha,scope=${{ github.ref_name }}-deb
85+
cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb
86+
# - name: Extract Postgres deb
87+
# run: |
88+
# mkdir -p ansible/files/postgres
89+
# tar xvf /tmp/pg-deb.tar -C ansible/files/postgres --strip-components 1
90+
#TODO remove this block as deb is build in nix prior to this step
91+
92+
- name: Build AMI stage 1
93+
run: |
94+
packer init amazon-arm64-nix.pkr.hcl
95+
GIT_SHA=${{github.sha}}
96+
packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl
97+
98+
- name: Build AMI stage 1
99+
run: |
100+
packer init amazon-arm64-nix.pkr.hcl
101+
GIT_SHA=${{github.sha}}
102+
packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl
103+
104+
- name: Grab release version
105+
id: process_release_version
106+
run: |
107+
VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl)
108+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
109+
110+
- name: configure aws credentials - staging
111+
uses: aws-actions/configure-aws-credentials@v1
112+
with:
113+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
114+
aws-region: "us-east-1"
115+
116+
- name: Upload software manifest to s3 staging
117+
run: |
118+
cd ansible
119+
ansible-playbook -i localhost \
120+
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
121+
-e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
122+
manifest-playbook.yml
123+
124+
# - name: Upload pg binaries to s3 staging
125+
# run: |
126+
# aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
127+
# #TODO look to see if this only pg binaries and if so, remove this as it is covered by nix build
128+
# TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build
129+
- name: configure aws credentials - prod
130+
uses: aws-actions/configure-aws-credentials@v1
131+
with:
132+
role-to-assume: ${{ secrets.PROD_AWS_ROLE }}
133+
aws-region: "us-east-1"
134+
135+
- name: Upload software manifest to s3 prod
136+
run: |
137+
cd ansible
138+
ansible-playbook -i localhost \
139+
-e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
140+
-e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
141+
manifest-playbook.yml
142+
143+
# - name: Upload pg binaries to s3 prod
144+
# run: |
145+
# aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
146+
#TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build
147+
148+
149+
- name: Create release
150+
uses: softprops/action-gh-release@v1
151+
with:
152+
name: ${{ steps.process_release_version.outputs.version }}
153+
tag_name: ${{ steps.process_release_version.outputs.version }}
154+
target_commitish: ${{github.sha}}
155+
156+
- name: Slack Notification on Failure
157+
if: ${{ failure() }}
158+
uses: rtCamp/action-slack-notify@v2
159+
env:
160+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
161+
SLACK_USERNAME: 'gha-failures-notifier'
162+
SLACK_COLOR: 'danger'
163+
SLACK_MESSAGE: 'Building Postgres AMI failed'
164+
SLACK_FOOTER: ''
165+
166+
- name: Cleanup resources on build cancellation
167+
if: ${{ cancelled() }}
168+
run: |
169+
aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {}

‎.github/workflows/text-nix.yml

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
name: Test Database
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
build:
12+
strategy:
13+
matrix:
14+
include:
15+
- runner: [self-hosted, X64]
16+
arch: amd64
17+
- runner: arm-runner
18+
arch: arm64
19+
runs-on: ${{ matrix.runner }}
20+
timeout-minutes: 180
21+
env:
22+
POSTGRES_PORT: 5478
23+
POSTGRES_PASSWORD: password
24+
steps:
25+
- uses: actions/checkout@v3
26+
- id: args
27+
uses: mikefarah/yq@master
28+
with:
29+
cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
30+
31+
- run: docker context create builders
32+
- uses: docker/setup-buildx-action@v3
33+
with:
34+
endpoint: builders
35+
- uses: docker/build-push-action@v5
36+
with:
37+
load: true
38+
context: .
39+
target: production
40+
build-args: |
41+
${{ steps.args.outputs.result }}
42+
tags: samrose/nix-experimental-postgresql-15-aarch64-linux:latest
43+
cache-from: |
44+
type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
45+
type=gha,scope=${{ github.base_ref }}-latest-${{ matrix.arch }}
46+
cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
47+
48+
- name: Start Postgres
49+
run: |
50+
docker run --rm --pull=never \
51+
-e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \
52+
-p ${{ env.POSTGRES_PORT }}:5432 \
53+
--name supabase_postgres \
54+
-d supabase/postgres:latest
55+
56+
- name: Install psql
57+
run: |
58+
sudo apt update
59+
sudo apt install -y --no-install-recommends postgresql-client
60+
61+
- name: Install pg_prove
62+
run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP
63+
env:
64+
SHELL: /bin/bash
65+
66+
- name: Wait for healthy database
67+
run: |
68+
count=0
69+
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do
70+
exit=$?
71+
count=$((count + 1))
72+
if [ $count -ge "$retries" ]; then
73+
echo "Retry $count/$retries exited $exit, no more retries left."
74+
docker stop -t 2 "$container"
75+
return $exit
76+
fi
77+
sleep 1;
78+
done;
79+
echo "$container container is healthy"
80+
env:
81+
retries: 20
82+
container: supabase_postgres
83+
84+
- name: Run tests
85+
run: pg_prove migrations/tests/test.sql
86+
env:
87+
PGHOST: localhost
88+
PGPORT: ${{ env.POSTGRES_PORT }}
89+
PGDATABASE: postgres
90+
PGUSER: supabase_admin
91+
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
92+
93+
- name: Check migrations are idempotent
94+
run: |
95+
for sql in ./migrations/db/migrations/*.sql; do
96+
echo "$0: running $sql"
97+
psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql"
98+
done
99+
env:
100+
PGHOST: localhost
101+
PGPORT: ${{ env.POSTGRES_PORT }}
102+
PGDATABASE: postgres
103+
PGUSER: supabase_admin
104+
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
105+
106+
schema:
107+
runs-on: ubuntu-latest
108+
steps:
109+
- uses: actions/checkout@v3
110+
- name: verify schema.sql is committed
111+
run: |
112+
docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit
113+
if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then
114+
echo "Detected uncommitted changes after build. See status below:"
115+
git diff
116+
exit 1
117+
fi

0 commit comments

Comments
 (0)