feat: move stage 1 to nightly build

samrose · samrose · commit c5b932e6bd40 · 2025-11-21T13:38:16.000-05:00
diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml
@@ -61,14 +61,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml
@@ -89,15 +89,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
diff --git a/.github/workflows/base-image-nightly.yml b/.github/workflows/base-image-nightly.yml
@@ -0,0 +1,85 @@
+name: Build Base Image Nightly
+
+on:
+  schedule:
+    - cron: '0 2 * * *'  # 2 AM UTC daily
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to build from'
+        required: false
+        default: 'develop'
+        type: string
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build-base-image:
+    runs-on: blacksmith-4vcpu-ubuntu-2404-arm
+    timeout-minutes: 150
+
+    steps:
+      - name: Checkout Repo
+        uses: supabase/postgres/.github/actions/shared-checkout@HEAD
+        with:
+          ref: ${{ github.event.inputs.branch || 'develop' }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+          output-credentials: true
+          role-duration-seconds: 7200
+
+      - name: Install nix
+        uses: cachix/install-nix-action@v27
+        with:
+          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
+          extra_nix_config: |
+            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
+            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+
+      - name: Set execution ID and timestamp
+        run: |
+          echo "EXECUTION_ID=${{ github.run_id }}-base-nightly" >> $GITHUB_ENV
+          echo "BUILD_TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+
+      - name: Build base stage 1 AMI
+        env:
+          AWS_MAX_ATTEMPTS: 10
+          AWS_RETRY_MODE: adaptive
+        run: |
+          GIT_SHA=${{ github.sha }}
+          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
+          nix run github:supabase/postgres/${GIT_SHA}#packer -- build \
+            -var "git-head-version=${GIT_SHA}" \
+            -var "packer-execution-id=${EXECUTION_ID}" \
+            -var-file="development-arm.vars.pkr.hcl" \
+            -var "base-image-nightly=true" \
+            -var "build-timestamp=${BUILD_TIMESTAMP}" \
+            -var "region=us-east-1" \
+            -var 'ami_regions=["us-east-1","ap-southeast-1"]' \
+            amazon-arm64-nix.pkr.hcl
+
+      - name: Slack Notification on Failure
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
+          SLACK_USERNAME: 'gha-failures-notifier'
+          SLACK_COLOR: 'danger'
+          SLACK_MESSAGE: 'Building base image nightly failed'
+          SLACK_FOOTER: ''
+
+      - name: Cleanup resources after build
+        if: ${{ always() }}
+        run: |
+          aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
+
+      - name: Cleanup resources on build cancellation
+        if: ${{ cancelled() }}
+        run: |
+          aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml
@@ -108,15 +108,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          AWS_MAX_ATTEMPTS: 10
-          AWS_RETRY_MODE: adaptive
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           AWS_MAX_ATTEMPTS: 10
diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl
@@ -92,6 +92,18 @@ variable "force-deregister" {
   default = false
 }
 
+variable "base-image-nightly" {
+  type    = bool
+  default = false
+  description = "Build as version-agnostic base image for nightly"
+}
+
+variable "build-timestamp" {
+  type    = string
+  default = ""
+  description = "Timestamp for nightly builds"
+}
+
 packer {
   required_plugins {
     amazon = {
@@ -106,7 +118,7 @@ source "amazon-ebssurrogate" "source" {
   profile = "${var.profile}"
   #access_key    = "${var.aws_access_key}"
   #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}"
-  ami_name = "${var.ami_name}-${var.postgres-version}-stage-1"
+  ami_name = var.base-image-nightly ? "${var.ami_name}-base-stage-1-nightly" : "${var.ami_name}-${var.postgres-version}-stage-1"
   ami_virtualization_type = "hvm"
   ami_architecture = "arm64"
   ami_regions   = "${var.ami_regions}"
@@ -170,8 +182,10 @@ source "amazon-ebssurrogate" "source" {
   tags = {
     creator = "packer"
     appType = "postgres"
-    postgresVersion = "${var.postgres-version}-stage1"
+    postgresVersion = var.base-image-nightly ? "base-nightly" : "${var.postgres-version}-stage1"
     sourceSha = "${var.git-head-version}"
+    buildTimestamp = var.base-image-nightly ? "${var.build-timestamp}" : ""
+    buildType = var.base-image-nightly ? "nightly" : "release"
   }
 
   communicator = "ssh"
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
@@ -46,13 +46,13 @@
         - install-gotrue
         - install-supabase-internal
       when: debpkg_mode or nixpkg_mode
-  
+
     - name: Install PostgREST
       import_tasks: tasks/setup-postgrest.yml
       tags:
         - install-postgrest
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or stage2_nix
 
     - name: Install Envoy
       import_tasks: tasks/setup-envoy.yml
diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml
@@ -11,7 +11,7 @@
 - name: PostgREST - add Postgres PPA main
   ansible.builtin.apt_repository:
     filename: 'postgresql-pgdg'
-    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg {{ postgresql_major }}"
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main"
     state: 'present'
 
 - name: PostgREST - install system dependencies
@@ -36,7 +36,7 @@
 
 - name: PostgREST - remove Postgres PPA
   ansible.builtin.apt_repository:
-    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg {{ postgresql_major }}"
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ noble-pgdg main"
     state: 'absent'
 
 - name: postgis - ensure dependencies do not get autoremoved
diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh
@@ -303,11 +303,10 @@ tee /etc/ansible/ansible.cfg <<EOF
 callbacks_enabled = timer, profile_tasks, profile_roles
 EOF
 	# Run Ansible playbook
-	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp 
+	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
 	export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
 	ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \
 	--extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false} ' \
-	--extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \
 	$ARGS
 }
 
diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl
@@ -66,12 +66,12 @@ source "amazon-ebs" "ubuntu" {
   region        = "${var.region}"
   source_ami_filter {
     filters = {
-      name   = "${var.ami_name}-${var.postgres-version}-stage-1"
+      name   = "${var.ami_name}-base-stage-1-nightly"
       root-device-type    = "ebs"
       virtualization-type = "hvm"
     }
     most_recent = true
-    owners      = ["amazon", "self"]
+    owners      = ["self"]
   }
 
   communicator = "ssh"

Original file line number	Diff line number	Diff line change
`@@ -66,12 +66,12 @@ source "amazon-ebs" "ubuntu" {`
`66`	`66`	`region = "${var.region}"`
`67`	`67`	`source_ami_filter {`
`68`	`68`	`filters = {`
`69`		`- name = "${var.ami_name}-${var.postgres-version}-stage-1"`
	`69`	`+ name = "${var.ami_name}-base-stage-1-nightly"`
`70`	`70`	`root-device-type = "ebs"`
`71`	`71`	`virtualization-type = "hvm"`
`72`	`72`	`}`
`73`	`73`	`most_recent = true`
`74`		`- owners = ["amazon", "self"]`
	`74`	`+ owners = ["self"]`
`75`	`75`	`}`
`76`	`76`
`77`	`77`	`communicator = "ssh"`