Clean up build to a functional state

darora · darora · commit 2ae639cea790 · 2025-02-05T11:30:30.000+08:00
diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml
@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
-      
+
       - uses: DeterminateSystems/nix-installer-action@main
 
       - name: Set PostgreSQL versions - only builds pg15 atm
@@ -35,15 +35,7 @@ jobs:
     strategy:
       matrix:
         postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
-        # include:
-        #   # TODO (darora): change to use non-self-hosted runners, since they now allow nested virtualization
-        #   - runner: arm-runner
-        #     arch: arm64
-        #     ubuntu_release: focal
-        #     ubuntu_version: 20.04
-        #     mcpu: neoverse-n1
-    runs-on:
-      group: pg-arm-large
+    runs-on: arm-native-runner
     timeout-minutes: 150
     permissions:
       contents: write
@@ -56,7 +48,6 @@ jobs:
 
       - uses: DeterminateSystems/nix-installer-action@main
 
-      # TODO (darora): might need to temporarily disable this
       - name: Run checks if triggered manually
         if: ${{ github.event_name == 'workflow_dispatch' }}
         run: |
@@ -66,37 +57,55 @@ jobs:
             exit 1
           fi
 
+      - name: enable KVM support
+        run: |
+          sudo chown runner /dev/kvm
+          sudo chmod 666 /dev/kvm
+
+      # - name: Install dependencies
+      #   run: |
+      #     sudo apt-get update
+      #     sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils
+      #     cd && curl -L "https://releases.hashicorp.com/packer/1.7.8/packer_1.7.8_linux_$(dpkg --print-architecture).zip" -o packer.zip && unzip packer.zip  && rm -f packer.zip  && sudo mv packer /usr/local/bin/
+
       - name: Set PostgreSQL version environment variable
         run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
 
       - name: Generate common-nix.vars.pkr.hcl
         run: |
-          PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq
+          chmod +x yq
+          PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
           PG_VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl
-          echo 'postgresql_major = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl
+          echo 'postgres-major-version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl
+          # echo 'postgresql_major = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Get binaries
+
+      - name: Install dependencies
         run: |
-          hash
+          sudo apt-get update
+          sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils
+          sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils
 
       - name: Build QEMU artifact
         run: |
           make init
           GIT_SHA=${{github.sha}}
-          # TODO (darora): possibly need to install qemu deps here
+          export PACKER_LOG=1
           packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl
-          # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
 
-      # TODO (darora): publishing things is temporarily disabled
-      # - name: Grab release version
-      #   id: process_release_version
-      #   run: |
-      #     VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-      #     echo "version=$VERSION" >> $GITHUB_OUTPUT
+      - name: Install dependencies
+        run: |
+          ls -lh output-cloudimg/
+
+      - name: Grab release version
+        id: process_release_version
+        run: |
+          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
 
       # - name: Create nix flake revision tarball
       #   run: |
@@ -107,11 +116,27 @@ jobs:
       #     echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
       #     tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" .
 
-      # - name: configure aws credentials - staging
-      #   uses: aws-actions/configure-aws-credentials@v4
-      #   with:
-      #     role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
-      #     aws-region: "us-east-1"
+      - name: configure aws credentials - staging
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Login to ECR
+        uses: docker/login-action@v2
+        with:
+          registry: public.ecr.aws
+
+      - id: build
+        uses: docker/build-push-action@v5
+        with:
+          file: Dockerfile-kubernetes
+          push: true
+          tags: public.ecr.aws/w9p6e7k7/supabase/postgres-v3:${{ steps.process_release_version.outputs.version }}
+          platforms: linux/arm64
+          context: .
 
       # - name: Upload software manifest to s3 staging
       #   run: |
diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes
@@ -1,8 +1,8 @@
 FROM alpine:3.21
 
-ADD output-cloudimg/packer-cloudimg /disk/focal.qcow2
+ADD ./output-cloudimg/packer-cloudimg /disk/focal.qcow2
 
-RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh nftables cloud-utils-localds aavmf
+RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client nftables cloud-utils-localds aavmf
 # dev stuff
 # RUN apk add --no-cache iproute2
 
diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl
@@ -21,6 +21,11 @@ variable "postgres-version" {
   default = ""
 }
 
+variable "postgres-major-version" {
+  type = string
+  default = ""
+}
+
 variable "git-head-version" {
   type = string
   default = "unknown"
@@ -63,18 +68,18 @@ build {
 
 source "qemu" "cloudimg" {
   boot_wait      = "2s"
-  cpus           = 12
+  cpus           = 8
   disk_image     = true
   disk_size      = "15G"
   format         = "qcow2"
   headless       = true
   http_directory = "http"
   iso_checksum   = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS"
   iso_url        = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img"
-  memory         = 20000
+  memory         = 40000
   qemu_binary    = "qemu-system-aarch64"
   qemuargs = [
-    ["-machine", "virt"],
+    ["-machine", "virt,gic-version=3"],
     ["-cpu", "host"],
     ["-device", "virtio-gpu-pci"],
     ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"],
@@ -124,9 +129,9 @@ build {
 
   provisioner "shell" {
     environment_vars = [
-      "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}"
+      "POSTGRES_MAJOR_VERSION=${var.postgres-major-version}",
       "POSTGRES_SUPABASE_VERSION=${var.postgres-version}",
-      "GIT_SHA=${var.git_sha}",
+      "GIT_SHA=${var.git_sha}"
     ]
     use_env_var_file = true
     script = "ebssurrogate/scripts/qemu-bootstrap-nix.sh"
