preserve AMI build process

Author: darora

Makefile

@@ -2,11 +2,11 @@
 UPSTREAM_NIX_GIT_SHA := $(shell git rev-parse origin/release/15.6)
 GIT_SHA := $(shell git describe --tags --always --dirty)
 
-init: amazon-arm64-nix.pkr.hcl
-	packer init amazon-arm64-nix.pkr.hcl
+init: qemu-arm64-nix.pkr.hcl
+	packer init qemu-arm64-nix.pkr.hcl
 
-output-cloudimg/packer-cloudimg: ansible amazon-arm64-nix.pkr.hcl
-	packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" amazon-arm64-nix.pkr.hcl
+output-cloudimg/packer-cloudimg: ansible qemu-arm64-nix.pkr.hcl
+	packer build -var "git_sha=$(UPSTREAM_NIX_GIT_SHA)" qemu-arm64-nix.pkr.hcl
 
 disk/focal-raw.img: output-cloudimg/packer-cloudimg
 	mkdir -p disk

ebssurrogate/scripts/qemu-bootstrap-nix.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+#
+# This script creates filesystem and setups up chrooted
+# enviroment for further processing. It also runs
+# ansible playbook and finally does system cleanup.
+#
+# Adapted from: https://github.com/jen20/packer-ubuntu-zfs
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+if [ $(dpkg --print-architecture) = "amd64" ]; 
+then 
+	ARCH="amd64";
+else
+        ARCH="arm64";
+fi
+
+function waitfor_boot_finished {
+	export DEBIAN_FRONTEND=noninteractive
+
+	echo "args: ${ARGS}"
+	# Wait for cloudinit on the surrogate to complete before making progress
+	while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do
+	    echo 'Waiting for cloud-init...'
+	    sleep 1
+	done
+}
+
+function install_packages {
+	apt-get update && sudo apt-get install software-properties-common e2fsprogs -y
+	add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y
+	ansible-galaxy collection install community.general
+}
+
+function execute_playbook {
+
+tee /etc/ansible/ansible.cfg <<EOF
+[defaults]
+callbacks_enabled = timer, profile_tasks, profile_roles
+EOF
+	# Run Ansible playbook
+	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp 
+	export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
+	ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' # $ARGS - I think this is being not passed in correctly
+}
+
+function setup_postgesql_env {
+	    # Create the directory if it doesn't exist
+    sudo mkdir -p /etc/environment.d
+
+    # Define the contents of the PostgreSQL environment file
+    cat <<EOF | sudo tee /etc/environment.d/postgresql.env >/dev/null
+LOCALE_ARCHIVE=/usr/lib/locale/locale-archive
+LANG="en_US.UTF-8"
+LANGUAGE="en_US.UTF-8"
+LC_ALL="en_US.UTF-8"
+LC_CTYPE="en_US.UTF-8"
+EOF
+}
+
+function setup_locale {
+cat << EOF >> /etc/locale.gen
+en_US.UTF-8 UTF-8
+EOF
+
+cat << EOF > /etc/default/locale
+LANG="C.UTF-8"
+LC_CTYPE="C.UTF-8"
+EOF
+	locale-gen en_US.UTF-8
+}
+
+waitfor_boot_finished
+install_packages
+setup_postgesql_env
+setup_locale
+execute_playbook
+
+# stage 2 things
+function install_nix() {
+    sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \
+    --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \
+    --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root
+    . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
+
+}
+
+function execute_stage2_playbook {
+    sudo tee /etc/ansible/ansible.cfg <<EOF
+[defaults]
+callbacks_enabled = timer, profile_tasks, profile_roles
+EOF
+    # sed -i 's/- hosts: all/- hosts: localhost/' /tmp/ansible-playbook/ansible/playbook.yml
+    # Run Ansible playbook
+    export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp
+    ansible-playbook ./ansible/playbook.yml \
+        --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false}' \
+        --extra-vars "git_commit_sha=${GIT_SHA}"
+}
+
+function clean_system {
+    # Copy cleanup scripts
+    chmod +x /tmp/ansible-playbook/scripts/90-cleanup-qemu.sh
+    /tmp/ansible-playbook/scripts/90-cleanup-qemu.sh
+
+    # # Cleanup logs
+    rm -rf /var/log/*
+    # # https://github.com/fail2ban/fail2ban/issues/1593
+    touch /var/log/auth.log
+
+    touch /var/log/pgbouncer.log
+    chown pgbouncer:postgres /var/log/pgbouncer.log
+
+    # # Setup postgresql logs
+    mkdir -p /var/log/postgresql
+    chown postgres:postgres /var/log/postgresql
+    # # Setup wal-g logs
+    mkdir /var/log/wal-g
+    touch /var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log}
+
+    # #Creatre Sysstat directory for SAR
+    mkdir /var/log/sysstat
+
+    chown -R postgres:postgres /var/log/wal-g
+    chmod -R 0300 /var/log/wal-g
+
+    # # audit logs directory for apparmor
+    mkdir /var/log/audit
+
+    # # unwanted files
+    rm -rf /var/lib/apt/lists/*
+    rm -rf /root/.cache
+    rm -rf /root/.vpython*
+    rm -rf /root/go
+    rm -rf /mnt/usr/share/doc
+}
+
+install_nix
+execute_stage2_playbook
+cloud-init clean --logs

qemu-arm64-nix.pkr.hcl

@@ -0,0 +1,205 @@
+variable "ami" {
+  type    = string
+  default = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-*"
+}
+
+variable "profile" {
+  type    = string
+  default = "${env("AWS_PROFILE")}"
+}
+
+variable "ami_name" {
+  type    = string
+  default = "supabase-postgres"
+}
+
+variable "ami_regions" {
+  type    = list(string)
+  default = ["ap-southeast-2"]
+}
+
+variable "ansible_arguments" {
+  type    = string
+  default = "--skip-tags install-postgrest,install-pgbouncer,install-supabase-internal"
+}
+
+variable "aws_access_key" {
+  type    = string
+  default = ""
+}
+
+variable "aws_secret_key" {
+  type    = string
+  default = ""
+}
+
+variable "environment" {
+  type    = string
+  default = "prod"
+}
+
+variable "git_sha" {
+  type    = string
+}
+
+# variable "region" {
+#   type    = string
+# }
+
+variable "build-vol" {
+  type    = string
+  default = "xvdc"
+}
+
+# ccache docker image details
+variable "docker_user" {
+  type    = string
+  default = ""
+}
+
+variable "docker_passwd" {
+  type    = string
+  default = ""
+}
+
+variable "docker_image" {
+  type    = string
+  default = ""
+}
+
+variable "docker_image_tag" {
+  type    = string
+  default = "latest"
+}
+
+locals {
+  creator = "packer"
+}
+
+variable "postgres-version" {
+  type = string
+  default = ""
+}
+
+variable "git-head-version" {
+  type = string
+  default = "unknown"
+}
+
+variable "packer-execution-id" {
+  type = string
+  default = "unknown"
+}
+
+variable "force-deregister" {
+  type    = bool
+  default = false
+}
+
+packer {
+  required_plugins {
+    amazon = {
+      source  = "github.com/hashicorp/amazon"
+      version = "~> 1"
+    }
+    qemu = {
+      version = "~> 1.0"
+      source  = "github.com/hashicorp/qemu"
+    }
+  }
+}
+
+source "null" "dependencies" {
+  communicator = "none"
+}
+
+build {
+  name    = "cloudimg.deps"
+  sources = ["source.null.dependencies"]
+
+  provisioner "shell-local" {
+    inline = [
+      "cp /usr/share/AAVMF/AAVMF_VARS.fd AAVMF_VARS.fd",
+      "cloud-localds seeds-cloudimg.iso user-data-cloudimg meta-data"
+    ]
+    inline_shebang = "/bin/bash -e"
+  }
+}
+
+source "qemu" "cloudimg" {
+  boot_wait      = "2s"
+  cpus           = 12
+  disk_image     = true
+  disk_size      = "15G"
+  format         = "qcow2"
+  # TODO (darora): disable backing image for qcow2
+  headless       = true
+  http_directory = "http"
+  iso_checksum   = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS"
+  iso_url        = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img"
+  memory         = 20000
+  qemu_binary    = "qemu-system-aarch64"
+  qemu_img_args {
+    create = ["-F", "qcow2"]
+  }
+  qemuargs = [
+    ["-machine", "virt"],
+    ["-cpu", "host"],
+    ["-device", "virtio-gpu-pci"],
+    ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"],
+    ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"],
+    ["-drive", "file=output-cloudimg/packer-cloudimg,format=qcow2"],
+    ["-drive", "file=seeds-cloudimg.iso,format=raw"],
+    ["--enable-kvm"]
+  ]
+  shutdown_command       = "sudo -S shutdown -P now"
+  ssh_handshake_attempts = 500
+  ssh_password           = "ubuntu"
+  ssh_timeout            = "1h"
+  ssh_username           = "ubuntu"
+  ssh_wait_timeout       = "1h"
+  use_backing_file       = true
+  accelerator = "kvm"
+}
+
+build {
+  name    = "cloudimg.image"
+  sources = ["source.qemu.cloudimg"]
+
+  # Copy ansible playbook
+  provisioner "shell" {
+    inline = ["mkdir /tmp/ansible-playbook"]
+  }
+
+  provisioner "file" {
+    source = "ansible"
+    destination = "/tmp/ansible-playbook"
+  }
+
+  provisioner "file" {
+    source = "scripts"
+    destination = "/tmp/ansible-playbook"
+  }
+
+  provisioner "file" {
+    source = "migrations"
+    destination = "/tmp"
+  }
+
+  provisioner "file" {
+    source = "ebssurrogate/files/unit-tests"
+    destination = "/tmp"
+  }
+
+  provisioner "shell" {
+    environment_vars = [
+      "POSTGRES_SUPABASE_VERSION=${var.postgres-version}",
+      "GIT_SHA=${var.git_sha}"
+    ]
+    use_env_var_file = true
+    script = "ebssurrogate/scripts/qemu-bootstrap-nix.sh"
+    execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && cd /tmp/ansible-playbook && {{.Path}}'"
+    start_retry_timeout = "5m"
+    skip_clean = true
+  }
+}