chore: move out of tmp

pcnc · pcnc · commit 7610c6abf5dc · 2024-09-11T18:44:22.000+03:00
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/check.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/check.sh
@@ -1,16 +1,19 @@
 #! /usr/bin/env bash
 ## This script provides a method to check the status of the database upgrade
-## process, which is updated in /tmp/pg-upgrade-status by initiate.sh
+## process, which is updated in /root/pg_upgrade/status by initiate.sh
 ## This runs on the old (source) instance.
 
 set -euo pipefail
 
-STATUS_FILE="/tmp/pg-upgrade-status"
+SCRIPT_DIR=$(dirname -- "$0")
+# shellcheck disable=SC1091
+source "$SCRIPT_DIR/common.sh"
+
+STATUS_FILE="/root/pg_upgrade/status"
 
 if [ -f "${STATUS_FILE}" ]; then
     STATUS=$(cat "${STATUS_FILE}")
     echo -n "${STATUS}"
 else
     echo -n "unknown"
 fi
-
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh
@@ -5,6 +5,8 @@
 REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo"
 REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials"
 
+UPGRADE_STATUS_FILE="/root/pg_upgrade/status"
+
 REPORTING_ANON_KEY=""
 if [ -f "$REPORTING_CREDENTIALS_FILE" ]; then
     REPORTING_ANON_KEY=$(cat "$REPORTING_CREDENTIALS_FILE")
@@ -24,6 +26,11 @@ function ship_logs {
         return 0
     fi
 
+    if [ ! -f "$UPGRADE_STATUS_FILE" ]; then
+        echo "No upgrade status file found. Skipping log upload."
+        return 0
+    fi
+
     if [ ! -f "$LOG_FILE" ]; then
         echo "No log file found. Skipping log upload."
         return 0
@@ -36,32 +43,47 @@ function ship_logs {
 
     HOSTNAME=$(hostname)
     DERIVED_REF="${HOSTNAME##*-}"
+    STATUS=$(cat "$UPGRADE_STATUS_FILE")
 
-    printf -v BODY '{ "ref": "%s", "step": "%s", "content": %s }' "$DERIVED_REF" "completion" "$(cat "$LOG_FILE" | jq -Rs '.')"
+    printf -v BODY '{ "ref": "%s", "step": "%s", "status": "%", "content": %s }' "$DERIVED_REF" "completion" "$STATUS" "$(cat "$LOG_FILE" | jq -Rs '.')"
     curl -sf -X POST "https://$REPORTING_PROJECT_REF.supabase.co/rest/v1/error_logs" \
-         -H "apikey: ${REPORTING_ANON_KEY}" \
-         -H 'Content-type: application/json' \
-         -d "$BODY"
+        -H "apikey: ${REPORTING_ANON_KEY}" \
+        -H 'Content-type: application/json' \
+        -d "$BODY"
 }
 
+ function create_pgupgrade_files_dir {
+     if [ ! -d /root/pg_upgrade ]; then
+         mkdir -p /root/pg_upgrade
+         chown postgres:postgres /root/pg_upgrade
+     fi
+ }
+
+ function report_upgrade_status {
+     create_pgupgrade_files_dir
+
+     STATUS=$1
+     echo "$STATUS" > "$UPGRADE_STATUS_FILE"
+ }
+ 
 function retry {
-  local retries=$1
-  shift
-
-  local count=0
-  until "$@"; do
-    exit=$?
-    wait=$((2 ** (count + 1)))
-    count=$((count + 1))
-    if [ $count -lt "$retries" ]; then
-        echo "Command $* exited with code $exit, retrying..."
-        sleep $wait
-    else
-        echo "Command $* exited with code $exit, no more retries left."
-        return $exit
-    fi
-  done
-  return 0
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** (count + 1)))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            echo "Command $* exited with code $exit, retrying..."
+            sleep $wait
+        else
+            echo "Command $* exited with code $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
 }
 
 CI_stop_postgres() {
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh
@@ -18,7 +18,7 @@ function cleanup {
     UPGRADE_STATUS=${1:-"failed"}
     EXIT_CODE=${?:-0}
 
-    echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status
+    report_upgrade_status "$UPGRADE_STATUS"
 
     ship_logs "$LOG_FILE" || true
 
@@ -81,12 +81,12 @@ EOF
 }
 
 function complete_pg_upgrade {
-    if [ -f /tmp/pg-upgrade-status ]; then
+    if [ -f "$UPGRADE_STATUS_FILE" ]; then
         echo "Upgrade job already started. Bailing."
         exit 0
     fi
 
-    echo "running" > /tmp/pg-upgrade-status
+    report_upgrade_status "running"
 
     echo "1. Mounting data disk"
     if [ -z "$IS_CI" ]; then
@@ -169,7 +169,7 @@ function apply_auth_scheme_updates {
 }
 
 function start_vacuum_analyze {
-    echo "complete" > /tmp/pg-upgrade-status
+    report_upgrade_status "complete"
     if ! command -v nix &> /dev/null; then
         su -c 'vacuumdb --all --analyze-in-stages' -s "$SHELL" postgres
     else
@@ -184,12 +184,14 @@ echo "C.UTF-8 UTF-8" > /etc/locale.gen
 echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
 locale-gen
 
+create_pgupgrade_files_dir
+
 if [ -z "$IS_CI" ]; then
     complete_pg_upgrade >> $LOG_FILE 2>&1 &
 else 
     CI_stop_postgres || true
 
-    rm -f /tmp/pg-upgrade-status
+    rm -f "$UPGRADE_STATUS_FILE"
     mv /data_migration /data
 
     rm -rf /var/lib/postgresql/data
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh
@@ -3,7 +3,7 @@
 ## This script is run on the old (source) instance, mounting the data disk
 ## of the newly launched instance, disabling extensions containing regtypes,
 ## and running pg_upgrade.
-## It reports the current status of the upgrade process to /tmp/pg-upgrade-status,
+## It reports the current status of the upgrade process to /root/pg_upgrade/status,
 ## which can then be subsequently checked through check.sh.
 
 # Extensions to disable before running pg_upgrade.
@@ -37,7 +37,11 @@ PGVERSION=$1
 MOUNT_POINT="/data_migration"
 LOG_FILE="/var/log/pg-upgrade-initiate.log"
 
-POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql"
+create_pgupgrade_files_dir
+PG_UPGRADE_DIR="/root/pg_upgrade/$(date +%s)"
+
+POST_UPGRADE_EXTENSION_SCRIPT="${PG_UPGRADE_DIR}/pg_upgrade_extensions.sql"
+ 
 OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;")
 
 SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;")
@@ -48,7 +52,7 @@ POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf"
 PGBINOLD="/usr/lib/postgresql/bin"
 PGLIBOLD="/usr/lib/postgresql/lib"
 
-PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION"
+PG_UPGRADE_BIN_DIR="${PG_UPGRADE_DIR}/${PGVERSION}"
 NIX_INSTALLER_PATH="/tmp/persistent/nix-installer"
 NIX_INSTALLER_PACKAGE_PATH="$NIX_INSTALLER_PATH.tar.gz"
 
@@ -131,7 +135,7 @@ cleanup() {
         echo "Unmounting data disk from ${MOUNT_POINT}"
         umount $MOUNT_POINT
     fi
-    echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status
+    report_upgrade_status "$UPGRADE_STATUS"
 
     if [ -z "$IS_CI" ]; then
         exit "$EXIT_CODE"
@@ -227,6 +231,30 @@ function patch_wrappers {
     fi
 }
 
+function mount_data_migration_disk {
+    if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then
+        # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere
+        # excluding nvme0 since it is the root disk
+        echo "5. Determining block device to mount"
+        BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }')
+        echo "Block device found: $BLOCK_DEVICE"
+
+        mkdir -p "$MOUNT_POINT"
+        echo "6. Mounting block device"
+
+        sleep 5
+        e2fsck -pf "$BLOCK_DEVICE"
+
+        sleep 1
+        mount "$BLOCK_DEVICE" "$MOUNT_POINT"
+
+        sleep 1
+        resize2fs "$BLOCK_DEVICE"
+    else 
+        mkdir -p "$MOUNT_POINT"
+    fi
+}
+
 function initiate_upgrade {
     mkdir -p "$MOUNT_POINT"
     SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/")
@@ -266,13 +294,17 @@ function initiate_upgrade {
         mkdir -p "$PG_UPGRADE_BIN_DIR"
         mkdir -p /tmp/persistent/
         echo "a7189a68ed4ea78c1e73991b5f271043636cf074" > "$PG_UPGRADE_BIN_DIR/nix_flake_version"
-        tar -czf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" .
+        tar -czf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "$PG_UPGRADE_DIR" .
         rm -rf /tmp/pg_upgrade_bin/
     fi
     
     echo "1. Extracting pg_upgrade binaries"
-    mkdir -p "/tmp/pg_upgrade_bin"
-    tar zxf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin"
+    mkdir -p "$PG_UPGRADE_DIR"
+
+    # upgrade job outputs a log in the cwd; needs write permissions
+    chown postgres:postgres "$PG_UPGRADE_DIR"
+
+    tar zxf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "$PG_UPGRADE_DIR"
 
     PGSHARENEW="$PG_UPGRADE_BIN_DIR/share"
 
@@ -289,7 +321,10 @@ function initiate_upgrade {
                         echo "1.1.1. Installing Nix using the provided installer"
                         tar -xzf "$NIX_INSTALLER_PACKAGE_PATH" -C /tmp/persistent/
                         chmod +x "$NIX_INSTALLER_PATH"
-                        "$NIX_INSTALLER_PATH" install --no-confirm
+
+                        "$NIX_INSTALLER_PATH" install --no-confirm \
+                        --extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \
+                        --extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
                     else
                         echo "1.1.1. Installing Nix using the official installer"
 
@@ -323,7 +358,7 @@ function initiate_upgrade {
         chown postgres:postgres "/var/lib/postgresql/extension/pgsodium_getkey"
     fi
 
-    chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION"
+    chown -R postgres:postgres "$PG_UPGRADE_BIN_DIR"
 
     # upgrade job outputs a log in the cwd; needs write permissions
     mkdir -p /tmp/pg_upgrade/
@@ -350,27 +385,7 @@ function initiate_upgrade {
     fi
     locale-gen
 
-    if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then
-        # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere
-        # excluding nvme0 since it is the root disk
-        echo "5. Determining block device to mount"
-        BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }')
-        echo "Block device found: $BLOCK_DEVICE"
-
-        mkdir -p "$MOUNT_POINT"
-        echo "6. Mounting block device"
-
-        sleep 5
-        e2fsck -pf "$BLOCK_DEVICE"
-
-        sleep 1
-        mount "$BLOCK_DEVICE" "$MOUNT_POINT"
-
-        sleep 1
-        resize2fs "$BLOCK_DEVICE"
-    else 
-        mkdir -p "$MOUNT_POINT"
-    fi
+    retry 3 mount_data_migration_disk
 
     if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then
         cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key
@@ -481,11 +496,11 @@ EOF
 
 trap cleanup ERR
 
-echo "running" > /tmp/pg-upgrade-status
+report_upgrade_status "running"
 if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then
     initiate_upgrade >> "$LOG_FILE" 2>&1 &
     echo "Upgrade initiate job completed"
 else
-    rm -f /tmp/pg-upgrade-status
+    rm -f "$UPGRADE_STATUS_FILE"
     initiate_upgrade
 fi
