Merge branch 'release/15.6-lw12' into pcnc/pgroongatesting

pcnc · pcnc · commit a967eb15a1b1 · 2024-08-21T10:59:08.000+03:00
diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - develop
+      - release/*
     paths:
       - '.github/workflows/ami-release-nix.yml'
       - 'common-nix.vars.pkr.hcl'
@@ -54,7 +55,7 @@ jobs:
         run: |
           packer init stage2-nix-psql.pkr.hcl
           GIT_SHA=${{github.sha}}
-          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
+          packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
 
       - name: Grab release version
         id: process_release_version
diff --git a/.github/workflows/dockerhub-release-15-6.yml b/.github/workflows/dockerhub-release-15-6.yml
@@ -4,9 +4,11 @@ on:
   push:
     branches:
       - develop
+      - release/*
     paths:
       - ".github/workflows/dockerhub-release-15-6.yml"
       - "common-nix.vars*"
+  workflow_dispatch:
  
 jobs:
   settings:
diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml
@@ -4,7 +4,9 @@ on:
   push:
     branches:
       - develop
+      - release/*
   pull_request:
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -56,4 +58,4 @@ jobs:
                     -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
                     base_nix bash -c "./workspace/docker/nix/build_nix.sh"
     name: build psql bundle on ${{ matrix.arch }}
-    
+    
diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml
@@ -0,0 +1,101 @@
+name: Publish nix pg_upgrade_bin flake version
+
+on:
+  workflow_dispatch:
+    inputs:
+      postgresVersion:
+        description: 'Optional. Postgres version to publish against, i.e. 15.1.1.78'
+        required: false
+
+permissions:
+  id-token: write
+    
+jobs:
+  publish-staging:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Grab release version
+        id: process_release_version
+        run: |
+          VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
+            VERSION=${{ inputs.postgresVersion }}
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT"
+
+      - name: Create a tarball containing the latest nix flake version
+        working-directory: /tmp/
+        run: |
+          mkdir -p ${{ steps.process_release_version.outputs.major_version }}
+          echo $GITHUB_SHA > ${{ steps.process_release_version.outputs.major_version }}/nix_flake_version
+          tar -czvf pg_upgrade_bin.tar.gz ${{ steps.process_release_version.outputs.major_version }}
+
+      - name: configure aws credentials - staging
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+
+      - name: Upload pg_upgrade scripts to s3 staging
+        run: |
+          aws s3 cp /tmp/pg_upgrade_bin.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
+
+      - name: Slack Notification on Failure
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
+          SLACK_USERNAME: 'gha-failures-notifier'
+          SLACK_COLOR: 'danger'
+          SLACK_MESSAGE: 'Publishing pg_upgrade binaries flake version failed'
+          SLACK_FOOTER: ''
+
+  publish-prod:
+    runs-on: ubuntu-latest
+    if: github.ref_name == 'develop' || contains( github.ref, 'release' )
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - name: Grab release version
+        id: process_release_version
+        run: |
+          VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
+            VERSION=${{ inputs.postgresVersion }}
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT"
+
+      - name: Create a tarball containing the latest nix flake version
+        working-directory: /tmp/
+        run: |
+          mkdir -p ${{ steps.process_release_version.outputs.major_version }}
+          echo $GITHUB_SHA > ${{ steps.process_release_version.outputs.major_version }}/nix_flake_version
+          tar -czvf pg_upgrade_bin.tar.gz ${{ steps.process_release_version.outputs.major_version }}
+
+      - name: configure aws credentials - prod
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.PROD_AWS_ROLE }}
+          aws-region: "us-east-1"
+  
+      - name: Upload pg_upgrade scripts to s3 prod
+        run: |
+          aws s3 cp /tmp/pg_upgrade_bin.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
+
+      - name: Slack Notification on Failure
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
+          SLACK_USERNAME: 'gha-failures-notifier'
+          SLACK_COLOR: 'danger'
+          SLACK_MESSAGE: 'Publishing pg_upgrade binaries flake version failed'
+          SLACK_FOOTER: ''
diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml
@@ -4,9 +4,9 @@ on:
   push:
     branches:
       - develop
-      - sam/nix-and-conventional-ami
+      - release/*
     paths:
-      - '.github/workflows/publish-pgupgrade-scripts.yml'
+      - '.github/workflows/publish-nix-pgupgrade-scripts.yml'
       - 'common-nix.vars.pkr.hcl'
   workflow_dispatch:
     inputs:
@@ -62,7 +62,7 @@ jobs:
 
   publish-prod:
     runs-on: ubuntu-latest
-    if: github.ref_name == 'develop'
+    if: github.ref_name == 'develop' || contains( github.ref, 'release' )
 
     steps:
       - name: Checkout Repo
diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh
@@ -171,6 +171,58 @@ EOF
     done
 }
 
+function patch_wrappers {
+    local IS_NIX_UPGRADE=$1
+
+    WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');")
+    if [ "$WRAPPERS_ENABLED" = "f" ]; then
+        echo "Wrappers extension not enabled. Skipping."
+        return
+    fi
+
+    # This is a workaround for older versions of wrappers which don't have the expected
+    #  naming scheme, containing the version in their library's file name
+    #  e.g. wrappers-0.1.16.so, rather than wrappers.so
+    # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's
+    #  library directory, so we're making sure the new version has the expected (old version's)
+    #  file name.
+    # After the upgrade completes, the new version's library file is used.
+    # i.e. 
+    #  - old version: wrappers-0.1.16.so
+    #  - new version: wrappers-0.1.18.so
+    #  - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so
+    if [ "$IS_NIX_UPGRADE" = "true" ]; then
+        if [ -d "$PGLIBOLD" ]; then
+            OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
+            OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
+
+            find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do
+                if [ -f "$WRAPPERS_LIB_PATH" ]; then
+                    WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH")
+                    if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then
+                        echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
+                        cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}"
+                    fi
+                fi
+            done
+        fi
+    else
+        if [ -d "$PGLIBOLD" ]; then
+            WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit)
+            if [ -f "$WRAPPERS_LIB_PATH" ]; then
+                OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
+                if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then
+                    LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
+                    if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then
+                        echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}"
+                        cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}"
+                    fi
+                fi
+            fi
+        fi
+    fi
+}
+
 function initiate_upgrade {
     mkdir -p "$MOUNT_POINT"
     SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/")
@@ -324,30 +376,7 @@ function initiate_upgrade {
         export LD_LIBRARY_PATH="${PGLIBNEW}"
     fi
 
-    # This is a workaround for older versions of wrappers which don't have the expected
-    #  naming scheme, containing the version in their library's file name
-    #  e.g. wrappers-0.1.16.so, rather than wrappers.so
-    # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's
-    #  library directory, so we're making sure the new version has the expected (old version's)
-    #  file name.
-    # After the upgrade completes, the new version's library file is used.
-    # i.e. 
-    #  - old version: wrappers-0.1.16.so
-    #  - new version: wrappers-0.1.18.so
-    #  - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so
-    if [ -d "$PGLIBOLD" ]; then
-        WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit)
-        if [ -f "$WRAPPERS_LIB_PATH" ]; then
-            OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit)
-            if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then
-                LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH")
-                if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then
-                    echo "Copying $OLD_WRAPPER_LIB_PATH to $WRAPPERS_LIB_PATH"
-                    cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}"
-                fi
-            fi
-        fi
-    fi
+    patch_wrappers "$IS_NIX_UPGRADE"
 
     echo "9. Creating new data directory, initializing database"
     chown -R postgres:postgres "$MOUNT_POINT/"
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
@@ -206,3 +206,9 @@
       shell: |
         sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove osquery"
       when: stage2_nix
+
+    - name: nix collect garbage
+      become: yes
+      shell: |
+        sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d"
+      when: stage2_nix
diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml
@@ -1,6 +1,20 @@
 - name: PostgREST - system user
   user: name=postgrest
 
+- name: PostgREST - add Postgres PPA gpg key
+  apt_key:
+    url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
+    state: present
+
+- name: PostgREST - add Postgres PPA
+  apt_repository:
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}"
+    state: present
+
+- name: PostgREST - update apt cache
+  apt:
+    update_cache: yes
+
 # libpq is a C library that enables user programs to communicate with
 # the PostgreSQL database server.
 - name: PostgREST - system dependencies
@@ -9,9 +23,20 @@
       - libpq5
       - libnuma-dev
 
+- name: PostgREST - remove Postgres PPA gpg key
+  apt_key:
+    url: https://www.postgresql.org/media/keys/ACCC4CF8.asc
+    state: absent
+
+- name: PostgREST - remove Postgres PPA
+  apt_repository:
+    repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}"
+    state: absent
+
 - name: postgis - ensure dependencies do not get autoremoved
   shell: |
     set -e
+    apt-mark manual libpq5*
     apt-mark manual libnuma*
     apt-mark auto libnuma*-dev
 
diff --git a/ansible/vars.yml b/ansible/vars.yml
@@ -50,7 +50,7 @@ postgres_exporter_release_checksum:
   arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
   amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
 
-adminapi_release: 0.64.2
+adminapi_release: 0.66.1
 adminmgr_release: 0.22.1
 
 # Postgres Extensions
diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl
@@ -1 +1 @@
-postgres-version = "15.6.1.110-pcgroonga"
+postgres-version = "15.6.1.114-pcgroonga"
diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix
@@ -14,7 +14,10 @@ buildPgrxExtension_0_11_3 rec {
 
   nativeBuildInputs = [ cargo ];
   buildInputs = [ postgresql ];
-  
+  # update the following array when the pg_jsonschema version is updated
+  # required to ensure that extensions update scripts from previous versions are generated
+
+  previousVersions = ["0.3.0" "0.2.0" "0.1.4" "0.1.4" "0.1.2" "0.1.1" "0.1.0"];
   CARGO="${cargo}/bin/cargo";
   env = lib.optionalAttrs stdenv.isDarwin {
     POSTGRES_LIB = "${postgresql}/lib";
@@ -26,6 +29,31 @@ buildPgrxExtension_0_11_3 rec {
   # to fix this a bit later.
   doCheck = false;
 
+  preBuild = ''
+    echo "Processing git tags..."
+    echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt
+  '';
+
+  postInstall = ''
+    echo "Creating SQL files for previous versions..."
+    current_version="${version}"
+    sql_file="$out/share/postgresql/extension/pg_jsonschema--$current_version.sql"
+    
+    if [ -f "$sql_file" ]; then
+      while read -r previous_version; do
+        if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then
+          new_file="$out/share/postgresql/extension/pg_jsonschema--$previous_version--$current_version.sql"
+          echo "Creating $new_file"
+          cp "$sql_file" "$new_file"
+        fi
+      done < git_tags.txt
+    else
+      echo "Warning: $sql_file not found"
+    fi
+    rm git_tags.txt
+  '';
+
+
   meta = with lib; {
     description = "JSON Schema Validation for PostgreSQL";
     homepage = "https://github.com/supabase/${pname}";
diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-postgres-version = "15.6.1.110-pcgroonga"`
	`1`	`+postgres-version = "15.6.1.114-pcgroonga"`