Merge branch 'develop' into local-infra-bootstrap

Author: samrose

.github/workflows/publish-nix-pgupgrade-scripts.yml

@@ -49,8 +49,8 @@ jobs:
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
-          VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
+          VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          VERSION=$(echo $VERSION | tr -d '"')  # Remove any surrounding quotes
           if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
             VERSION=${{ inputs.postgresVersion }}
           fi
@@ -81,7 +81,6 @@ jobs:
           SLACK_COLOR: 'danger'
           SLACK_MESSAGE: 'Publishing pg_upgrade scripts failed'
           SLACK_FOOTER: ''
-
   publish-prod:
     needs: prepare
     runs-on: ubuntu-latest
@@ -95,11 +94,13 @@ jobs:
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
-
+      
+      - uses: DeterminateSystems/nix-installer-action@main
+  
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
           VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
             VERSION=${{ inputs.postgresVersion }}

.github/workflows/test.yml

@@ -1,12 +1,10 @@
 name: Test Database
-
 on:
   push:
     branches:
       - develop
   pull_request:
   workflow_dispatch:
-
 jobs:
   prepare:
     runs-on: ubuntu-latest
@@ -15,150 +13,72 @@ jobs:
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
-      
-      - uses: DeterminateSystems/nix-installer-action@main
 
+      - name: Clear Nix cache
+        run: |
+          sudo rm -rf /home/runner/.cache/nix
+
+      - uses: DeterminateSystems/nix-installer-action@main
+        with:
+          extra-conf: |
+            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
+            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
       - name: Set PostgreSQL versions
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[] | select(. != "orioledb-17")' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
           echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
-
   build:
     needs: prepare
     strategy:
       matrix:
         postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
         include:
-          - runner: [self-hosted, X64]
+          - runner: ubuntu-22.04
             arch: amd64
-          - runner: arm-runner
+          - runner: ubuntu-22.04
             arch: arm64
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 180
     env:
       POSTGRES_PORT: 5478
       POSTGRES_PASSWORD: password
     steps:
-      - uses: actions/checkout@v3
-
+      - uses: actions/checkout@v4
+      - name: Clear Nix cache
+        run: |
+          sudo rm -rf /home/runner/.cache/nix
       - uses: DeterminateSystems/nix-installer-action@main
-
+        with:
+          extra-conf: |
+            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
+            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
       - name: Set PostgreSQL version environment variable
         run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
-
       - name: Strip quotes from pg major and set env var
         run: |
           stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/')
           echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV
-
       - name: Generate common-nix.vars.pkr.hcl
         run: |
-          PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
           PG_VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
-
       - id: settings
-        # Remove spaces and quotes to get the raw version string
         run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT
-
       - name: Generate args
         id: args
         run: |
-          ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml)
+          ARGS=$(nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml)
           echo "result<<EOF" >> $GITHUB_OUTPUT
           echo "$ARGS" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
-
-      - run: docker context create builders
-      - uses: docker/setup-buildx-action@v3
-        with:
-          endpoint: builders
-      - uses: docker/build-push-action@v5
-        with:
-          load: true
-          context: .
-          file: Dockerfile-${{ env.PGMAJOR }}
-          target: production
-          build-args: |
-            ${{ steps.args.outputs.result }}
-          tags: supabase/postgres:${{ steps.settings.outputs.postgres-version }},supabase_postgres
-          cache-from: |
-            type=gha,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-            type=gha,scope=${{ github.base_ref }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-
-      - name: Start Postgres
-        run: |
-          docker run --rm --pull=never \
-          -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \
-          -p ${{ env.POSTGRES_PORT }}:5432 \
-          --name supabase_postgres \
-          -d supabase/postgres:${{ steps.settings.outputs.postgres-version }}
-
-      - name: Install psql
-        run: |
-          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
-          sudo apt update
-          sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }}
-
-      - name: Install pg_prove
-        run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP
-        env:
-          SHELL: /bin/bash
-
-      - name: Wait for healthy database
-        run: |
-          count=0
-          until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do
-              exit=$?
-              count=$((count + 1))
-              if [ $count -ge "$retries" ]; then
-                  echo "Retry $count/$retries exited $exit, no more retries left."
-                  docker stop -t 2 "$container"
-                  return $exit
-              fi
-              sleep 1;
-          done;
-          echo "$container container is healthy"
-        env:
-          retries: 20
-          container: supabase_postgres
-
-      - name: Run tests
-        run: pg_prove migrations/tests/test.sql
-        env:
-          PGHOST: localhost
-          PGPORT: ${{ env.POSTGRES_PORT }}
-          PGDATABASE: postgres
-          PGUSER: supabase_admin
-          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
-
-      - name: Check migrations are idempotent
-        run: |
-          for sql in ./migrations/db/migrations/*.sql; do
-            echo "$0: running $sql"
-            psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql"
-          done
-        env:
-          PGHOST: localhost
-          PGPORT: ${{ env.POSTGRES_PORT }}
-          PGDATABASE: postgres
-          PGUSER: supabase_admin
-          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
-
-      - name: Update Dockerfile.dbmate version
-        run: |
-          sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate
-
       - name: verify schema.sql is committed
         run: |
-          docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit
+          nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }}
           if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then
             echo "Detected changes in schema.sql:"
             git diff migrations/schema-${{ env.PGMAJOR }}.sql
             exit 1
-          fi
+          fi

.github/workflows/testinfra-nix.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set PostgreSQL versions
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq --  -R -s -c 'split("\n")[:-1]')
           echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
 
   test-ami-nix:
@@ -61,6 +61,7 @@ jobs:
 
       - name: Set PostgreSQL version environment variable
         run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
+      
 
       - name: Generate common-nix.vars.pkr.hcl
         run: |

.gitignore

@@ -22,3 +22,5 @@ result*
 #IDE
 .idea/
 .vscode/
+
+db

ansible/files/postgres_prestart.sh.j2

@@ -1,9 +1,49 @@
 #!/bin/bash
 
+check_orioledb_enabled() {
+   local pg_conf="/etc/postgresql/postgresql.conf"
+   if [ ! -f "$pg_conf" ]; then
+       return 0
+    fi
+   grep "^shared_preload_libraries" "$pg_conf" | grep -c "orioledb" || return 0
+}
+
+get_shared_buffers() {
+    local opt_conf="/etc/postgresql-custom/generated-optimizations.conf"
+    if [ ! -f "$opt_conf" ]; then
+        return 0
+    fi
+    grep "^shared_buffers = " "$opt_conf" | cut -d "=" -f2 | tr -d ' ' || return 0
+}
+
+update_orioledb_buffers() {
+   local pg_conf="/etc/postgresql/postgresql.conf"
+   local value="$1"
+   if grep -q "^orioledb.main_buffers = " "$pg_conf"; then
+       sed -i "s/^orioledb.main_buffers = .*/orioledb.main_buffers = $value/" "$pg_conf"
+   else
+       echo "orioledb.main_buffers = $value" >> "$pg_conf"
+   fi
+}
+
+main() {
+   local has_orioledb=$(check_orioledb_enabled)
+   if [ "$has_orioledb" -lt 1 ]; then
+       return 0
+   fi
+   local shared_buffers_value=$(get_shared_buffers)
+   if [ ! -z "$shared_buffers_value" ]; then
+       update_orioledb_buffers "$shared_buffers_value"
+   fi
+}
+
+# Initial locale setup
 if [ $(cat /etc/locale.gen | grep -c en_US.UTF-8) -eq 0 ]; then
-    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
+   echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
 fi
 
 if [ $(locale -a | grep -c en_US.utf8) -eq 0 ]; then
-    locale-gen
+   locale-gen
 fi
+
+main

ansible/tasks/setup-postgres.yml

@@ -204,23 +204,49 @@
     ansible_command_timeout: 60
   when: debpkg_mode
 
-- name: Initialize the database stage2_nix
-  become: yes
-  become_user: postgres
-  shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin"
-  args:
-    executable: /bin/bash
-  environment:
-    LANG: en_US.UTF-8
-    LANGUAGE: en_US.UTF-8
-    LC_ALL: en_US.UTF-8
-    LC_CTYPE: en_US.UTF-8
-    LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
-  vars:
-    ansible_command_timeout: 60
-    # Circumvents the following error:
-    # "Timeout (12s) waiting for privilege escalation prompt"
-  when: stage2_nix 
+- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary
+  block:
+    - name: Check if psql_version is psql_orioledb
+      set_fact:
+        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
+
+    - name: Initialize the database stage2_nix (non-orioledb)
+      become: yes
+      become_user: postgres
+      shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin"
+      args:
+        executable: /bin/bash
+      environment:
+        LANG: en_US.UTF-8
+        LANGUAGE: en_US.UTF-8
+        LC_ALL: en_US.UTF-8
+        LC_CTYPE: en_US.UTF-8
+        LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
+      vars:
+        ansible_command_timeout: 60
+      when: stage2_nix and not is_psql_oriole
+
+    - name: Initialize the database stage2_nix (orioledb)
+      become: yes
+      become_user: postgres
+      shell: >
+        source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data 
+        --allow-group-access 
+        --username=supabase_admin 
+        --locale-provider=icu 
+        --encoding=UTF-8 
+        --icu-locale=en_US.UTF-8 
+      args:
+        executable: /bin/bash
+      environment:
+        LANG: en_US.UTF-8
+        LANGUAGE: en_US.UTF-8
+        LC_ALL: en_US.UTF-8
+        LC_CTYPE: en_US.UTF-8
+        LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
+      vars:
+        ansible_command_timeout: 60
+      when: stage2_nix and is_psql_oriole
 
 - name: copy PG systemd unit
   template: