Merge branch 'develop' into revoke-history

Author: soedirgo

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 * @supabase/backend
 migrations/ @supabase/cli @supabase/backend
-docker/orioledb @supabase/postgres
+docker/orioledb @supabase/postgres @supabase/backend
 common.vars.pkr.hcl @supabase/postgres @supabase/backend

.github/workflows/dockerhub-release-aio.yml

@@ -13,11 +13,17 @@ on:
       - develop
     types:
       - completed
+  workflow_dispatch:
+    inputs:
+      baseDockerVersion:
+        description: 'Base Docker Version. E.g., 15.1.1.27'
+        required: false
 
 jobs:
   settings:
     runs-on: ubuntu-latest
     outputs:
+      base_docker_version: ${{ steps.base_docker.outputs.base-docker-version }}
       docker_version: ${{ steps.settings.outputs.postgres-version }}
       image_tag: supabase/postgres:aio-${{ steps.settings.outputs.postgres-version }}
       fly_image_tag: supabase-postgres-image:aio-${{ steps.settings.outputs.postgres-version }}
@@ -27,6 +33,13 @@ jobs:
       - id: settings
         # Remove spaces and quotes to get the raw version string
         run: sed -r 's/(\s|\")+//g' common.vars.pkr.hcl >> $GITHUB_OUTPUT
+      - id: base_docker
+        run: |
+          if [[ "${{ inputs.baseDockerVersion }}" != "" ]]; then
+            echo "base-docker-version=${{ inputs.baseDockerVersion }}" >> $GITHUB_OUTPUT
+          else
+            echo "base-docker-version=${{ steps.settings.outputs.postgres-version }}" >> $GITHUB_OUTPUT
+          fi
       - id: args
         uses: mikefarah/yq@master
         with:
@@ -60,7 +73,7 @@ jobs:
           file: docker/all-in-one/Dockerfile
           push: true
           build-args: |
-            postgres_version=${{ needs.settings.outputs.docker_version }}
+            postgres_version=${{ needs.settings.outputs.base_docker_version }}
             ${{ needs.settings.outputs.build_args }}
           target: production
           tags: ${{ needs.settings.outputs.image_tag }}_${{ matrix.arch }}

.github/workflows/nix-build.yml

@@ -0,0 +1,46 @@
+name: Nix CI
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+
+permissions:
+  contents: read
+  id-token: write    
+    
+jobs:
+  build-run-image:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: larger-runner-4cpu
+            arch: amd64
+          - runner: arm-runner
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+
+    steps:
+
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: aws-creds
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+          output-credentials: true
+      - name: write secret key
+        # use python so we don't interpolate the secret into the workflow logs, in case of bugs
+        run: |
+          python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()"
+        env:
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+  
+      - name: Build docker images
+        run: docker build -t base_nix -f docker/nix/Dockerfile .
+      - name: Run build psql bundle
+        run:  docker run -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} base_nix bash -c "./workspace/docker/nix/build.sh"
+    name: build psql bundle on ${{ matrix.arch }}

.github/workflows/nix-cache-upload.yml

@@ -0,0 +1,52 @@
+name: Nix Cache upload
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write    
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: [self-hosted, X64]
+            arch: amd64
+          - runner: arm-runner
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+    name: nix-build
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+        with:
+          fetch-depth: 0
+      - uses: DeterminateSystems/nix-installer-action@65d7c888b2778e8cf30a07a88422ccb23499bfb8
+      - uses: DeterminateSystems/magic-nix-cache-action@749fc5bbc9fa49d60c2b93f6c4bc867b82e1d295
+      - name: configure aws credentials for s3
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: "us-east-1"
+          kvm: true
+          extra-conf: |
+            system-features = kvm
+
+      - name: write secret key
+        # use python so we don't interpolate the secret into the workflow logs, in case of bugs
+        run: |
+          python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()"
+        env:
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+
+      - name: build and copy to S3
+        run: |
+          for x in 15 16 orioledb_16; do
+            nix build .#psql_$x/bin -o result-$x
+          done
+          nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./result*

.gitignore

@@ -10,3 +10,8 @@ ansible/image-manifest*.json
 __pycache__/
 *.py[cod]
 *$py.class
+
+#nix related
+result*
+.env-local
+.history

Dockerfile

@@ -31,11 +31,12 @@ ARG pg_repack_release=1.4.8
 ARG vault_release=0.2.8
 ARG groonga_release=12.0.8
 ARG pgroonga_release=2.4.0
-ARG wrappers_release=0.2.0
+ARG wrappers_release=0.3.0
 ARG hypopg_release=1.3.1
 ARG pgvector_release=0.4.0
 ARG pg_tle_release=1.3.2
-ARG supautils_release=2.1.0
+ARG index_advisor_release=0.2.0
+ARG supautils_release=2.2.0
 ARG wal_g_release=2.0.1
 
 ####################
@@ -556,7 +557,7 @@ FROM ccache as libsodium
 ARG libsodium_release
 ARG libsodium_release_checksum
 ADD --checksum=${libsodium_release_checksum} \
-    "https://download.libsodium.org/libsodium/releases/libsodium-${libsodium_release}.tar.gz" \
+    "https://supabase-public-artifacts-bucket.s3.amazonaws.com/libsodium/libsodium-${libsodium_release}.tar.gz" \
     /tmp/libsodium.tar.gz
 RUN tar -xvf /tmp/libsodium.tar.gz -C /tmp && \
     rm -rf /tmp/libsodium.tar.gz
@@ -803,6 +804,24 @@ RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccac
 # Create debian package
 RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --nodoc
 
+######################
+# 30-index_advisor.yml
+######################
+FROM ccache as index_advisor
+ARG index_advisor_release
+ARG index_advisor_release_checksum
+ADD --checksum=${index_advisor_release_checksum} \
+    "https://github.com/olirice/index_advisor/archive/refs/tags/v${index_advisor_release}.tar.gz" \
+    /tmp/index_advisor.tar.gz
+RUN tar -xvf /tmp/index_advisor.tar.gz -C /tmp && \
+    rm -rf /tmp/index_advisor.tar.gz
+# Build from source
+WORKDIR /tmp/index_advisor-${index_advisor_release}
+RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccache \
+    make -j$(nproc)
+# Create debian package
+RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --nodoc
+
 ####################
 # internal/supautils.yml
 ####################
@@ -857,6 +876,7 @@ COPY --from=hypopg-source /tmp/*.deb /tmp/
 COPY --from=pg_repack-source /tmp/*.deb /tmp/
 COPY --from=pgvector-source /tmp/*.deb /tmp/
 COPY --from=pg_tle-source /tmp/*.deb /tmp/
+COPY --from=index_advisor /tmp/*.deb /tmp/
 COPY --from=supautils /tmp/*.deb /tmp/
 
 ####################
@@ -927,6 +947,7 @@ RUN sed -i \
 
 # Include schema migrations
 COPY migrations/db /docker-entrypoint-initdb.d/
+COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql
 COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql
 
 # Add upstream entrypoint script

README.md

@@ -109,3 +109,12 @@ $ time packer build -timestamp-ui \
 We are building the features of Firebase using enterprise-grade, open source products. We support existing communities wherever possible, and if the products don’t exist we build them and open source them ourselves.
 
 [![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase)
+
+
+## Experimental Nix Packaging of resources
+
+There is a `/nix` folder in this repo, plus a `flake.nix` and `flake.lock` that facilitate using the Nix package management system to package supabase/postgres, and all of our extensions and wrappers. A user will need nix installed on their machine. As of 4/1/2024 the package set only builds on target machines (`x86_64-linux` and `aarch64-linux`), however work is under way to also support building and using directly on `aarch64-darwin` (macOs). As of 4/1/2024, versions of packages and extensions are synced from `/ansible/vars.yml` via a utility that can be run by executing `nix run .#sync-exts-versions` (you must have nix installed and be on the supported `x86_64-linux` and `aarch64-linux` for this command to work). The short term goal is to sync these versions as they are updated by our infrastructure and postgres teams, then to see the nix packaged versions build successfully in parallel over time, along with tests of the nix packaged versions passing. 
+
+The supabase/postgres repo will continue to source it's dependencies from ansible for the short term, while we stabilize this nix build. 
+
+Forthcoming PR's will include: integrating the nix work into our ansible/packer builds, building natively on aarch64-darwin (macOs), more testing

ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

@@ -48,6 +48,28 @@ EOF
 
         run_sql -c "$PG_NET_GRANT_QUERY"
     fi
+
+    # Patching pg_cron ownership as it resets during upgrade
+    HAS_PG_CRON_OWNED_BY_POSTGRES=$(run_sql -A -t -c "select count(*) > 0 from pg_extension where extname = 'pg_cron' and extowner::regrole::text = 'postgres';")
+
+    if [ "$HAS_PG_CRON_OWNED_BY_POSTGRES" = "t" ]; then
+        RECREATE_PG_CRON_QUERY=$(cat <<EOF
+        begin;
+        create temporary table cron_job as select * from cron.job;
+        create temporary table cron_job_run_details as select * from cron.job_run_details;
+        drop extension pg_cron;
+        create extension pg_cron schema pg_catalog;
+        insert into cron.job select * from cron_job;
+        insert into cron.job_run_details select * from cron_job_run_details;
+        select setval('cron.jobid_seq', coalesce(max(jobid), 0) + 1, false) from cron.job;
+        select setval('cron.runid_seq', coalesce(max(runid), 0) + 1, false) from cron.job_run_details;
+        update cron.job set username = 'postgres' where username = 'supabase_admin';
+        commit;
+EOF
+        )
+
+        run_sql -c "$RECREATE_PG_CRON_QUERY"
+    fi
 }
 
 function complete_pg_upgrade {

ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh

@@ -306,6 +306,8 @@ EOF
     echo "11. Copying custom configurations"
     mkdir -p "$MOUNT_POINT/conf"
     cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/"
+    # removing supautils config as to allow the latest one provided by the latest image to be used
+    rm -f "$MOUNT_POINT/conf/supautils.conf" || true
 
     # removing wal-g config as to allow it to be explicitly enabled on the new instance
     rm -f "$MOUNT_POINT/conf/wal-g.conf"

ansible/files/postgresql_config/supautils.conf.j2

@@ -1,8 +1,9 @@
 supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}'
-# full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
+supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.broadcasts","realtime.channels","realtime.presences","storage.buckets","storage.migrations","storage.objects"]}'
+# full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
 # omitted because may be unsafe:             adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility
 # omitted because deprecated:                intagg, xml2
-supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
+supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
 supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
 supautils.privileged_extensions_superuser = 'supabase_admin'
 supautils.privileged_role = 'postgres'