Merge branch 'develop' into sam/wal-g-3

Author: samrose

Dockerfile-15

@@ -24,7 +24,7 @@ ARG rum_release=1.3.13
 ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
 ARG libsodium_release=1.0.18
 ARG pgsodium_release=3.1.6
-ARG pg_graphql_release=1.5.1
+ARG pg_graphql_release=1.5.11
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.1.4
 ARG pg_repack_release=1.4.8

Dockerfile-orioledb-17

@@ -24,7 +24,7 @@ ARG rum_release=1.3.13
 ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6
 ARG libsodium_release=1.0.18
 ARG pgsodium_release=3.1.6
-ARG pg_graphql_release=1.5.1
+ARG pg_graphql_release=1.5.11
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.1.4
 ARG pg_repack_release=1.4.8

ansible/files/admin_api_scripts/grow_fs.sh

@@ -9,15 +9,32 @@ if pgrep resizefs; then
     exit 1
 fi
 
+# Parses the output of lsblk to get the root partition number
+# Example output:
+# NAME        MOUNTPOINT
+# nvme0n1
+# ├─nvme0n1p1 /boot
+# └─nvme0n1p3 /
+# nvme1n1     /data
+#
+# Resulting in:
+# └─nvme0n1p3 / -> nvme0n1p3 -> 3
+ROOT_PARTITION_NUMBER=$(lsblk -no NAME,MOUNTPOINT | grep ' /$' | awk '{print $1;}' | sed 's/.*nvme[0-9]n[0-9]p//g')
+
+if ! [[ "$ROOT_PARTITION_NUMBER" =~ ^[0-9]+$ ]]; then
+  echo "Error: ROOT_PARTITION_NUMBER is not a valid number: $ROOT_PARTITION_NUMBER"
+  exit 1
+fi
+
 if [ -b /dev/nvme1n1 ] ; then
     if [[ "${VOLUME_TYPE}" == "data" ]]; then
         resize2fs /dev/nvme1n1
 
     elif [[ "${VOLUME_TYPE}" == "root" ]] ; then
         PLACEHOLDER_FL=/home/ubuntu/50M_PLACEHOLDER
         rm -f "${PLACEHOLDER_FL}" || true
-        growpart /dev/nvme0n1 2
-        resize2fs /dev/nvme0n1p2
+        growpart /dev/nvme0n1 "${ROOT_PARTITION_NUMBER}"
+        resize2fs "/dev/nvme0n1p${ROOT_PARTITION_NUMBER}"
         if [[ ! -f "${PLACEHOLDER_FL}" ]] ; then
             fallocate -l50M "${PLACEHOLDER_FL}"
         fi
@@ -26,7 +43,7 @@ if [ -b /dev/nvme1n1 ] ; then
         exit 1
     fi
 else
-    growpart /dev/nvme0n1 2
-    resize2fs /dev/nvme0n1p2
+    growpart /dev/nvme0n1 "${ROOT_PARTITION_NUMBER}"
+    resize2fs "/dev/nvme0n1p${ROOT_PARTITION_NUMBER}"
 fi
 echo "Done resizing disk"

ebssurrogate/scripts/qemu-bootstrap-nix.sh

@@ -22,7 +22,7 @@ function waitfor_boot_finished {
 }
 
 function install_packages {
-	apt-get update && sudo apt-get install software-properties-common e2fsprogs -y
+	apt-get update && sudo apt-get install software-properties-common e2fsprogs nfs-common -y
 	add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y
 	ansible-galaxy collection install community.general
 }
@@ -143,4 +143,6 @@ function clean_system {
 
 install_nix
 execute_stage2_playbook
+# we do not want to ship an initialized DB as this is performed as needed
+rm -rf /data/pgdata
 cloud-init clean --logs

flake.nix

@@ -68,6 +68,11 @@
               buildPgrxExtension_0_12_6 = prev.buildPgrxExtension.override {
                 cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_6;
               };
+
+              buildPgrxExtension_0_12_9 = prev.buildPgrxExtension.override {
+                cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_9;
+              };
+
             })
             (final: prev: {
               postgresql = final.callPackage ./nix/postgresql/default.nix {
@@ -395,6 +400,7 @@
           supabase-groonga = supabase-groonga;
           cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3;
           cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6;
+          cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9;
           # PostgreSQL versions.
           psql_15 = postgresVersions.psql_15;
           psql_orioledb-17 = postgresVersions.psql_orioledb-17;

migrations/README.md

@@ -78,15 +78,18 @@ Additionally, [supabase/postgres](https://github.com/supabase/postgres/blob/deve
 
 ### Add a Migration
 
+First, start a local postgres server and apply the migrations
+
 ```shell
 # Start the database server
-docker-compose up
+nix run .#dbmate-tool -- --version 15 --flake-url "."
 
 # create a new migration
+nix develop
 dbmate new '<some message>'
 ```
 
-Then, populate the migration at `./db/migrations/xxxxxxxxx_<some_message>` and make sure it execute sucessfully with
+Then, execute the migration at `./db/migrations/xxxxxxxxx_<some_message>` and make sure it runs sucessfully with
 
 ```shell
 dbmate up

migrations/db/migrations/20221207154255_create_pgsodium_and_vault.sql

@@ -5,34 +5,44 @@ DECLARE
   pgsodium_exists boolean;
   vault_exists boolean;
 BEGIN
-  pgsodium_exists = (
-    select count(*) = 1 
-    from pg_available_extensions 
-    where name = 'pgsodium'
-    and default_version in ('3.1.6', '3.1.7', '3.1.8', '3.1.9')
-  );
-  
-  vault_exists = (
+  IF EXISTS (SELECT FROM pg_available_extensions WHERE name = 'supabase_vault' AND default_version != '0.2.8') THEN
+    CREATE EXTENSION IF NOT EXISTS supabase_vault;
+
+    -- for some reason extension custom scripts aren't run during AMI build, so
+    -- we manually run it here
+    GRANT USAGE ON SCHEMA vault TO postgres WITH GRANT OPTION;
+    GRANT SELECT, DELETE ON vault.secrets, vault.decrypted_secrets TO postgres WITH GRANT OPTION;
+    GRANT EXECUTE ON FUNCTION vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt TO postgres WITH GRANT OPTION;
+  ELSE
+    pgsodium_exists = (
       select count(*) = 1 
       from pg_available_extensions 
-      where name = 'supabase_vault'
-  );
-
-  IF pgsodium_exists 
-  THEN
-    create extension if not exists pgsodium;
-
-    grant pgsodium_keyiduser to postgres with admin option;
-    grant pgsodium_keyholder to postgres with admin option;
-    grant pgsodium_keymaker  to postgres with admin option;
-
-    grant execute on function pgsodium.crypto_aead_det_decrypt(bytea, bytea, uuid, bytea) to service_role;
-    grant execute on function pgsodium.crypto_aead_det_encrypt(bytea, bytea, uuid, bytea) to service_role;
-    grant execute on function pgsodium.crypto_aead_det_keygen to service_role;
-
-    IF vault_exists
+      where name = 'pgsodium'
+      and default_version in ('3.1.6', '3.1.7', '3.1.8', '3.1.9')
+    );
+    
+    vault_exists = (
+        select count(*) = 1 
+        from pg_available_extensions 
+        where name = 'supabase_vault'
+    );
+  
+    IF pgsodium_exists 
     THEN
-      create extension if not exists supabase_vault;
+      create extension if not exists pgsodium;
+  
+      grant pgsodium_keyiduser to postgres with admin option;
+      grant pgsodium_keyholder to postgres with admin option;
+      grant pgsodium_keymaker  to postgres with admin option;
+  
+      grant execute on function pgsodium.crypto_aead_det_decrypt(bytea, bytea, uuid, bytea) to service_role;
+      grant execute on function pgsodium.crypto_aead_det_encrypt(bytea, bytea, uuid, bytea) to service_role;
+      grant execute on function pgsodium.crypto_aead_det_keygen to service_role;
+  
+      IF vault_exists
+      THEN
+        create extension if not exists supabase_vault;
+      END IF;
     END IF;
   END IF;
 END $$;

migrations/db/migrations/20230529180330_alter_api_roles_for_inherit.sql

@@ -4,7 +4,12 @@ ALTER ROLE authenticated inherit;
 ALTER ROLE anon inherit;
 ALTER ROLE service_role inherit;
 
-GRANT pgsodium_keyholder to service_role;
+DO $$
+BEGIN
+  IF EXISTS (SELECT FROM pg_roles WHERE rolname = 'pgsodium_keyholder') THEN
+    GRANT pgsodium_keyholder to service_role;
+  END IF;
+END $$;
 
 -- migrate:down
 

migrations/db/migrations/20250205144616_move_orioledb_to_extensions_schema.sql

@@ -0,0 +1,26 @@
+-- migrate:up
+do $$
+declare
+    ext_schema text;
+    extensions_schema_exists boolean;
+begin
+    -- check if the "extensions" schema exists
+    select exists (
+        select 1 from pg_namespace where nspname = 'extensions'
+    ) into extensions_schema_exists;
+
+    if extensions_schema_exists then
+        -- check if the "orioledb" extension is in the "public" schema
+        select nspname into ext_schema
+        from pg_extension e
+        join pg_namespace n on e.extnamespace = n.oid
+        where extname = 'orioledb';
+
+        if ext_schema = 'public' then
+            execute 'alter extension orioledb set schema extensions';
+        end if;
+    end if;
+end $$;
+
+-- migrate:down
+

migrations/db/migrations/20250218031949_pgsodium_mask_role.sql

@@ -1,25 +1,31 @@
 -- migrate:up
-CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text)
-RETURNS void
-LANGUAGE plpgsql
-SECURITY DEFINER
-SET search_path TO ''
-AS $function$
+
+DO $$
 BEGIN
-  EXECUTE format(
-    'GRANT SELECT ON pgsodium.key TO %s',
-    masked_role);
+  IF EXISTS (SELECT FROM pg_extension WHERE extname = 'pgsodium') THEN
+    CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text)
+    RETURNS void
+    LANGUAGE plpgsql
+    SECURITY DEFINER
+    SET search_path TO ''
+    AS $function$
+    BEGIN
+      EXECUTE format(
+        'GRANT SELECT ON pgsodium.key TO %s',
+        masked_role);
 
-  EXECUTE format(
-    'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s',
-    masked_role);
+      EXECUTE format(
+        'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s',
+        masked_role);
 
-  EXECUTE format(
-    'GRANT ALL ON %I TO %s',
-    view_name,
-    masked_role);
-  RETURN;
-END
-$function$;
+      EXECUTE format(
+        'GRANT ALL ON %I TO %s',
+        view_name,
+        masked_role);
+      RETURN;
+    END
+    $function$;
+  END IF;
+END $$;
 
 -- migrate:down