Merge branch 'develop' into fix/grant-vault-privs-to-service_role

Author: soedirgo

ansible/vars.yml

@@ -9,9 +9,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.068-orioledb"
-  postgres17: "17.4.1.018"
-  postgres15: "15.8.1.075"
+  postgresorioledb-17: "17.0.1.070-orioledb"
+  postgres17: "17.4.1.020"
+  postgres15: "15.8.1.077"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

flake.nix

@@ -154,12 +154,14 @@
           x: 
             x != ./nix/ext/timescaledb.nix &&
             x != ./nix/ext/timescaledb-2.9.1.nix &&
-            x != ./nix/ext/plv8.nix &&
-            x != ./nix/ext/pgjwt.nix
+            x != ./nix/ext/plv8.nix
         ) ourExtensions;
 
         orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ];
-        dbExtensions17 = orioleFilteredExtensions; 
+        dbExtensions17 = builtins.filter (
+          x: 
+            x != ./nix/ext/pgjwt.nix
+        ) orioleFilteredExtensions; 
         getPostgresqlPackage = version:
           pkgs.postgresql."postgresql_${version}";
         # Create a 'receipt' file for a given postgresql package. This is a way

migrations/README.md

@@ -134,7 +134,22 @@ After making changes to migrations, you should update the schema.sql files for e
 nix run .#dbmate-tool -- --version all
 ```
 
-This will create a schema.sql file for each major version of PostgreSQL. Commit these changes to your repository and push to your branch. The test.yml workflow will verify these changes against the test matrix.
+This will create automatically  schema.sql file for each major version of PostgreSQL and OrioleDB (the files are named like `schema-<ver>`, `schema-oriole-<ver>`). Commit these changes to your repository and push to your branch. The workflow in `.github/workflows/test.yml`  will re-run this command in CI, and perform a git diff to verify the idempotency of the migrations, and that the latest changes have been committed.
+
 ## Testing
 
-Migrations are tested in CI to ensure they do not raise an exception against previously released `supabase/postgres` docker images. The full version matrix is at [test.yml](./.github/workflows/test.yml) in the `supabase-version` variable.
+In addition to ci test mentioned above, you can test migrations locally by running the following test for each major version of postgres one at a time.
+
+Examples:
+
+```
+nix build .#checks.aarch64-darwin.psql_15 -L
+nix build .#checks.aarch64-darwin.psql_17 -L
+nix build .#checks.aarch64-darwin.psql_orioledb-17 -L
+```
+
+(Note that the evaluation and nix build of the postgres packages "bundle" of each major version must succeed here, even though we run one version at a time. If you made changes to postgres or extensions, or wrappers those may rebuild here when you run this. Otherwise they will usually download the prebuilt version from the supabase nix binary cache)
+
+At the end of these commands, you will see the output of both `pg_regress` tests, and migration tests
+
+see [Adding Tests](https://github.com/supabase/postgres/blob/develop/nix/docs/adding-tests.md) for more information.

migrations/db/init-scripts/00000000000000-initial-schema.sql

@@ -22,7 +22,9 @@ do $$
 begin 
     if exists (select 1 from pg_available_extensions where name = 'pgjwt') then
         if not exists (select 1 from pg_extension where extname = 'pgjwt') then
-            create extension if not exists pgjwt with schema "extensions" cascade;
+            if current_setting('server_version_num')::int / 10000 = 15 then
+                create extension if not exists pgjwt with schema "extensions" cascade;
+            end if;
         end if;
     end if;
 end $$;

migrations/schema-15.sql

@@ -1,3 +1,10 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 15.8
+-- Dumped by pg_dump version 15.8
+
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
@@ -1004,8 +1011,3 @@ CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop
 -- PostgreSQL database dump complete
 --
 
-
---
--- Dbmate schema migrations
---
-

migrations/schema-17.sql

@@ -1,3 +1,10 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 17.4
+-- Dumped by pg_dump version 17.4
+
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
@@ -991,8 +998,3 @@ CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop
 -- PostgreSQL database dump complete
 --
 
-
---
--- Dbmate schema migrations
---
-

migrations/schema-orioledb-17.sql

@@ -1,3 +1,10 @@
+--
+-- PostgreSQL database dump
+--
+
+-- Dumped from database version 17.0
+-- Dumped by pg_dump version 17.0
+
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
@@ -1005,8 +1012,3 @@ CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop
 -- PostgreSQL database dump complete
 --
 
-
---
--- Dbmate schema migrations
---
-

nix/tests/expected/pgbouncer.out

@@ -58,11 +58,30 @@ SELECT
 FROM pg_proc p 
 JOIN schema_obj s ON s.oid = p.pronamespace 
 CROSS JOIN LATERAL aclexplode(p.proacl) AS acl 
-ORDER BY object_name, grantee, privilege_type; 
+ORDER BY object_name, grantee, privilege_type;
   schema   | object_name |    grantee     | privilege_type 
 -----------+-------------+----------------+----------------
  pgbouncer | get_auth    | pgbouncer      | EXECUTE
  pgbouncer | get_auth    | postgres       | EXECUTE
  pgbouncer | get_auth    | supabase_admin | EXECUTE
 (3 rows)
 
+-- Ensure that pgbouncer.get_auth() function does not return an expired password
+create role test_expired_user_password with login password 'expired_password' valid until '2000-01-01 00:00:00+00';
+create role test_valid_user_password with login password 'valid_password' valid until '2100-01-01 00:00:00+00';
+-- Update the pg_authid catalog directly to replace with a known SCRAM hash
+update pg_authid set rolpassword = 'SCRAM-SHA-256$4096:testsaltbase64$storedkeybase64$serverkeybase64' where rolname = 'test_valid_user_password';
+select pgbouncer.get_auth('test_expired_user_password');
+           get_auth            
+-------------------------------
+ (test_expired_user_password,)
+(1 row)
+
+select pgbouncer.get_auth('test_valid_user_password');
+                                           get_auth                                           
+----------------------------------------------------------------------------------------------
+ (test_valid_user_password,SCRAM-SHA-256$4096:testsaltbase64$storedkeybase64$serverkeybase64)
+(1 row)
+
+drop role test_expired_user_password;
+drop role test_valid_user_password;

nix/tests/prime.sql

@@ -61,7 +61,9 @@ do $$
 begin 
     if exists (select 1 from pg_available_extensions where name = 'pgjwt') then
         if not exists (select 1 from pg_extension where extname = 'pgjwt') then
-            create extension if not exists pgjwt;
+            if current_setting('server_version_num')::int / 10000 = 15 then
+                create extension if not exists pgjwt;
+            end if;
         end if;
     end if;
 end $$;

nix/tests/sql/pgbouncer.sql

@@ -50,4 +50,17 @@ SELECT
 FROM pg_proc p 
 JOIN schema_obj s ON s.oid = p.pronamespace 
 CROSS JOIN LATERAL aclexplode(p.proacl) AS acl 
-ORDER BY object_name, grantee, privilege_type; 
+ORDER BY object_name, grantee, privilege_type;
+
+-- Ensure that pgbouncer.get_auth() function does not return an expired password
+create role test_expired_user_password with login password 'expired_password' valid until '2000-01-01 00:00:00+00';
+create role test_valid_user_password with login password 'valid_password' valid until '2100-01-01 00:00:00+00';
+-- Update the pg_authid catalog directly to replace with a known SCRAM hash
+update pg_authid set rolpassword = 'SCRAM-SHA-256$4096:testsaltbase64$storedkeybase64$serverkeybase64' where rolname = 'test_valid_user_password';
+
+select pgbouncer.get_auth('test_expired_user_password');
+
+select pgbouncer.get_auth('test_valid_user_password');
+
+drop role test_expired_user_password;
+drop role test_valid_user_password;