Skip to content

Commit d35d35e

Browse files
marceloneppelmarceloneppel
authored
[DPE-6899] User->databases pg_hba rules (#919)
* List accessible databases for each relation user Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Improve retrieval of relation users Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Upgrade groups from users created by PgBouncer Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Change pg_hba to match the user->databases map Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Update Patroni configuration when user is added/removed Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Update config after user creation/deletion also on replicas Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Update Patroni configuration when user is added/removed through db and db-admin relations Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Create event trigger to monitor user creation by Pgbouncer Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Handle user creation/deletion through PgBouncer correctly Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Use right connection type Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix lint Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix unit tests Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix audit test Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix first new relations test Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix TLS and restart tests Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fixes for async replication Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix unit test Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix backup tests Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Add initial part of pg_hba test Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix initial pg_hba test Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix linting Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix update_pg_hba Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Improve test setup speed Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Add needed checks to the test Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix linting Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Log test steps Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Check error through regex and fix conditional deployment Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Test with two units Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Notify replicas about new users Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix infinite pebble notify calls Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Consider users created by db-admin relation Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Move observer to charm container Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Improve relation flag Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Add comments to database function and remove debug tables Signed-off-by: Marcelo Henrique Neppel <[email protected]> * Fix leader check, retry check on test and skip event trigger function code when not a superuser Signed-off-by: Marcelo Henrique Neppel <[email protected]> --------- Signed-off-by: Marcelo Henrique Neppel <[email protected]>
1 parent fcfa631 commit d35d35e

File tree

19 files changed

+756
-66
lines changed

19 files changed

+756
-66
lines changed

‎lib/charms/postgresql_k8s/v0/postgresql.py

Lines changed: 176 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
# Increment this PATCH version before using `charmcraft publish-lib` or reset
3737
# to 0 if you are raising the major API version
38-
LIBPATCH = 53
38+
LIBPATCH = 54
3939

4040
# Groups to distinguish HBA access
4141
ACCESS_GROUP_IDENTITY = "identity_access"
@@ -113,6 +113,10 @@ class PostgreSQLGetPostgreSQLVersionError(Exception):
113113
"""Exception raised when retrieving PostgreSQL version fails."""
114114

115115

116+
class PostgreSQLListAccessibleDatabasesForUserError(Exception):
117+
"""Exception raised when retrieving the accessible databases for a user fails."""
118+
119+
116120
class PostgreSQLListGroupsError(Exception):
117121
"""Exception raised when retrieving PostgreSQL groups list fails."""
118122

@@ -190,6 +194,11 @@ def create_access_groups(self) -> None:
190194
try:
191195
with self._connect_to_database() as connection, connection.cursor() as cursor:
192196
for group in ACCESS_GROUPS:
197+
cursor.execute(
198+
SQL("SELECT TRUE FROM pg_roles WHERE rolname={};").format(Literal(group))
199+
)
200+
if cursor.fetchone() is not None:
201+
continue
193202
cursor.execute(
194203
SQL("CREATE ROLE {} NOLOGIN;").format(
195204
Identifier(group),
@@ -622,15 +631,22 @@ def is_tls_enabled(self, check_current_host: bool = False) -> bool:
622631
# Connection errors happen when PostgreSQL has not started yet.
623632
return False
624633

625-
def list_access_groups(self) -> Set[str]:
634+
def list_access_groups(self, current_host=False) -> Set[str]:
626635
"""Returns the list of PostgreSQL database access groups.
627636
637+
Args:
638+
current_host: whether to check the current host
639+
instead of the primary host.
640+
628641
Returns:
629642
List of PostgreSQL database access groups.
630643
"""
631644
connection = None
645+
host = self.current_host if current_host else None
632646
try:
633-
with self._connect_to_database() as connection, connection.cursor() as cursor:
647+
with self._connect_to_database(
648+
database_host=host
649+
) as connection, connection.cursor() as cursor:
634650
cursor.execute(
635651
"SELECT groname FROM pg_catalog.pg_group WHERE groname LIKE '%_access';"
636652
)
@@ -643,16 +659,69 @@ def list_access_groups(self) -> Set[str]:
643659
if connection is not None:
644660
connection.close()
645661

646-
def list_users(self) -> Set[str]:
662+
def list_accessible_databases_for_user(self, user: str, current_host=False) -> Set[str]:
663+
"""Returns the list of accessible databases for a specific user.
664+
665+
Args:
666+
user: the user to check.
667+
current_host: whether to check the current host
668+
instead of the primary host.
669+
670+
Returns:
671+
List of accessible database (the ones where
672+
the user has the CONNECT privilege).
673+
"""
674+
connection = None
675+
host = self.current_host if current_host else None
676+
try:
677+
with self._connect_to_database(
678+
database_host=host
679+
) as connection, connection.cursor() as cursor:
680+
cursor.execute(
681+
SQL(
682+
"SELECT TRUE FROM pg_catalog.pg_user WHERE usename = {} AND usesuper;"
683+
).format(Literal(user))
684+
)
685+
if cursor.fetchone() is not None:
686+
return {"all"}
687+
cursor.execute(
688+
SQL(
689+
"SELECT datname FROM pg_catalog.pg_database WHERE has_database_privilege({}, datname, 'CONNECT') AND NOT datistemplate;"
690+
).format(Literal(user))
691+
)
692+
databases = cursor.fetchall()
693+
return {database[0] for database in databases}
694+
except psycopg2.Error as e:
695+
logger.error(f"Failed to list accessible databases for user {user}: {e}")
696+
raise PostgreSQLListAccessibleDatabasesForUserError() from e
697+
finally:
698+
if connection is not None:
699+
connection.close()
700+
701+
def list_users(self, group: Optional[str] = None, current_host=False) -> Set[str]:
647702
"""Returns the list of PostgreSQL database users.
648703
704+
Args:
705+
group: optional group to filter the users.
706+
current_host: whether to check the current host
707+
instead of the primary host.
708+
649709
Returns:
650710
List of PostgreSQL database users.
651711
"""
652712
connection = None
713+
host = self.current_host if current_host else None
653714
try:
654-
with self._connect_to_database() as connection, connection.cursor() as cursor:
655-
cursor.execute("SELECT usename FROM pg_catalog.pg_user;")
715+
with self._connect_to_database(
716+
database_host=host
717+
) as connection, connection.cursor() as cursor:
718+
if group:
719+
query = SQL(
720+
"SELECT usename FROM (SELECT UNNEST(grolist) AS user_id FROM pg_catalog.pg_group WHERE groname = {}) AS g JOIN pg_catalog.pg_user AS u ON g.user_id = u.usesysid;"
721+
).format(Literal(group))
722+
else:
723+
query = "SELECT usename FROM pg_catalog.pg_user;"
724+
cursor.execute(query)
656725
usernames = cursor.fetchall()
657726
return {username[0] for username in usernames}
658727
except psycopg2.Error as e:
@@ -662,19 +731,27 @@ def list_users(self) -> Set[str]:
662731
if connection is not None:
663732
connection.close()
664733

665-
def list_users_from_relation(self) -> Set[str]:
734+
def list_users_from_relation(self, current_host=False) -> Set[str]:
666735
"""Returns the list of PostgreSQL database users that were created by a relation.
667736
737+
Args:
738+
current_host: whether to check the current host
739+
instead of the primary host.
740+
668741
Returns:
669742
List of PostgreSQL database users.
670743
"""
671744
connection = None
745+
host = self.current_host if current_host else None
672746
try:
673-
with self._connect_to_database() as connection, connection.cursor() as cursor:
747+
with self._connect_to_database(
748+
database_host=host
749+
) as connection, connection.cursor() as cursor:
674750
cursor.execute(
675751
"SELECT usename "
676752
"FROM pg_catalog.pg_user "
677-
"WHERE usename LIKE 'relation_id_%' OR usename LIKE 'relation-%';"
753+
"WHERE usename LIKE 'relation_id_%' OR usename LIKE 'relation-%' "
754+
"OR usename LIKE 'pgbouncer_auth_relation_id_%' OR usename LIKE '%_user_%_%';"
678755
)
679756
usernames = cursor.fetchall()
680757
return {username[0] for username in usernames}
@@ -704,25 +781,100 @@ def set_up_database(self) -> None:
704781
"""Set up postgres database with the right permissions."""
705782
connection = None
706783
try:
784+
with self._connect_to_database(
785+
database="template1"
786+
) as connection, connection.cursor() as cursor:
787+
# Create database function and event trigger to identify users created by PgBouncer.
788+
cursor.execute(
789+
"SELECT TRUE FROM pg_event_trigger WHERE evtname = 'update_pg_hba_on_create_schema';"
790+
)
791+
if cursor.fetchone() is None:
792+
cursor.execute("""
793+
CREATE OR REPLACE FUNCTION update_pg_hba()
794+
RETURNS event_trigger
795+
LANGUAGE plpgsql
796+
AS $$
797+
DECLARE
798+
hba_file TEXT;
799+
copy_command TEXT;
800+
connection_type TEXT;
801+
rec record;
802+
insert_value TEXT;
803+
changes INTEGER = 0;
804+
BEGIN
805+
-- Don't execute on replicas.
806+
IF NOT pg_is_in_recovery() THEN
807+
-- Load the current authorisation rules.
808+
DROP TABLE IF EXISTS pg_hba;
809+
CREATE TEMPORARY TABLE pg_hba (lines TEXT);
810+
SELECT setting INTO hba_file FROM pg_settings WHERE name = 'hba_file';
811+
IF hba_file IS NOT NULL THEN
812+
copy_command='COPY pg_hba FROM ''' || hba_file || '''' ;
813+
EXECUTE copy_command;
814+
-- Build a list of the relation users and the databases they can access.
815+
DROP TABLE IF EXISTS relation_users;
816+
CREATE TEMPORARY TABLE relation_users AS
817+
SELECT t.user, STRING_AGG(DISTINCT t.database, ',') AS databases FROM( SELECT u.usename AS user, CASE WHEN u.usesuper THEN 'all' ELSE d.datname END AS database FROM ( SELECT usename, usesuper FROM pg_catalog.pg_user WHERE usename NOT IN ('backup', 'monitoring', 'operator', 'postgres', 'replication', 'rewind')) AS u JOIN ( SELECT datname FROM pg_catalog.pg_database WHERE NOT datistemplate ) AS d ON has_database_privilege(u.usename, d.datname, 'CONNECT') ) AS t GROUP BY 1;
818+
IF (SELECT COUNT(lines) FROM pg_hba WHERE lines LIKE 'hostssl %') > 0 THEN
819+
connection_type := 'hostssl';
820+
ELSE
821+
connection_type := 'host';
822+
END IF;
823+
-- Add the new users to the pg_hba file.
824+
FOR rec IN SELECT * FROM relation_users
825+
LOOP
826+
insert_value := connection_type || ' ' || rec.databases || ' ' || rec.user || ' 0.0.0.0/0 md5';
827+
IF (SELECT COUNT(lines) FROM pg_hba WHERE lines = insert_value) = 0 THEN
828+
INSERT INTO pg_hba (lines) VALUES (insert_value);
829+
changes := changes + 1;
830+
END IF;
831+
END LOOP;
832+
-- Remove users that don't exist anymore from the pg_hba file.
833+
FOR rec IN SELECT h.lines FROM pg_hba AS h LEFT JOIN relation_users AS r ON SPLIT_PART(h.lines, ' ', 3) = r.user WHERE r.user IS NULL AND (SPLIT_PART(h.lines, ' ', 3) LIKE 'relation_id_%' OR SPLIT_PART(h.lines, ' ', 3) LIKE 'pgbouncer_auth_relation_id_%' OR SPLIT_PART(h.lines, ' ', 3) LIKE '%_user_%_%')
834+
LOOP
835+
DELETE FROM pg_hba WHERE lines = rec.lines;
836+
changes := changes + 1;
837+
END LOOP;
838+
-- Apply the changes to the pg_hba file.
839+
IF changes > 0 THEN
840+
copy_command='COPY pg_hba TO ''' || hba_file || '''' ;
841+
EXECUTE copy_command;
842+
PERFORM pg_reload_conf();
843+
END IF;
844+
END IF;
845+
END IF;
846+
END;
847+
$$;
848+
""")
849+
cursor.execute("""
850+
CREATE EVENT TRIGGER update_pg_hba_on_create_schema
851+
ON ddl_command_end
852+
WHEN TAG IN ('CREATE SCHEMA')
853+
EXECUTE FUNCTION update_pg_hba();
854+
""")
855+
cursor.execute("""
856+
CREATE EVENT TRIGGER update_pg_hba_on_drop_schema
857+
ON ddl_command_end
858+
WHEN TAG IN ('DROP SCHEMA')
859+
EXECUTE FUNCTION update_pg_hba();
860+
""")
707861
with self._connect_to_database() as connection, connection.cursor() as cursor:
708862
cursor.execute("SELECT TRUE FROM pg_roles WHERE rolname='admin';")
709-
if cursor.fetchone() is not None:
710-
return
711-
712-
# Allow access to the postgres database only to the system users.
713-
cursor.execute("REVOKE ALL PRIVILEGES ON DATABASE postgres FROM PUBLIC;")
714-
cursor.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC;")
715-
for user in self.system_users:
716-
cursor.execute(
717-
SQL("GRANT ALL PRIVILEGES ON DATABASE postgres TO {};").format(
718-
Identifier(user)
863+
if cursor.fetchone() is None:
864+
# Allow access to the postgres database only to the system users.
865+
cursor.execute("REVOKE ALL PRIVILEGES ON DATABASE postgres FROM PUBLIC;")
866+
cursor.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC;")
867+
for user in self.system_users:
868+
cursor.execute(
869+
SQL("GRANT ALL PRIVILEGES ON DATABASE postgres TO {};").format(
870+
Identifier(user)
871+
)
719872
)
873+
self.create_user(
874+
PERMISSIONS_GROUP_ADMIN,
875+
extra_user_roles=["pg_read_all_data", "pg_write_all_data"],
720876
)
721-
self.create_user(
722-
PERMISSIONS_GROUP_ADMIN,
723-
extra_user_roles=["pg_read_all_data", "pg_write_all_data"],
724-
)
725-
cursor.execute("GRANT CONNECT ON DATABASE postgres TO admin;")
877+
cursor.execute("GRANT CONNECT ON DATABASE postgres TO admin;")
726878
except psycopg2.Error as e:
727879
logger.error(f"Failed to set up databases: {e}")
728880
raise PostgreSQLDatabasesSetupError() from e

‎poetry.lock

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)