fix: BI-6847 fix postgres schema filters by adding escaping (#1468)

* fix: BI-6847 fix postgres schema filters by adding escaping

* fix test

Author: KonstantAnxiety

lib/dl_connector_greenplum/dl_connector_greenplum/core/adapters.py

@@ -27,8 +27,8 @@ def get_list_all_tables_query(
             JOIN pg_namespace
             ON pg_namespace.oid = pg_class.relnamespace
         WHERE
-            pg_namespace.nspname not like 'pg_%'
-            AND pg_namespace.nspname not like 'gp_%'
+            pg_namespace.nspname not like 'pg\_%'
+            AND pg_namespace.nspname not like 'gp\_%'
             AND pg_namespace.nspname != 'session_state'
             AND pg_namespace.nspname != 'information_schema'
             AND pg_class.relkind in ('m', 'p', 'r', 'v')
@@ -58,8 +58,8 @@ def get_list_schema_names_query(
         sql_parts = [
             """
         SELECT nspname FROM pg_namespace
-        WHERE nspname NOT LIKE 'pg_%'
-        AND nspname NOT LIKE 'gp_%'
+        WHERE nspname NOT LIKE 'pg\_%'
+        AND nspname NOT LIKE 'gp\_%'
         AND nspname != 'session_state'
         AND nspname != 'information_schema'
         """

lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/adapters_base_postgres.py

@@ -387,7 +387,7 @@ def get_list_all_tables_query(
             JOIN pg_namespace
             ON pg_namespace.oid = pg_class.relnamespace
         WHERE
-            pg_namespace.nspname not like 'pg_%'
+            pg_namespace.nspname not like 'pg\_%'
             AND pg_namespace.nspname != 'information_schema'
             AND pg_class.relkind in ('m', 'p', 'r', 'v')
             AND NOT COALESCE((row_to_json(pg_class)->>'relispartition')::boolean, false)
@@ -412,7 +412,7 @@ def get_list_schema_names_query(
         sql_parts = [
             """
         SELECT nspname FROM pg_namespace
-        WHERE nspname NOT LIKE 'pg_%'
+        WHERE nspname NOT LIKE 'pg\_%'
         """
         ]
 

lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_adapter.py

@@ -34,6 +34,7 @@ async def test_tables_list(self, conn_bi_context: RequestContextInfo, target_con
             if t.schema_name != "public"  # skip public schema because it contains garbage tables from other tests
             # TODO: clean up tables created in other tests
         ] == [
+            "pgmyschema.test_table",
             "test_data.sample",
             "test_data_partitions.sample_partition",
         ]
@@ -54,3 +55,26 @@ async def test_tables_list_schema(
         )
 
         assert [f"{t.schema_name}.{t.table_name}" for t in tables] == [f"{schema}.{t}" for t in expected_tables]
+
+    @pytest.mark.asyncio
+    async def test_list_system_schemas(
+        self, conn_bi_context: RequestContextInfo, target_conn_dto: PostgresConnTargetDTO
+    ):
+        """Test that tables from schemas starting with 'pg' but not 'pg_' are included in the table list."""
+        dba = self._make_dba(target_conn_dto, conn_bi_context)
+        tables = await dba.get_tables(SchemaIdent(db_name="test_data", schema_name=None))
+        table_schemas = {t.schema_name for t in tables}
+
+        assert "pgmyschema" in table_schemas, (
+            f"Schema 'pgmyschema' should be included in table list. " f"Found schemas: {sorted(table_schemas)}"
+        )
+
+        pgmyschema_tables = [f"{t.schema_name}.{t.table_name}" for t in tables if t.schema_name == "pgmyschema"]
+        assert "pgmyschema.test_table" in pgmyschema_tables, (
+            f"Table 'pgmyschema.test_table' should be in the results. " f"Found: {pgmyschema_tables}"
+        )
+
+        pg_underscore_schemas = [s for s in table_schemas if s.startswith("pg_")]
+        assert len(pg_underscore_schemas) == 0, (
+            f"System schemas starting with 'pg_' should be filtered out. " f"Found: {pg_underscore_schemas}"
+        )

lib/dl_connector_postgresql/docker-compose/db-postgres/initdb.d/03_prepare_edge_case_schemas.sql

@@ -0,0 +1,9 @@
+-- Create schemas that test edge cases for schema filtering
+-- This schema name starts with 'pg' but doesn't have an underscore after it,
+-- so it should NOT be filtered out by the pattern 'pg\_%'
+CREATE SCHEMA IF NOT EXISTS pgmyschema;
+CREATE TABLE pgmyschema.test_table (
+    id INTEGER PRIMARY KEY,
+    name VARCHAR(255)
+);
+INSERT INTO pgmyschema.test_table (id, name) VALUES (1, 'test');