perf: add root level filtering to all queries

Author: avallete

src/lib/PostgresMetaConfig.ts

@@ -1,4 +1,4 @@
-import { CONFIG_SQL } from './sql/index.js'
+import { CONFIG_SQL } from './sql/config.sql.js'
 import { PostgresMetaResult, PostgresConfig } from './types.js'
 
 export default class PostgresMetaConfig {
@@ -15,7 +15,7 @@ export default class PostgresMetaConfig {
     limit?: number
     offset?: number
   } = {}): Promise<PostgresMetaResult<PostgresConfig[]>> {
-    let sql = CONFIG_SQL({ limit, offset })
+    const sql = CONFIG_SQL({ limit, offset })
     return await this.query(sql)
   }
 }

src/lib/PostgresMetaExtensions.ts

@@ -1,6 +1,6 @@
 import { ident, literal } from 'pg-format'
 import { PostgresMetaResult, PostgresExtension } from './types.js'
-import { EXTENSIONS_SQL } from './sql/index.js'
+import { EXTENSIONS_SQL } from './sql/extensions.sql.js'
 import { filterByValue } from './helpers.js'
 
 export default class PostgresMetaExtensions {

src/lib/PostgresMetaForeignTables.ts

@@ -37,14 +37,8 @@ export default class PostgresMetaForeignTables {
     offset?: number
     includeColumns?: boolean
   } = {}): Promise<PostgresMetaResult<PostgresForeignTable[]>> {
-    const filter = filterByList(includedSchemas, excludedSchemas)
-    let sql = generateEnrichedForeignTablesSql({ includeColumns, schemaFilter: filter })
-    if (limit) {
-      sql += ` limit ${limit}`
-    }
-    if (offset) {
-      sql += ` offset ${offset}`
-    }
+    const schemaFilter = filterByList(includedSchemas, excludedSchemas)
+    const sql = generateEnrichedForeignTablesSql({ includeColumns, schemaFilter, limit, offset })
     return await this.query(sql)
   }
 
@@ -65,7 +59,6 @@ export default class PostgresMetaForeignTables {
     name?: string
     schema?: string
   }): Promise<PostgresMetaResult<PostgresForeignTable>> {
-    const schemaFilter = schema ? filterByList([schema], []) : undefined
     if (id) {
       const idsFilter = filterByValue([`${id}`])
       const sql = generateEnrichedForeignTablesSql({
@@ -84,7 +77,6 @@ export default class PostgresMetaForeignTables {
       const nameFilter = filterByValue([`${schema}.${name}`])
       const sql = generateEnrichedForeignTablesSql({
         includeColumns: true,
-        schemaFilter,
         tableIdentifierFilter: nameFilter,
       })
       const { data, error } = await this.query(sql)
@@ -109,13 +101,17 @@ const generateEnrichedForeignTablesSql = ({
   schemaFilter,
   idsFilter,
   tableIdentifierFilter,
+  limit,
+  offset,
 }: {
   includeColumns: boolean
   schemaFilter?: string
   idsFilter?: string
   tableIdentifierFilter?: string
+  limit?: number
+  offset?: number
 }) => `
-with foreign_tables as (${FOREIGN_TABLES_SQL({ schemaFilter, tableIdentifierFilter })})
+with foreign_tables as (${FOREIGN_TABLES_SQL({ schemaFilter, tableIdentifierFilter, limit, offset })})
   ${includeColumns ? `, columns as (${COLUMNS_SQL({ schemaFilter, tableIdentifierFilter, tableIdFilter: idsFilter })})` : ''}
 select
   *

src/lib/PostgresMetaFunctions.ts

@@ -2,7 +2,7 @@ import { ident, literal } from 'pg-format'
 import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
 import { filterByList, filterByValue } from './helpers.js'
 import { PostgresMetaResult, PostgresFunction, PostgresFunctionCreate } from './types.js'
-import { FUNCTIONS_SQL } from './sql/index.js'
+import { FUNCTIONS_SQL } from './sql/functions.sql.js'
 
 export default class PostgresMetaFunctions {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -29,7 +29,7 @@ export default class PostgresMetaFunctions {
       excludedSchemas,
       !includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
     )
-    let sql = FUNCTIONS_SQL({ schemaFilter, limit, offset })
+    const sql = FUNCTIONS_SQL({ schemaFilter, limit, offset })
     return await this.query(sql)
   }
 
@@ -56,7 +56,7 @@ export default class PostgresMetaFunctions {
   }): Promise<PostgresMetaResult<PostgresFunction>> {
     const schemaFilter = schema ? filterByList([schema], []) : undefined
     if (id) {
-      const idsFilter = filterByValue([`${id}`])
+      const idsFilter = filterByValue([id])
       const sql = FUNCTIONS_SQL({ idsFilter })
       const { data, error } = await this.query(sql)
       if (error) {

src/lib/PostgresMetaIndexes.ts

@@ -1,8 +1,7 @@
-import { literal } from 'pg-format'
 import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
-import { filterByList } from './helpers.js'
+import { filterByList, filterByValue } from './helpers.js'
 import { PostgresMetaResult, PostgresIndex } from './types.js'
-import { INDEXES_SQL } from './sql/index.js'
+import { INDEXES_SQL } from './sql/indexes.sql.js'
 
 export default class PostgresMetaIndexes {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -29,7 +28,7 @@ export default class PostgresMetaIndexes {
       excludedSchemas,
       !includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
     )
-    let sql = INDEXES_SQL({ schemaFilter, limit, offset })
+    const sql = INDEXES_SQL({ schemaFilter, limit, offset })
     return await this.query(sql)
   }
 
@@ -45,13 +44,13 @@ export default class PostgresMetaIndexes {
   }): Promise<PostgresMetaResult<PostgresIndex>>
   async retrieve({
     id,
-    args = [],
   }: {
     id?: number
     args?: string[]
   }): Promise<PostgresMetaResult<PostgresIndex>> {
     if (id) {
-      const sql = `${INDEXES_SQL({})} WHERE id = ${literal(id)};`
+      const idsFilter = filterByValue([id])
+      const sql = INDEXES_SQL({ idsFilter })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }

src/lib/PostgresMetaMaterializedViews.ts

@@ -1,7 +1,7 @@
-import { literal } from 'pg-format'
-import { filterByList, coalesceRowsToArray } from './helpers.js'
+import { filterByList, coalesceRowsToArray, filterByValue } from './helpers.js'
 import { PostgresMetaResult, PostgresMaterializedView } from './types.js'
-import { MATERIALIZED_VIEWS_SQL, COLUMNS_SQL } from './sql/index.js'
+import { MATERIALIZED_VIEWS_SQL } from './sql/materialized_views.sql.js'
+import { COLUMNS_SQL } from './sql/columns.sql.js'
 
 export default class PostgresMetaMaterializedViews {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -45,12 +45,12 @@ export default class PostgresMetaMaterializedViews {
     name?: string
     schema?: string
   }): Promise<PostgresMetaResult<PostgresMaterializedView>> {
-    const schemaFilter = schema ? filterByList([schema], []) : undefined
     if (id) {
-      const sql = `${generateEnrichedMaterializedViewsSql({
+      const idsFilter = filterByValue([id])
+      const sql = generateEnrichedMaterializedViewsSql({
         includeColumns: true,
-        schemaFilter,
-      })} where materialized_views.id = ${literal(id)};`
+        idsFilter,
+      })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }
@@ -60,12 +60,11 @@ export default class PostgresMetaMaterializedViews {
         return { data: data[0], error }
       }
     } else if (name) {
-      const sql = `${generateEnrichedMaterializedViewsSql({
+      const materializedViewIdentifierFilter = filterByValue([`${schema}.${name}`])
+      const sql = generateEnrichedMaterializedViewsSql({
         includeColumns: true,
-        schemaFilter,
-      })} where materialized_views.name = ${literal(
-        name
-      )} and materialized_views.schema = ${literal(schema)};`
+        materializedViewIdentifierFilter,
+      })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }
@@ -86,16 +85,20 @@ export default class PostgresMetaMaterializedViews {
 const generateEnrichedMaterializedViewsSql = ({
   includeColumns,
   schemaFilter,
+  materializedViewIdentifierFilter,
+  idsFilter,
   limit,
   offset,
 }: {
   includeColumns: boolean
   schemaFilter?: string
+  materializedViewIdentifierFilter?: string
+  idsFilter?: string
   limit?: number
   offset?: number
 }) => `
-with materialized_views as (${MATERIALIZED_VIEWS_SQL({ schemaFilter, limit, offset })})
-  ${includeColumns ? `, columns as (${COLUMNS_SQL({ schemaFilter, limit, offset })}` : ''}
+with materialized_views as (${MATERIALIZED_VIEWS_SQL({ schemaFilter, limit, offset, materializedViewIdentifierFilter, idsFilter })})
+  ${includeColumns ? `, columns as (${COLUMNS_SQL({ schemaFilter, limit, offset, tableIdentifierFilter: materializedViewIdentifierFilter, tableIdFilter: idsFilter })})` : ''}
 select
   *
   ${

src/lib/PostgresMetaPolicies.ts

@@ -2,7 +2,7 @@ import { ident } from 'pg-format'
 import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
 import { filterByList, filterByValue } from './helpers.js'
 import { PostgresMetaResult, PostgresPolicy } from './types.js'
-import { POLICIES_SQL } from './sql/index.js'
+import { POLICIES_SQL } from './sql/policies.sql.js'
 
 export default class PostgresMetaPolicies {
   query: (sql: string) => Promise<PostgresMetaResult<any>>

src/lib/PostgresMetaPublications.ts

@@ -1,6 +1,7 @@
 import { ident, literal } from 'pg-format'
 import { PostgresMetaResult, PostgresPublication } from './types.js'
-import { PUBLICATIONS_SQL } from './sql/index.js'
+import { PUBLICATIONS_SQL } from './sql/publications.sql.js'
+import { filterByValue } from './helpers.js'
 
 export default class PostgresMetaPublications {
   query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -30,7 +31,8 @@ export default class PostgresMetaPublications {
     name?: string
   }): Promise<PostgresMetaResult<PostgresPublication>> {
     if (id) {
-      const sql = `${PUBLICATIONS_SQL({})} WHERE p.oid = ${literal(id)};`
+      const idsFilter = filterByValue([id])
+      const sql = PUBLICATIONS_SQL({ idsFilter })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }
@@ -40,7 +42,8 @@ export default class PostgresMetaPublications {
         return { data: data[0], error }
       }
     } else if (name) {
-      const sql = `${PUBLICATIONS_SQL({})} WHERE p.pubname = ${literal(name)};`
+      const nameFilter = filterByValue([name])
+      const sql = PUBLICATIONS_SQL({ nameFilter })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }

src/lib/PostgresMetaRelationships.ts

@@ -24,14 +24,14 @@ export default class PostgresMetaRelationships {
     includedSchemas?: string[]
     excludedSchemas?: string[]
   } = {}): Promise<PostgresMetaResult<PostgresRelationship[]>> {
-    const filter = filterByList(
+    const schemaFilter = filterByList(
       includedSchemas,
       excludedSchemas,
       !includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
     )
     let allTableM2oAndO2oRelationships: PostgresRelationship[]
     {
-      const sql = TABLE_RELATIONSHIPS_SQL(filter)
+      const sql = TABLE_RELATIONSHIPS_SQL({ schemaFilter })
       const { data, error } = (await this.query(sql)) as PostgresMetaResult<PostgresRelationship[]>
       if (error) {
         return { data: null, error }
@@ -59,8 +59,9 @@ export default class PostgresMetaRelationships {
         column_dependencies: ColDep[]
       }
 
+      const viewsKeyDependenciesSql = VIEWS_KEY_DEPENDENCIES_SQL({ schemaFilter })
       const { data: viewsKeyDependencies, error } = (await this.query(
-        VIEWS_KEY_DEPENDENCIES_SQL(filter)
+        viewsKeyDependenciesSql
       )) as PostgresMetaResult<KeyDep[]>
       if (error) {
         return { data: null, error }

src/lib/PostgresMetaRoles.ts

@@ -1,11 +1,12 @@
 import { ident, literal } from 'pg-format'
-import { ROLES_SQL } from './sql/index.js'
+import { ROLES_SQL } from './sql/roles.sql.js'
 import {
   PostgresMetaResult,
   PostgresRole,
   PostgresRoleCreate,
   PostgresRoleUpdate,
 } from './types.js'
+import { filterByValue } from './helpers.js'
 export function changeRoleConfig2Object(config: string[]) {
   if (!config) {
     return null
@@ -32,26 +33,7 @@ export default class PostgresMetaRoles {
     limit?: number
     offset?: number
   } = {}): Promise<PostgresMetaResult<PostgresRole[]>> {
-    let sql = `
-WITH
-  roles AS (${ROLES_SQL({ limit, offset })})
-SELECT
-  *
-FROM
-  roles
-WHERE
-  true`
-    if (!includeDefaultRoles) {
-      // All default/predefined roles start with pg_: https://www.postgresql.org/docs/15/predefined-roles.html
-      // The pg_ prefix is also reserved:
-      //
-      // ```
-      // postgres=# create role pg_mytmp;
-      // ERROR:  role name "pg_mytmp" is reserved
-      // DETAIL:  Role names starting with "pg_" are reserved.
-      // ```
-      sql += ` AND NOT pg_catalog.starts_with(name, 'pg_')`
-    }
+    const sql = ROLES_SQL({ limit, offset, includeDefaultRoles })
     const result = await this.query(sql)
     if (result.data) {
       result.data = result.data.map((role: any) => {
@@ -72,7 +54,8 @@ WHERE
     name?: string
   }): Promise<PostgresMetaResult<PostgresRole>> {
     if (id) {
-      const sql = `${ROLES_SQL({})} WHERE oid = ${literal(id)};`
+      const idsFilter = filterByValue([id])
+      const sql = ROLES_SQL({ idsFilter })
       const { data, error } = await this.query(sql)
 
       if (error) {
@@ -84,7 +67,8 @@ WHERE
         return { data: data[0], error }
       }
     } else if (name) {
-      const sql = `${ROLES_SQL({})} WHERE rolname = ${literal(name)};`
+      const nameFilter = filterByValue([name])
+      const sql = ROLES_SQL({ nameFilter })
       const { data, error } = await this.query(sql)
       if (error) {
         return { data, error }