Skip to content

Commit 56a1f74

Browse files
avalleteavallete
committed
wip: optimize queries
1 parent 4205b26 commit 56a1f74

File tree

11 files changed

+237
-21
lines changed

11 files changed

+237
-21
lines changed

‎src/lib/PostgresMetaColumns.ts‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { ident, literal } from 'pg-format'
22
import PostgresMetaTables from './PostgresMetaTables.js'
33
import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
4-
import { columnsSql } from './sql/index.js'
54
import { PostgresMetaResult, PostgresColumn } from './types.js'
65
import { filterByList } from './helpers.js'
6+
import { COLUMNS_SQL } from './sql/columns.sql.js'
77

88
export default class PostgresMetaColumns {
99
query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -29,23 +29,20 @@ export default class PostgresMetaColumns {
2929
limit?: number
3030
offset?: number
3131
} = {}): Promise<PostgresMetaResult<PostgresColumn[]>> {
32+
const schemaFilter = filterByList(
33+
includedSchemas,
34+
excludedSchemas,
35+
!includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
36+
)
3237
let sql = `
3338
WITH
34-
columns AS (${columnsSql})
39+
columns AS (${COLUMNS_SQL(schemaFilter)})
3540
SELECT
3641
*
3742
FROM
3843
columns
3944
WHERE
4045
true`
41-
const filter = filterByList(
42-
includedSchemas,
43-
excludedSchemas,
44-
!includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
45-
)
46-
if (filter) {
47-
sql += ` AND schema ${filter}`
48-
}
4946
if (tableId !== undefined) {
5047
sql += ` AND table_id = ${literal(tableId)}`
5148
}
@@ -79,14 +76,15 @@ WHERE
7976
table?: string
8077
schema?: string
8178
}): Promise<PostgresMetaResult<PostgresColumn>> {
79+
const schemaFilter = schema ? filterByList([schema], []) : undefined
8280
if (id) {
8381
const regexp = /^(\d+)\.(\d+)$/
8482
if (!regexp.test(id)) {
8583
return { data: null, error: { message: 'Invalid format for column ID' } }
8684
}
8785
const matches = id.match(regexp) as RegExpMatchArray
8886
const [tableId, ordinalPos] = matches.slice(1).map(Number)
89-
const sql = `${columnsSql} AND c.oid = ${tableId} AND a.attnum = ${ordinalPos};`
87+
const sql = `${COLUMNS_SQL(schemaFilter)} AND c.oid = ${tableId} AND a.attnum = ${ordinalPos};`
9088
const { data, error } = await this.query(sql)
9189
if (error) {
9290
return { data, error }
@@ -96,9 +94,9 @@ WHERE
9694
return { data: data[0], error }
9795
}
9896
} else if (name && table) {
99-
const sql = `${columnsSql} AND a.attname = ${literal(name)} AND c.relname = ${literal(
97+
const sql = `${COLUMNS_SQL(schemaFilter)} AND a.attname = ${literal(name)} AND c.relname = ${literal(
10098
table
101-
)} AND nc.nspname = ${literal(schema)};`
99+
)};`
102100
const { data, error } = await this.query(sql)
103101
if (error) {
104102
return { data, error }

‎src/lib/PostgresMetaTables.ts‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
import { ident, literal } from 'pg-format'
22
import { DEFAULT_SYSTEM_SCHEMAS } from './constants.js'
33
import { coalesceRowsToArray, filterByList } from './helpers.js'
4-
import { columnsSql, tablesSql } from './sql/index.js'
54
import {
65
PostgresMetaResult,
76
PostgresTable,
87
PostgresTableCreate,
98
PostgresTableUpdate,
109
} from './types.js'
10+
import { TABLES_SQL } from './sql/table.sql.js'
11+
import { COLUMNS_SQL } from './sql/columns.sql.js'
1112

1213
export default class PostgresMetaTables {
1314
query: (sql: string) => Promise<PostgresMetaResult<any>>
@@ -47,12 +48,12 @@ export default class PostgresMetaTables {
4748
offset?: number
4849
includeColumns?: boolean
4950
} = {}): Promise<PostgresMetaResult<PostgresTable[]>> {
50-
let sql = generateEnrichedTablesSql({ includeColumns })
5151
const filter = filterByList(
5252
includedSchemas,
5353
excludedSchemas,
5454
!includeSystemSchemas ? DEFAULT_SYSTEM_SCHEMAS : undefined
5555
)
56+
let sql = generateEnrichedTablesSql({ includeColumns, schemaFilter: filter })
5657
if (filter) {
5758
sql += ` where schema ${filter}`
5859
}
@@ -62,6 +63,7 @@ export default class PostgresMetaTables {
6263
if (offset) {
6364
sql += ` offset ${offset}`
6465
}
66+
console.log('sql tables: ', sql)
6567
return await this.query(sql)
6668
}
6769

@@ -82,8 +84,10 @@ export default class PostgresMetaTables {
8284
name?: string
8385
schema?: string
8486
}): Promise<PostgresMetaResult<PostgresTable>> {
87+
const schemaFilter = schema ? filterByList([schema], []) : undefined
8588
if (id) {
8689
const sql = `${generateEnrichedTablesSql({
90+
schemaFilter,
8791
includeColumns: true,
8892
})} where tables.id = ${literal(id)};`
8993
const { data, error } = await this.query(sql)
@@ -96,6 +100,7 @@ export default class PostgresMetaTables {
96100
}
97101
} else if (name) {
98102
const sql = `${generateEnrichedTablesSql({
103+
schemaFilter,
99104
includeColumns: true,
100105
})} where tables.name = ${literal(name)} and tables.schema = ${literal(schema)};`
101106
const { data, error } = await this.query(sql)
@@ -247,9 +252,15 @@ COMMIT;`
247252
}
248253
}
249254

250-
const generateEnrichedTablesSql = ({ includeColumns }: { includeColumns: boolean }) => `
251-
with tables as (${tablesSql})
252-
${includeColumns ? `, columns as (${columnsSql})` : ''}
255+
const generateEnrichedTablesSql = ({
256+
includeColumns,
257+
schemaFilter,
258+
}: {
259+
includeColumns: boolean
260+
schemaFilter?: string
261+
}) => `
262+
with tables as (${TABLES_SQL(schemaFilter)})
263+
${includeColumns ? `, columns as (${COLUMNS_SQL(schemaFilter)})` : ''}
253264
select
254265
*
255266
${includeColumns ? `, ${coalesceRowsToArray('columns', 'columns.table_id = tables.id')}` : ''}

‎src/lib/generators.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ export async function getGeneratorMetadata(
3131
excludedSchemas: [],
3232
}
3333
): Promise<PostgresMetaResult<GeneratorMetadata>> {
34+
const start = Date.now()
35+
console.log('getGeneratorMetadata start: ')
3436
const includedSchemas = filters.includedSchemas ?? []
3537
const excludedSchemas = filters.excludedSchemas ?? []
3638

@@ -48,6 +50,8 @@ export async function getGeneratorMetadata(
4850
return { data: null, error: tablesError }
4951
}
5052

53+
const startForeignTables = Date.now()
54+
console.log('getGeneratorMetadata foreignTables start: ', startForeignTables)
5155
const { data: foreignTables, error: foreignTablesError } = await pgMeta.foreignTables.list({
5256
includedSchemas: includedSchemas.length > 0 ? includedSchemas : undefined,
5357
excludedSchemas,
@@ -56,7 +60,12 @@ export async function getGeneratorMetadata(
5660
if (foreignTablesError) {
5761
return { data: null, error: foreignTablesError }
5862
}
63+
const endForeignTables = Date.now()
64+
console.log('getGeneratorMetadata foreignTables end: ', endForeignTables)
65+
console.log('elapsedForeignTables: ', endForeignTables - startForeignTables)
5966

67+
const startViews = Date.now()
68+
console.log('getGeneratorMetadata views start: ', startViews)
6069
const { data: views, error: viewsError } = await pgMeta.views.list({
6170
includedSchemas: includedSchemas.length > 0 ? includedSchemas : undefined,
6271
excludedSchemas,
@@ -75,28 +84,48 @@ export async function getGeneratorMetadata(
7584
if (materializedViewsError) {
7685
return { data: null, error: materializedViewsError }
7786
}
87+
const endViews = Date.now()
88+
console.log('getGeneratorMetadata views end: ', endViews)
89+
console.log('elapsedViews: ', endViews - startViews)
7890

91+
const startColumns = Date.now()
92+
console.log('getGeneratorMetadata columns start: ', startColumns)
7993
const { data: columns, error: columnsError } = await pgMeta.columns.list({
8094
includedSchemas: includedSchemas.length > 0 ? includedSchemas : undefined,
8195
excludedSchemas,
8296
})
8397
if (columnsError) {
8498
return { data: null, error: columnsError }
8599
}
100+
const endColumns = Date.now()
101+
console.log('getGeneratorMetadata columns end: ', endColumns)
102+
console.log('elapsedColumns: ', endColumns - startColumns)
86103

104+
const startRelationships = Date.now()
105+
console.log('getGeneratorMetadata relationships start: ', startRelationships)
87106
const { data: relationships, error: relationshipsError } = await pgMeta.relationships.list()
88107
if (relationshipsError) {
89108
return { data: null, error: relationshipsError }
90109
}
110+
const endRelationships = Date.now()
111+
console.log('getGeneratorMetadata relationships end: ', endRelationships)
112+
console.log('elapsedRelationships: ', endRelationships - startRelationships)
91113

114+
const startFunctions = Date.now()
115+
console.log('getGeneratorMetadata functions start: ', startFunctions)
92116
const { data: functions, error: functionsError } = await pgMeta.functions.list({
93117
includedSchemas: includedSchemas.length > 0 ? includedSchemas : undefined,
94118
excludedSchemas,
95119
})
96120
if (functionsError) {
97121
return { data: null, error: functionsError }
98122
}
123+
const endFunctions = Date.now()
124+
console.log('getGeneratorMetadata functions end: ', endFunctions)
125+
console.log('elapsedFunctions: ', endFunctions - startFunctions)
99126

127+
const startTypes = Date.now()
128+
console.log('getGeneratorMetadata types start: ', startTypes)
100129
const { data: types, error: typesError } = await pgMeta.types.list({
101130
includeTableTypes: true,
102131
includeArrayTypes: true,
@@ -105,9 +134,16 @@ export async function getGeneratorMetadata(
105134
if (typesError) {
106135
return { data: null, error: typesError }
107136
}
137+
const endTypes = Date.now()
138+
console.log('getGeneratorMetadata types end: ', endTypes)
139+
console.log('elapsedTypes: ', endTypes - startTypes)
108140

109141
await pgMeta.end()
110142

143+
const end = Date.now()
144+
console.log('getGeneratorMetadata end: ', end)
145+
console.log('elapsed: ', end - start)
146+
111147
return {
112148
data: {
113149
schemas: schemas.filter(

‎src/lib/sql/columns.sql‎ renamed to ‎src/lib/sql/columns.sql.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
export const COLUMNS_SQL = (schemaFilter?: string) => /* SQL */ `
12
-- Adapted from information_schema.columns
23
34
SELECT
@@ -97,6 +98,7 @@ FROM
9798
ORDER BY table_id, ordinal_position, oid asc
9899
) AS check_constraints ON check_constraints.table_id = c.oid AND check_constraints.ordinal_position = a.attnum
99100
WHERE
101+
${schemaFilter ? `nc.nspname ${schemaFilter} AND` : ''}
100102
NOT pg_is_other_temp_schema(nc.oid)
101103
AND a.attnum > 0
102104
AND NOT a.attisdropped
@@ -109,3 +111,4 @@ WHERE
109111
'SELECT, INSERT, UPDATE, REFERENCES'
110112
)
111113
)
114+
`

‎src/lib/sql/foreign_tables.sql.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
export const FOREIGN_TABLES_SQL = (schemaFilter?: string) => /* SQL */ `
2+
SELECT
3+
c.oid :: int8 AS id,
4+
n.nspname AS schema,
5+
c.relname AS name,
6+
obj_description(c.oid) AS comment
7+
FROM
8+
pg_class c
9+
JOIN pg_namespace n ON n.oid = c.relnamespace
10+
WHERE
11+
${schemaFilter ? `n.nspname ${schemaFilter} AND` : ''}
12+
c.relkind = 'f'
13+
`

‎src/lib/sql/functions.sql.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
export const FUNCTIONS_SQL = (schemaFilter?: string) => /* SQL */ `
2+
-- CTE with sane arg_modes, arg_names, and arg_types.
3+
-- All three are always of the same length.
4+
-- All three include all args, including OUT and TABLE args.
5+
with functions as (
6+
select
7+
*,
8+
-- proargmodes is null when all arg modes are IN
9+
coalesce(
10+
p.proargmodes,
11+
array_fill('i'::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))])
12+
) as arg_modes,
13+
-- proargnames is null when all args are unnamed
14+
coalesce(
15+
p.proargnames,
16+
array_fill(''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))])
17+
) as arg_names,
18+
-- proallargtypes is null when all arg modes are IN
19+
coalesce(p.proallargtypes, p.proargtypes) as arg_types,
20+
array_cat(
21+
array_fill(false, array[pronargs - pronargdefaults]),
22+
array_fill(true, array[pronargdefaults])) as arg_has_defaults
23+
from
24+
pg_proc as p
25+
${schemaFilter ? `join pg_namespace n on p.pronamespace = n.oid` : ''}
26+
where
27+
${schemaFilter ? `n.nspname ${schemaFilter} AND` : ''}
28+
p.prokind = 'f'
29+
)
30+
select
31+
f.oid::int8 as id,
32+
n.nspname as schema,
33+
f.proname as name,
34+
l.lanname as language,
35+
case
36+
when l.lanname = 'internal' then ''
37+
else f.prosrc
38+
end as definition,
39+
case
40+
when l.lanname = 'internal' then f.prosrc
41+
else pg_get_functiondef(f.oid)
42+
end as complete_statement,
43+
coalesce(f_args.args, '[]') as args,
44+
pg_get_function_arguments(f.oid) as argument_types,
45+
pg_get_function_identity_arguments(f.oid) as identity_argument_types,
46+
f.prorettype::int8 as return_type_id,
47+
pg_get_function_result(f.oid) as return_type,
48+
nullif(rt.typrelid::int8, 0) as return_type_relation_id,
49+
f.proretset as is_set_returning_function,
50+
case
51+
when f.provolatile = 'i' then 'IMMUTABLE'
52+
when f.provolatile = 's' then 'STABLE'
53+
when f.provolatile = 'v' then 'VOLATILE'
54+
end as behavior,
55+
f.prosecdef as security_definer,
56+
f_config.config_params as config_params
57+
from
58+
functions f
59+
left join pg_namespace n on f.pronamespace = n.oid
60+
left join pg_language l on f.prolang = l.oid
61+
left join pg_type rt on rt.oid = f.prorettype
62+
left join (
63+
select
64+
oid,
65+
jsonb_object_agg(param, value) filter (where param is not null) as config_params
66+
from
67+
(
68+
select
69+
oid,
70+
(string_to_array(unnest(proconfig), '='))[1] as param,
71+
(string_to_array(unnest(proconfig), '='))[2] as value
72+
from
73+
functions
74+
) as t
75+
group by
76+
oid
77+
) f_config on f_config.oid = f.oid
78+
left join (
79+
select
80+
oid,
81+
jsonb_agg(jsonb_build_object(
82+
'mode', t2.mode,
83+
'name', name,
84+
'type_id', type_id,
85+
'has_default', has_default
86+
)) as args
87+
from
88+
(
89+
select
90+
oid,
91+
unnest(arg_modes) as mode,
92+
unnest(arg_names) as name,
93+
unnest(arg_types)::int8 as type_id,
94+
unnest(arg_has_defaults) as has_default
95+
from
96+
functions
97+
) as t1,
98+
lateral (
99+
select
100+
case
101+
when t1.mode = 'i' then 'in'
102+
when t1.mode = 'o' then 'out'
103+
when t1.mode = 'b' then 'inout'
104+
when t1.mode = 'v' then 'variadic'
105+
else 'table'
106+
end as mode
107+
) as t2
108+
group by
109+
t1.oid
110+
) f_args on f_args.oid = f.oid
111+
`

0 commit comments

Comments
 (0)