feat: option to exclude materialized views (#392)

Author: dmfay

docs/connecting.md

@@ -86,12 +86,15 @@ massive(connectionInfo, {
   // never load functions on the blacklist
   functionBlacklist: 'authorizeUser,disableUser',
 
+  // streamline function return values: a function with a scalar
+  // value will return just the scalar instead of an array, etc.
+  enhancedFunctions: true,
+
   // don't load database functions at all
   excludeFunctions: true,
 
-  // streamline function return values: a function with a scalar
-  // value will return just the scalar instead of an array, etc.
-  enhancedFunctions: true
+  // don't load materialized views (required for Postgres < 9.3)
+  excludeMatViews: true
 }).then(instance => {...});
 ```
 

lib/database.js

@@ -29,8 +29,9 @@ const getFilterString = require('./util/get-filter-string');
  * @param {Array|String} loader.exceptions - Table/view blacklist exceptions.
  * @param {Array|String} loader.functionWhitelist - Function name whitelist.
  * @param {Array|String} loader.functionBlacklist - Function name blacklist.
- * @param {Boolean} loader.excludeFunctions - Ignore functions entirely.
  * @param {Boolean} loader.enhancedFunctions - Streamline function return values.
+ * @param {Boolean} loader.excludeFunctions - Ignore functions entirely.
+ * @param {Boolean} loader.excludeMatViews - Ignore materialized views.
  * @param {Object} [driverConfig] - A pg-promise configuration object.
  */
 const Database = function (connection = {}, loader = {}, driverConfig = {}) {

lib/loader/functions.js

@@ -5,9 +5,7 @@ const _ = require('lodash');
 exports = module.exports = function (instance, config) {
   if (config.excludeFunctions) { return []; }
 
-  const parameters = [config.functionBlacklist, config.functionWhitelist];
-
-  return instance.query(config.queryFiles['functions.sql'], parameters).then(fns => {
+  return instance.query(config.queryFiles['functions.sql'], config).then(fns => {
     return fns.map(fn => {
       const params = _.times(fn.paramCount, i => `$${i + 1}`).join(',');
 

lib/loader/tables.js

@@ -1,9 +1,5 @@
 'use strict';
 
 exports = module.exports = function (instance, config) {
-  if (config.whitelist) {
-    return instance.query(config.queryFiles['tables-whitelist.sql'], [config.whitelist]);
-  }
-
-  return instance.query(config.queryFiles['tables.sql'], [config.allowedSchemas, config.blacklist, config.exceptions]);
+  return instance.query(config.queryFiles['tables.sql'], config);
 };

lib/loader/views.js

@@ -1,9 +1,7 @@
 'use strict';
 
 exports = module.exports = function (instance, config) {
-  if (config.whitelist) {
-    return instance.query(config.queryFiles['views-whitelist.sql'], [config.whitelist]);
-  }
+  const file = config.excludeMatViews ? 'views-legacy.sql' : 'views.sql';
 
-  return instance.query(config.queryFiles['views.sql'], [config.allowedSchemas, config.blacklist, config.exceptions]);
+  return instance.query(config.queryFiles[file], config);
 };

lib/scripts/functions.sql

@@ -1,5 +1,11 @@
--- Include and/or exclude functions with a name matching the string pattern passed in (i.e. "pg_%")
--- inclusion/exclusion is schema - aspecific, no schema assumes 'public'
+-- Load non-system functions. The whitelist and blacklist are overlapped such
+-- that more specific blacklist entries may override more general whitelist
+-- entries.
+--
+-- Parameters:
+-- functionWhitelist: array or comma-delimited string of LIKE conditions.
+-- functionBlacklist: array or comma-delimited string of LIKE conditions.
+
 SELECT DISTINCT
   n.nspname AS schema,
   (NOT p.proretset) AS "singleRow",
@@ -11,13 +17,15 @@ JOIN pg_namespace n ON p.pronamespace = n.oid
 JOIN pg_type t on p.prorettype = t.oid
 WHERE n.nspname NOT IN ('pg_catalog', 'information_schema')
   AND n.nspname NOT LIKE 'pgp%'
-  AND (-- blacklist functions using LIKE by fully-qualified name (no schema assumes public):
-    CASE WHEN $1 = '' THEN 1=1
-    ELSE replace((n.nspname || '.'|| p.proname), 'public.', '') NOT LIKE ALL(string_to_array(replace($1, ' ', ''), ','))
-    END
-  ) AND (-- whitelist functions using LIKE by fully-qualified name (no schema assumes public):
-    CASE WHEN $2 = '' THEN 1=1
-    ELSE replace((n.nspname || '.'|| p.proname), 'public.', '') LIKE ANY(string_to_array(replace($2, ' ', ''), ','))
-    END
+  AND (
+    -- blacklist functions using LIKE by fully-qualified name (no schema assumes public):
+    $(functionBlacklist) = ''
+    OR
+    replace((n.nspname || '.'|| p.proname), 'public.', '') NOT LIKE ALL(string_to_array(replace($(functionBlacklist), ' ', ''), ','))
+  ) AND (
+    -- whitelist functions using LIKE by fully-qualified name (no schema assumes public):
+    $(functionWhitelist) = ''
+    OR
+    replace((n.nspname || '.'|| p.proname), 'public.', '') LIKE ANY(string_to_array(replace($(functionWhitelist), ' ', ''), ','))
   )
 ORDER BY n.nspname, p.proname;

lib/scripts/tables-whitelist.sql

@@ -1,5 +1,15 @@
--- REQUIRES THREE ARGUMENTS:
--- $1, $2, $2 all must be empty string, or comma-delimited string, or array of string:
+-- Load tables.
+--
+-- Parameters:
+-- whitelist: array or comma-delimited string of LIKE conditions applied to
+--   tables. If specified, overrides all other parameters.
+-- allowedSchemas: array or comma-delimited string of LIKE conditions applied
+--   to schemas.
+-- blacklist: array or comma-delimited string of LIKE conditions applied
+--   negatively to tables.
+-- exceptions: array or comma-delimited string of LIKE conditions which
+--   override blacklisted tables.
+
 SELECT * FROM (
   SELECT tc.table_schema AS schema, tc.table_name AS name, NULL AS parent, kc.column_name AS pk, TRUE AS is_insertable_into
   FROM information_schema.table_constraints tc
@@ -23,19 +33,23 @@ SELECT * FROM (
   SELECT t.table_schema AS schema, t.table_name AS name, NULL AS parent, NULL AS pk, CASE t.is_insertable_into WHEN 'YES' THEN TRUE ELSE FALSE END AS is_insertable_into
   FROM information_schema.tables t
   WHERE table_type = 'FOREIGN TABLE'
-) tables WHERE ((
+) tables
+WHERE CASE
+  WHEN $(whitelist) <> '' THEN
+    -- whitelist specific tables, with fully-qualified name (no schema assumes public).
+    replace((tables.schema || '.'|| tables.name), 'public.', '') LIKE ANY(string_to_array(replace($(whitelist), ' ', ''), ','))
+  WHEN $(allowedSchemas) <> '' OR $(blacklist) <> '' THEN ((
+    $(allowedSchemas) = ''
+    OR
     -- allow specific schemas (none or '' assumes all):
-    CASE WHEN $1 ='' THEN 1=1
-    ELSE schema = ANY(string_to_array(replace($1, ' ', ''), ','))
-    END
+    schema = ANY(string_to_array(replace($(allowedSchemas), ' ', ''), ','))
   ) AND (
+    $(blacklist) = ''
+    OR
     -- blacklist tables using LIKE by fully-qualified name (no schema assumes public):
-    CASE WHEN $2 = '' THEN 1=1
-    ELSE replace((schema || '.'|| name), 'public.', '') NOT LIKE ALL(string_to_array(replace($2, ' ', ''), ','))
-    END
-  )
-) OR (-- make exceptions for specific tables, with fully-qualified name or wildcard pattern (no schema assumes public).
-  CASE WHEN $3 = '' THEN 1=0
-  ELSE replace((schema || '.'|| name), 'public.', '') LIKE ANY(string_to_array(replace($3, ' ', ''), ','))
-  END
-);
+    replace((schema || '.'|| name), 'public.', '') NOT LIKE ALL(string_to_array(replace($(blacklist), ' ', ''), ','))
+  )) OR
+    -- make exceptions for specific tables, with fully-qualified name or wildcard pattern (no schema assumes public).
+    replace((schema || '.'|| name), 'public.', '') LIKE ANY(string_to_array(replace($(exceptions), ' ', ''), ','))
+  ELSE TRUE
+END;

lib/scripts/tables.sql

@@ -0,0 +1,36 @@
+-- Load views (excluding materialized views).
+--
+-- Parameters:
+-- whitelist: array or comma-delimited string of LIKE conditions applied to
+--   tables. If specified, overrides all other parameters.
+-- allowedSchemas: array or comma-delimited string of LIKE conditions applied
+--   to schemas.
+-- blacklist: array or comma-delimited string of LIKE conditions applied
+--   negatively to tables.
+-- exceptions: array or comma-delimited string of LIKE conditions which
+--   override blacklisted tables.
+
+SELECT * FROM (
+  SELECT schemaname AS schema, viewname AS name
+  FROM pg_views
+  WHERE schemaname <> 'pg_catalog' AND schemaname <> 'information_schema'
+) views
+WHERE CASE
+  WHEN $(whitelist) <> '' THEN
+    -- whitelist specific tables, with fully-qualified name (no schema assumes public).
+    replace((views.schema || '.'|| views.name), 'public.', '') LIKE ANY(string_to_array(replace($(whitelist), ' ', ''), ','))
+  WHEN $(allowedSchemas) <> '' OR $(blacklist) <> '' THEN ((
+    $(allowedSchemas) = ''
+    OR
+    -- allow specific schemas (none or '' assumes all):
+    schema = ANY(string_to_array(replace($(allowedSchemas), ' ', ''), ','))
+  ) AND (
+    $(blacklist) = ''
+    OR
+    -- blacklist tables using LIKE by fully-qualified name (no schema assumes public):
+    replace((schema || '.'|| name), 'public.', '') NOT LIKE ALL(string_to_array(replace($(blacklist), ' ', ''), ','))
+  )) OR
+    -- make exceptions for specific tables, with fully-qualified name or wildcard pattern (no schema assumes public).
+    replace((schema || '.'|| name), 'public.', '') LIKE ANY(string_to_array(replace($(exceptions), ' ', ''), ','))
+  ELSE TRUE
+END;