feature: expand values implementation for cosmic file loaders

Author: vsamofal

.nvmrc

@@ -0,0 +1 @@
+v18

lib/interfaces/typed-config-module-options.interface.ts

@@ -1,8 +1,8 @@
 import type { ClassConstructor } from 'class-transformer';
 import type { ValidatorOptions } from 'class-validator';
 
-export type ConfigLoader = () => Record<string, any>;
-export type AsyncConfigLoader = () => Promise<Record<string, any>>;
+export type ConfigLoader = (...args: any) => Record<string, any>;
+export type AsyncConfigLoader = (...args: any) => Promise<Record<string, any>>;
 
 export interface TypedConfigModuleOptions {
   /**

lib/loader/file-loader.ts

@@ -68,44 +68,6 @@ const getSearchOptions = (options: FileLoaderOptions) => {
   };
 };
 
-/**
- * Will fill in some placeholders.
- * @param template - Text with placeholders for `data` properties.
- * @param data - Data to interpolate into `template`.
- *
- * @example
- ```
- placeholderResolver('Hello ${name}', {
-    name: 'John',
-  });
- //=> 'Hello John'
- ```
- */
-const placeholderResolver = (
-  template: string,
-  data: Record<string, any>,
-  disallowUndefinedEnvironmentVariables: boolean,
-): string => {
-  const replace = (placeholder: any, key: string) => {
-    let value = data;
-    for (const property of key.split('.')) {
-      value = value[property];
-    }
-
-    if (!value && disallowUndefinedEnvironmentVariables) {
-      throw new Error(
-        `Environment variable is not set for variable name: '${key}'`,
-      );
-    }
-    return String(value);
-  };
-
-  // The regex tries to match either a number inside `${{ }}` or a valid JS identifier or key path.
-  const braceRegex = /\${(\d+|[a-z$_][\w\-$]*?(?:\.[\w\-$]*?)*?)}/gi;
-
-  return template.replace(braceRegex, replace);
-};
-
 /**
  * File loader loads configuration with `cosmiconfig` from file system.
  *
@@ -126,7 +88,7 @@ export const fileLoader = (
   cosmiconfig = loadPackage('cosmiconfig', 'fileLoader');
 
   const { cosmiconfigSync } = cosmiconfig;
-  return (): Record<string, any> => {
+  return (additionalContext: Record<string, any> = {}): Record<string, any> => {
     const { searchPlaces, searchFrom } = getSearchOptions(options);
     const loaders = {
       '.toml': loadToml,
@@ -136,6 +98,31 @@ export const fileLoader = (
       searchPlaces,
       ...options,
       loaders,
+      transform: (result: Record<string, any>) => {
+        if (options.ignoreEnvironmentVariableSubstitution ?? true) {
+          return result;
+        }
+
+        try {
+          const updatedResult = transformFileLoaderResult(
+            result.config,
+            {
+              // additionalContext must be first so the actual config will override it
+              ...additionalContext,
+              ...result.config,
+              ...process.env,
+            },
+            options.disallowUndefinedEnvironmentVariables ?? true,
+          );
+
+          result.config = updatedResult;
+          return result;
+        } catch (error: any) {
+          // enrich error with options information
+          error.details = options;
+          throw error;
+        }
+      },
     });
     const result = explorer.search(searchFrom);
 
@@ -147,17 +134,132 @@ export const fileLoader = (
       `File-loader has loaded a configuration file from ${result.filepath}`,
     );
 
-    let config = result.config;
+    return result.config;
+  };
+};
+
+function resolveReference(
+  reference: string,
+  currentContext: Record<string, any>,
+): Record<string, any> | string | number | null {
+  const parts = reference.split('.');
+  let value = currentContext;
 
-    if (!(options.ignoreEnvironmentVariableSubstitution ?? true)) {
-      const replacedConfig = placeholderResolver(
-        JSON.stringify(result.config),
-        process.env,
-        options.disallowUndefinedEnvironmentVariables ?? true,
-      );
-      config = JSON.parse(replacedConfig);
+  for (const part of parts) {
+    value = value[part];
+    if (value === undefined) {
+      return null;
     }
+  }
 
-    return config;
-  };
-};
+  return value;
+}
+
+function transformFileLoaderResult(
+  obj: Record<string, any> | string | number,
+  context: Record<string, any>,
+  disallowUndefinedEnvironmentVariables: boolean,
+  visited = new Set<Record<string, any> | string | number>(),
+): Record<string, any> | string | number {
+  if (typeof obj === 'string') {
+    const match = obj.match(/\$\{(.+?)\}/g);
+    if (match) {
+      for (const placeholder of match) {
+        const variable = placeholder.slice(2, -1);
+        let resolvedValue = resolveReference(variable, context);
+
+        if (obj === resolvedValue) {
+          throw new Error(
+            `Circular self reference detected: ${obj} -> ${resolvedValue}`,
+          );
+        }
+
+        if (resolvedValue !== null) {
+          if (typeof resolvedValue === 'string') {
+            // resolve reference first
+            if (resolvedValue.match(/\$\{(.+?)\}/)) {
+              try {
+                resolvedValue = transformFileLoaderResult(
+                  resolvedValue,
+                  context,
+                  disallowUndefinedEnvironmentVariables,
+                  visited,
+                );
+              } catch (error) {
+                if (error instanceof RangeError) {
+                  debug(
+                    `Can not resolve a circular reference in ${obj} -> ${resolvedValue} -> ${error.message}`,
+                    error,
+                  );
+                }
+
+                throw error;
+              }
+            }
+
+            obj = obj.toString().replace(placeholder, resolvedValue.toString());
+          } else if (typeof resolvedValue === 'object') {
+            obj = transformFileLoaderResult(
+              obj,
+              resolvedValue,
+              disallowUndefinedEnvironmentVariables,
+              visited,
+            );
+          } else if (
+            typeof resolvedValue === 'number' ||
+            typeof resolvedValue === 'boolean'
+          ) {
+            // if it's one to one reference, just return it as a number
+            if (obj === placeholder) {
+              obj = resolvedValue;
+            } else {
+              //   this means that we're embedding some number into string
+              obj = obj
+                .toString()
+                .replace(placeholder, resolvedValue.toString());
+            }
+          }
+        } else if (disallowUndefinedEnvironmentVariables) {
+          throw new Error(
+            `Environment variable is not set for variable name: '${variable}'`,
+          );
+        }
+      }
+    }
+  } else if (typeof obj === 'number' || typeof obj === 'boolean') {
+    return obj;
+  } else if (typeof obj === 'object' && obj !== null) {
+    /**
+     * it's not really possible to have circular reference in JSON and yaml like this
+     * but probably one day there will be more complex scenarios for this function
+     */
+    /* istanbul ignore next */
+    if (visited.has(obj)) {
+      return obj; // Avoid infinite loops on circular references
+    }
+    visited.add(obj);
+
+    if (Array.isArray(obj)) {
+      for (let i = 0; i < obj.length; i++) {
+        obj[i] = transformFileLoaderResult(
+          obj[i],
+          context,
+          disallowUndefinedEnvironmentVariables,
+          visited,
+        );
+      }
+    } else {
+      for (const key in obj) {
+        obj[key] = transformFileLoaderResult(
+          obj[key],
+          context,
+          disallowUndefinedEnvironmentVariables,
+          visited,
+        );
+      }
+    }
+
+    visited.delete(obj);
+  }
+  return obj;
+}

lib/typed-config.module.ts

@@ -61,11 +61,17 @@ export class TypedConfigModule {
     if (Array.isArray(load)) {
       const config = {};
       for (const fn of load) {
+        // we shouldn't silently catch errors here, because app shouldn't start without the proper config
+        // same way as it doesn't start without the proper database connection
+        // and the same way as it now fail for the single loader
         try {
-          const conf = fn();
+          const conf = fn(config);
           merge(config, conf);
-        } catch (err: any) {
-          debug(`Config load failed: ${err.message}`);
+        } catch (e: any) {
+          debug(
+            `Config load failed: ${e}. Details: ${JSON.stringify(e.details)}`,
+          );
+          throw e;
         }
       }
       return config;
@@ -80,10 +86,13 @@ export class TypedConfigModule {
       const config = {};
       for (const fn of load) {
         try {
-          const conf = await fn();
+          const conf = await fn(config);
           merge(config, conf);
-        } catch (err: any) {
-          debug(`Config load failed: ${err.message}`);
+        } catch (e: any) {
+          debug(
+            `Config load failed: ${e}. Details: ${JSON.stringify(e.details)}`,
+          );
+          throw e;
         }
       }
       return config;

tests/e2e/multiple-loaders.spec.ts

@@ -35,14 +35,11 @@ describe('Local toml', () => {
     expect(databaseConfig.host).toBe('host.part1');
   });
 
-  it(`should be able load config when some of the loaders fail`, async () => {
-    await init(['reject', 'part1', 'part2']);
-    const tableConfig = app.get(TableConfig);
-    expect(tableConfig.name).toBe('test');
-
-    await init(['reject', 'part1', 'part2'], false);
-    const tableConfig2 = app.get(TableConfig);
-    expect(tableConfig2.name).toBe('test');
+  /**
+   * this is a subject for discussion
+   * */
+  it(`should not be able load config when some of the loaders fail`, async () => {
+    expect(init(['reject', 'part1', 'part2'])).rejects.toThrowError();
   });
 
   afterEach(async () => {