Merge pull request #95 from launchcodedev/feat/exec-extension

$exec extension

Author: joelgallant

.github/workflows/main.yml

@@ -35,6 +35,7 @@ jobs:
           ./app-config-cypress/coverage/coverage-final.json,
           ./app-config-default-extensions/coverage/coverage-final.json,
           ./app-config-encryption/coverage/coverage-final.json,
+          ./app-config-exec/coverage/coverage-final.json,
           ./app-config-extension-utils/coverage/coverage-final.json,
           ./app-config-extensions/coverage/coverage-final.json,
           ./app-config-generate/coverage/coverage-final.json,

.github/workflows/publishing.yml

@@ -46,6 +46,11 @@ jobs:
         token: ${{ secrets.NPM_TOKEN }}
         access: public
         package: ./app-config-encryption/package.json
+    - uses: JS-DevTools/npm-publish@v1
+      with:
+        token: ${{ secrets.NPM_TOKEN }}
+        access: public
+        package: ./app-config-exec/package.json
     - uses: JS-DevTools/npm-publish@v1
       with:
         token: ${{ secrets.NPM_TOKEN }}

app-config-exec/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@lcdev/eslint-config/cwd')(__dirname);

app-config-exec/README.md

@@ -0,0 +1,96 @@
+## App Config Exec Plugin
+
+Run arbitrary programs or shell commands to generate config.
+
+```yaml
+$exec: echo Hello World!
+```
+
+## Usage
+
+#### Basic:
+
+```yaml
+$exec: echo Hello World!
+```
+
+#### Auto-parse output as YAML/TOML/JSON/JSON5:
+
+```yaml
+$exec:
+  command: curl https://my-api.example.com
+  parseOutput: true
+```
+
+#### Build a custom script to generate config:
+
+```yaml
+$exec: node ./my-custom-config-generator.js
+```
+
+#### Retrieve arbitrary information:
+
+_Node v8 version:_
+
+```yaml
+$exec: node -p -e "process.versions.v8"
+```
+
+_System's architecture:_
+
+```yaml
+$exec: uname -p
+```
+
+#### Retrieve list of AWS S3 buckets via `aws` & `jq` CLI:
+
+```yaml
+$exec:
+  command: aws s3api list-buckets --output json | jq -r '.Buckets'
+  parseOutput: true
+```
+
+_Note: When possible, we encourage you to build a dedicated extension to better support features you are looking for (and help out the community). eg. `$aws` directive instead of the above._
+
+## Installing
+
+Install and use:
+
+```sh
+yarn add @app-config/exec
+```
+
+In `.app-config.meta.yml` file:
+
+```yaml
+parsingExtensions:
+  - '@app-config/exec'
+```
+
+## Options
+
+The following options can be passed to each `$exec` directive:
+
+```yaml
+$exec:
+  command: echo Hello World!
+  trimWhitespace: true
+  parseOutput: false
+  failOnStderr: false
+```
+
+#### `command: string`:
+
+The command to run. Should be a single `string` containing both command and arguments. The command's `stdout` will be used as the resulting value. Runs in a shell, `/bin/sh` on Unix and `process.env.ComSpec` on Windows.
+
+#### `trimWhitespace: boolean`:
+
+If `true`, automatically trim whitespace from the start and end of the command's output. Useful to remove the trailing newline produced by most commands. If `false`, output will be left in its raw form. Default `true`.
+
+#### `parseOutput: boolean`:
+
+If `true`, automatically guess output content type (YAML, TOML, JSON, or JSON5) and parse. Throws if parse fails. If `false`, output will be read as a `string`. Default `false`.
+
+#### `failOnStderr: boolean`:
+
+If `true`, fail if any output is found in command's `stderr`. Default `false`.

app-config-exec/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@app-config/exec",
+  "description": "Generate config by running arbitrary programs",
+  "version": "2.0.3",
+  "license": "MPL-2.0",
+  "author": {
+    "name": "Launchcode",
+    "email": "[email protected]",
+    "url": "https://lc.dev"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/launchcodedev/app-config.git"
+  },
+  "main": "dist/index.js",
+  "module": "dist/es/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "/dist",
+    "!*.tsbuildinfo",
+    "!*.test.*"
+  ],
+  "scripts": {
+    "build": "tsc -b",
+    "build:es": "tsc -b tsconfig.es.json",
+    "clean": "rm -rf dist *.tsbuildinfo",
+    "lint": "eslint src",
+    "fix": "eslint --fix src",
+    "test": "jest",
+    "prepublishOnly": "yarn clean && yarn build && yarn build:es"
+  },
+  "dependencies": {
+    "@app-config/core": "^2.1.8",
+    "@app-config/extension-utils": "^2.1.8",
+    "@app-config/utils": "^2.1.8"
+  },
+  "devDependencies": {
+    "@app-config/main": "^2.1.8"
+  },
+  "prettier": "@lcdev/prettier",
+  "jest": {
+    "preset": "@lcdev/jest",
+    "setupFilesAfterEnv": [
+      "<rootDir>/test-setup.js"
+    ]
+  }
+}

app-config-exec/src/index.test.ts

@@ -0,0 +1,128 @@
+import { defaultEnvExtensions, defaultExtensions, loadUnvalidatedConfig } from '@app-config/main';
+import execParsingExtension from '.';
+
+const defaultOptions = {
+  environmentExtensions: defaultEnvExtensions().concat(execParsingExtension()),
+  parsingExtensions: defaultExtensions().concat(execParsingExtension()),
+};
+
+describe('execParsingExtension', () => {
+  it('reads from command as root level string', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: 'echo test123',
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toEqual('test123');
+  });
+
+  it('reads from command within nested object options', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: { command: 'echo test123' },
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toEqual('test123');
+  });
+
+  it('reads JSON as string by default', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: { command: `echo '{"test": true}'` },
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toBe('{"test": true}');
+  });
+
+  it('parses JSON if parseOutput true', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: { command: `echo '{"test": true}'`, parseOutput: true },
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toMatchObject({ test: true });
+  });
+
+  it('trims whitespace by default', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: { command: `echo '  test123\n'` },
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toBe('test123');
+  });
+
+  it('reads raw output if trimWhitespace false', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: { command: `echo '  test123'`, trimWhitespace: false },
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toBe('  test123\n');
+  });
+
+  it('does not fail on stderr by default', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: {
+        command: `node -e 'process.stdout.write("stdout"); process.stderr.write("stderr");'`,
+      },
+    });
+
+    const { fullConfig } = await loadUnvalidatedConfig(defaultOptions);
+
+    expect(fullConfig).toEqual('stdout');
+  });
+
+  it('fails on stderr when failOnStderr true', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: {
+        command: `node -e 'process.stdout.write("stdout"); process.stderr.write("stderr");'`,
+        failOnStderr: true,
+      },
+    });
+
+    const action = async () => {
+      await loadUnvalidatedConfig(defaultOptions);
+    };
+
+    await expect(action()).rejects.toThrow();
+  });
+
+  it('fails if options is not a string or object', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: 12345,
+    });
+
+    const action = async () => {
+      await loadUnvalidatedConfig(defaultOptions);
+    };
+
+    await expect(action()).rejects.toThrow();
+  });
+
+  it('fails if options dont include command', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: {},
+    });
+
+    await expect(loadUnvalidatedConfig(defaultOptions)).rejects.toThrow();
+  });
+
+  it('invalid command fails', async () => {
+    process.env.APP_CONFIG = JSON.stringify({
+      $exec: { command: 'non-existing-command' },
+    });
+
+    const action = async () => {
+      await loadUnvalidatedConfig(defaultOptions);
+    };
+
+    await expect(action()).rejects.toThrow();
+  });
+});

app-config-exec/src/index.ts

@@ -0,0 +1,94 @@
+import { Json } from '@app-config/utils';
+import { ParsingExtension, parseRawString, guessFileType, AppConfigError } from '@app-config/core';
+import { forKey, validateOptions } from '@app-config/extension-utils';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+
+export interface Options {
+  command: string;
+  failOnStderr: boolean;
+  parseOutput: boolean;
+  trimWhitespace: boolean;
+}
+
+class ExecError extends AppConfigError {
+  name = 'ExecError';
+}
+
+function execParsingExtension(): ParsingExtension {
+  return forKey(
+    '$exec',
+    validateOptions(
+      (SchemaBuilder) =>
+        SchemaBuilder.oneOf(
+          SchemaBuilder.stringSchema(),
+          SchemaBuilder.emptySchema()
+            .addString('command')
+            .addBoolean('failOnStderr', {}, false)
+            .addBoolean('parseOutput', {}, false)
+            .addBoolean('trimWhitespace', {}, false),
+        ),
+      (value) => async (parse) => {
+        let options;
+
+        if (typeof value === 'string') {
+          options = { command: value };
+        } else {
+          options = value;
+        }
+
+        const {
+          command,
+          failOnStderr = false,
+          parseOutput = false,
+          trimWhitespace = true,
+        } = options;
+
+        try {
+          const { stdout, stderr } = await execAsync(command);
+
+          if (failOnStderr && stderr) {
+            throw new ExecError(`$exec command "${command}" produced stderr: ${stderr}`);
+          }
+
+          let result: Json = stdout;
+
+          if (trimWhitespace) {
+            result = stdout.trim();
+          }
+
+          if (parseOutput) {
+            const fileType = await guessFileType(stdout);
+            result = await parseRawString(stdout, fileType);
+          }
+
+          return parse(result, { shouldFlatten: true });
+        } catch (err: unknown) {
+          if (!isError(err)) {
+            throw err;
+          }
+
+          if (err instanceof ExecError) {
+            throw err;
+          }
+
+          throw new ExecError(`$exec command "${command}" failed with error:\n${err.message}`);
+        }
+      },
+    ),
+  );
+}
+
+/**
+ * Check if a value is an Error.
+ *
+ * This was created because `value instanceof Error` was returning false
+ * for normal-looking errors coming from `child_process` `exec()` command
+ */
+function isError(value: unknown): value is Error {
+  return typeof value === 'object' && (value as Error).message !== undefined;
+}
+
+export default execParsingExtension;

app-config-exec/test-setup.js

@@ -0,0 +1,5 @@
+const originalEnvironment = { ...process.env };
+
+afterEach(() => {
+  process.env = { ...originalEnvironment };
+});