feat: compositional env validation

Author: aryanjassal

npmDepsHash

@@ -1 +1 @@
-sha256-/SFcicpWX1SFllzmz0acUYxtIV0LuBw1PU32Uk5hLzw=
+sha256-5lr9ukYvA7s8sAgnX5rN6ef12tOb9lPwAHGlOrD0k+w=

package-lock.json

@@ -136,6 +136,7 @@
     "sodium-native": "*"
   },
   "devDependencies": {
+    "@apidevtools/json-schema-ref-parser": "^11.9.3",
     "@fast-check/jest": "^2.1.1",
     "@matrixai/errors": "^2.1.3",
     "@matrixai/exec": "^1.0.3",

package.json

@@ -0,0 +1,14 @@
+{
+  "type": "object",
+  "allOf": [
+    { "$ref": "./test.schema.json"}
+  ],
+  "properties": {
+    "SECRET2": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "SECRET2"
+  ]
+}

ref.schema.json

@@ -1,7 +1,9 @@
 import type PolykeyClient from 'polykey/PolykeyClient.js';
-import type { ParsedSecretPathValue } from '../types.js';
+import type { JSONSchema, ParsedSecretPathValue } from '../types.js';
 import path from 'node:path';
 import os from 'node:os';
+import $RefParser from '@apidevtools/json-schema-ref-parser';
+import { Ajv2019 as Ajv } from 'ajv/dist/2019.js';
 import { InvalidArgumentError } from 'commander';
 import * as utils from 'polykey/utils/index.js';
 import CommandPolykey from '../CommandPolykey.js';
@@ -26,6 +28,7 @@ class CommandEnv extends CommandPolykey {
     this.addOption(binOptions.envDuplicate);
     this.addOption(binOptions.envExport);
     this.addOption(binOptions.preserveNewline);
+    this.addOption(binOptions.egressSchema);
     this.argument(
       '<args...>',
       'command and arguments formatted as <envPaths...> [-- cmd [cmdArgs...]]',
@@ -53,6 +56,7 @@ class CommandEnv extends CommandPolykey {
         if (secretPath == null) preservedSecrets.add(vaultName);
         else preservedSecrets.add(`${vaultName}:${secretPath}`);
       }
+
       // There are a few stages here
       // 1. parse the desired secrets
       // 2. obtain the desired secrets
@@ -226,7 +230,50 @@ class CommandEnv extends CommandPolykey {
             };
           }
           await writeP;
-          return [envp, envpPath];
+
+          // Apply validation using the schema
+          // TODO: filter before pulling instead of after
+          const filteredEnvp: Record<string, string> = {};
+          if (options.egressSchema != null) {
+            // Resolve references and bundle schema
+            const schema: JSONSchema = await $RefParser.bundle(
+              options.egressSchema,
+            );
+
+            // Validate the incoming secrets against the schema
+            const ajv = new Ajv({
+              coerceTypes: true,
+              useDefaults: false,
+              allErrors: true,
+            });
+            const validate = ajv.compile(schema);
+            validate(envp);
+
+            // Extract relevant keys, discarding the rest
+            const { requiredKeys, allKeys, defaults } =
+              binUtils.loadSchema(schema);
+
+            for (const key of allKeys) {
+              let value = envp[key];
+              if (value == null && defaults[key] != null) {
+                value = defaults[key];
+              }
+              if (
+                requiredKeys.includes(key) &&
+                (value == null || value === '')
+              ) {
+                throw new Error('TMP missing required variable');
+              }
+              if (value != null) {
+                filteredEnvp[key] = value.toString();
+              }
+            }
+          }
+
+          return [
+            utils.isEmptyObject(filteredEnvp) ? envp : filteredEnvp,
+            envpPath,
+          ];
         }, meta);
         // End connection early to avoid errors on server
         await pkClient.stop();

src/secrets/CommandEnv.ts

@@ -63,6 +63,32 @@ type PromiseDeconstructed<T> = {
 
 type ParsedSecretPathValue = [string, string?, string?];
 
+type JSONSchemaProps = Record<
+  string,
+  {
+    type?: string;
+    default?: unknown;
+    [key: string]: unknown;
+  }
+>;
+
+type JSONSchema = {
+  type?: string;
+  properties?: Record<string, JSONSchemaProps>;
+  required?: Array<string>;
+  allOf?: Array<JSONSchema>;
+  anyOf?: Array<JSONSchema>;
+  oneOf?: Array<JSONSchema>;
+  [key: string]: unknown;
+};
+
+// Only strings are supported for the time being
+type JSONSchemaInfo = {
+  allKeys: Array<string>;
+  requiredKeys: Array<string>;
+  defaults: Record<string, string>;
+};
+
 export type {
   TableRow,
   TableOptions,
@@ -72,4 +98,7 @@ export type {
   AgentChildProcessOutput,
   PromiseDeconstructed,
   ParsedSecretPathValue,
+  JSONSchemaProps,
+  JSONSchema,
+  JSONSchemaInfo,
 };

src/types.ts

@@ -325,6 +325,11 @@ const returnURLPath = new Option(
   'Which path on the website to send the token to',
 );
 
+const egressSchema = new Option(
+  '--egress-schema <path>',
+  'A JSON schema controlling the egressing secrets',
+);
+
 export {
   nodePath,
   format,
@@ -371,4 +376,5 @@ export {
   parents,
   preserveNewline,
   returnURLPath,
+  egressSchema,
 };

src/utils/options.ts

@@ -5,6 +5,8 @@ import type {
   TableOptions,
   DictOptions,
   PromiseDeconstructed,
+  JSONSchema,
+  JSONSchemaInfo,
 } from '../types.js';
 import process from 'node:process';
 import { LogLevel } from '@matrixai/logger';
@@ -646,6 +648,47 @@ function jsonToCompactJWT(token: SignedTokenEncoded): string {
   return `${token.signatures[0].protected}.${token.payload}.${token.signatures[0].signature}`;
 }
 
+function loadSchema(bundledSchema: JSONSchema): JSONSchemaInfo {
+  const props: Set<string> = new Set();
+  const required: Set<string> = new Set();
+  const defaults: Record<string, string> = {};
+
+  const unwrapSchema = (schema: JSONSchema) => {
+    // Collect properties and their defaults
+    if (schema.properties != null) {
+      for (const [k, p] of Object.entries(schema.properties)) {
+        props.add(k);
+        if (p.default == null) continue;
+        if (typeof p.default !== 'string') {
+          throw new Error('TMP wrong type');
+        }
+        defaults[k] = p.default;
+      }
+    }
+
+    // Collect required properties
+    if (schema.required != null) {
+      schema.required.forEach((requiredSecret) => required.add(requiredSecret));
+    }
+
+    // Process composition keywords
+    const compositionKeywords = ['allOf', 'anyOf', 'oneOf'] as const;
+    compositionKeywords.forEach(
+      (keyword) =>
+        schema[keyword] != null &&
+        Array.isArray(schema[keyword]) &&
+        schema[keyword]!.forEach(unwrapSchema),
+    );
+  };
+  unwrapSchema(bundledSchema);
+
+  return {
+    allKeys: [...props],
+    requiredKeys: [...required],
+    defaults: defaults,
+  };
+}
+
 export {
   verboseToLogLevel,
   standardErrorReplacer,
@@ -670,6 +713,7 @@ export {
   promise,
   importFS,
   jsonToCompactJWT,
+  loadSchema,
 };
 
 export type { OutputObject };

src/utils/utils.ts

@@ -0,0 +1,11 @@
+{
+  "type": "object",
+  "properties": {
+    "SECRET1": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "SECRET1"
+  ]
+}