feat(variables): add support for yaml variable input. fixes #615

Author: justinedelson

package.json

@@ -19,6 +19,7 @@
     "figures": "^3.2.0",
     "halfred": "^2.0.0",
     "inquirer": "^8.1.0",
+    "js-yaml": "^4.1.0",
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.15",
     "moment": "^2.29.0"

src/ValidationErrors.js

@@ -52,6 +52,8 @@ E('INTERNAL_VARIABLE_USAGE', 'The variable name %s is reserved for internal usag
 E('IP_ALLOWLIST_NOT_FOUND', 'Could not find IP Allowlist with id %s in program id %s.')
 E('VARIABLES_JSON_PARSE_ERROR', 'Unable to parse variables from provided data.')
 E('VARIABLES_JSON_NOT_ARRAY', 'Provided variables input was not an array.')
+E('VARIABLES_YAML_PARSE_ERROR', 'Unable to parse variables from provided data.')
+E('VARIABLES_YAML_NOT_ARRAY', 'Provided variables input was not an array.')
 E('BOTH_BRANCH_AND_TAG_PROVIDED', 'Both branch and tag cannot be specified.')
 E('MISSING_PROGRAM_ID', 'Program ID must be specified either as --programId flag or through cloudmanager_programid config value.')
 E('MISSING_METRICS', 'Metrics for action %s on execution %s could not be found.')

src/base-variables-command.js

@@ -15,6 +15,7 @@ const fs = require('fs')
 
 const BaseCommand = require('./base-command')
 const { cli } = require('cli-ux')
+const yaml = require('js-yaml')
 const { flags } = require('@oclif/command')
 const { getProgramId, createKeyValueObjectFromFlag, getOutputFormat } = require('./cloudmanager-helpers')
 const commonFlags = require('./common-flags')
@@ -126,44 +127,25 @@ class BaseVariablesCommand extends BaseCommand {
       }
     })
 
+    const getFileData = promisify(fs.readFile)
+
     if (flags.jsonStdin) {
       const rawStdinData = await getPipedData()
-      this.loadVariablesFromJson(rawStdinData, currentVariableTypes, variables)
+      BaseVariablesCommand.loadVariablesFromJson(rawStdinData, currentVariableTypes, variables)
     } else if (flags.jsonFile) {
-      const getFileData = promisify(fs.readFile)
       const rawFileData = await getFileData(flags.jsonFile)
-      this.loadVariablesFromJson(rawFileData, currentVariableTypes, variables)
+      BaseVariablesCommand.loadVariablesFromJson(rawFileData, currentVariableTypes, variables)
+    } else if (flags.yamlStdin) {
+      const rawStdinData = await getPipedData()
+      BaseVariablesCommand.loadVariablesFromYaml(rawStdinData, currentVariableTypes, variables)
+    } else if (flags.yamlFile) {
+      const rawFileData = await getFileData(flags.yamlFile)
+      BaseVariablesCommand.loadVariablesFromYaml(rawFileData, currentVariableTypes, variables)
     }
 
     return variables
   }
 
-  loadVariablesFromJson (rawData, currentVariableTypes, variables) {
-    let data
-    try {
-      data = JSON.parse(rawData)
-    } catch (e) {
-      throw new validationCodes.VARIABLES_JSON_PARSE_ERROR()
-    }
-    if (!_.isArray(data)) {
-      throw new validationCodes.VARIABLES_JSON_NOT_ARRAY()
-    }
-    data.forEach(item => {
-      if (item.name && !_.isUndefined(item.value)) {
-        if (!item.type) {
-          item.type = 'string'
-        }
-        const currentVariableKey = item.service ? `${item.service}:${item.name}` : item.name
-        if (currentVariableTypes[currentVariableKey] && !item.value) {
-          item.type = currentVariableTypes[currentVariableKey]
-        }
-        if (!variables.find(variable => variable.name === item.name && variables.services === item.service)) {
-          variables.push(item)
-        }
-      }
-    })
-  }
-
   validateVariables (flags, variables) { }
 }
 
@@ -190,10 +172,20 @@ BaseVariablesCommand.setterFlags = {
   jsonStdin: flags.boolean({
     default: false,
     description: 'if set, read variables from a JSON array provided as standard input; variables set through --variable or --secret flag will take precedence',
+    exclusive: ['jsonFile', 'yamlStdin', 'yamlFile'],
   }),
   jsonFile: flags.string({
     description: 'if set, read variables from a JSON array provided as a file; variables set through --variable or --secret flag will take precedence',
-    exclusive: ['jsonStdin'],
+    exclusive: ['jsonStdin', 'yamlStdin', 'yamlFile'],
+  }),
+  yamlStdin: flags.boolean({
+    default: false,
+    description: 'if set, read variables from a YAML array provided as standard input; variables set through --variable or --secret flag will take precedence',
+    exclusive: ['jsonStdin', 'jsonFile', 'yamlFile'],
+  }),
+  yamlFile: flags.string({
+    description: 'if set, read variables from a YAML array provided as a file; variables set through --variable or --secret flag will take precedence',
+    exclusive: ['jsonStdin', 'jsonFile', 'yamlStdin'],
   }),
   ...commonFlags.outputFormat,
 }
@@ -202,4 +194,47 @@ BaseVariablesCommand.getterFlags = {
   ...commonFlags.outputFormat,
 }
 
+const loadVariableData = (array, currentVariableTypes, variables) => {
+  array.forEach(item => {
+    if (item.name && !_.isUndefined(item.value)) {
+      if (!item.type) {
+        item.type = 'string'
+      }
+      const currentVariableKey = item.service ? `${item.service}:${item.name}` : item.name
+      if (currentVariableTypes[currentVariableKey] && !item.value) {
+        item.type = currentVariableTypes[currentVariableKey]
+      }
+      if (!variables.find(variable => variable.name === item.name && variables.services === item.service)) {
+        variables.push(item)
+      }
+    }
+  })
+}
+
+BaseVariablesCommand.loadVariablesFromJson = (rawData, currentVariableTypes, variables) => {
+  let data
+  try {
+    data = JSON.parse(rawData)
+  } catch (e) {
+    throw new validationCodes.VARIABLES_JSON_PARSE_ERROR()
+  }
+  if (!_.isArray(data)) {
+    throw new validationCodes.VARIABLES_JSON_NOT_ARRAY()
+  }
+  loadVariableData(data, currentVariableTypes, variables)
+}
+
+BaseVariablesCommand.loadVariablesFromYaml = (rawData, currentVariableTypes, variables) => {
+  let data
+  try {
+    data = yaml.load(rawData)
+  } catch (e) {
+    throw new validationCodes.VARIABLES_YAML_PARSE_ERROR()
+  }
+  if (!_.isArray(data)) {
+    throw new validationCodes.VARIABLES_YAML_NOT_ARRAY()
+  }
+  loadVariableData(data, currentVariableTypes, variables)
+}
+
 module.exports = BaseVariablesCommand

test/base-variables-command.test.js

@@ -0,0 +1,38 @@
+/*
+Copyright 2022 Adobe. All rights reserved.
+This file is licensed to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License. You may obtain a copy
+of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under
+the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+OF ANY KIND, either express or implied. See the License for the specific language
+governing permissions and limitations under the License.
+*/
+
+const { promisify } = require('util')
+const fs = require('fs')
+const path = require('path')
+const BaseVariablesCommand = require('../src/base-variables-command')
+
+test('multiline json', async () => {
+  const getFileData = promisify(fs.readFile)
+  const rawFileData = await getFileData(path.join(__dirname, 'data', 'variables.json'))
+
+  const variables = []
+  BaseVariablesCommand.loadVariablesFromJson(rawFileData, {}, variables)
+
+  expect(variables).toHaveLength(1)
+  expect(variables[0].value.split('\n')).toHaveLength(3)
+})
+
+test('multiline yaml', async () => {
+  const getFileData = promisify(fs.readFile)
+  const rawFileData = await getFileData(path.join(__dirname, 'data', 'variables.yaml'))
+
+  const variables = []
+  BaseVariablesCommand.loadVariablesFromYaml(rawFileData, {}, variables)
+
+  expect(variables).toHaveLength(1)
+  expect(variables[0].value.split('\n')).toHaveLength(3)
+})

test/commands/environment/set-variables.test.js

@@ -291,6 +291,22 @@ test('set-environment-variables - stdin - not JSON', async () => {
   await expect(runResult).rejects.toThrow('Unable to parse variables from provided data.')
 })
 
+test('set-environment-variables - stdin - bad YAML', async () => {
+  setCurrentOrgId('good')
+  setStore({
+    cloudmanager_programid: '4',
+  })
+
+  getPipedData.mockResolvedValue('{garbage')
+
+  expect.assertions(2)
+
+  const runResult = SetEnvironmentVariablesCommand.run(['1', '--yamlStdin'])
+
+  await expect(runResult instanceof Promise).toBeTruthy()
+  await expect(runResult).rejects.toThrow('Unable to parse variables from provided data.')
+})
+
 test('set-environment-variables - file - not JSON', async () => {
   setCurrentOrgId('good')
   setStore({
@@ -307,7 +323,23 @@ test('set-environment-variables - file - not JSON', async () => {
   await expect(runResult).rejects.toThrow('Unable to parse variables from provided data.')
 })
 
-test('set-environment-variables - stdin - not array', async () => {
+test('set-environment-variables - file - bad YAML', async () => {
+  setCurrentOrgId('good')
+  setStore({
+    cloudmanager_programid: '4',
+  })
+
+  mockFileContent = '{garbage'
+
+  expect.assertions(2)
+
+  const runResult = SetEnvironmentVariablesCommand.run(['1', '--yamlFile', mockFileName])
+
+  await expect(runResult instanceof Promise).toBeTruthy()
+  await expect(runResult).rejects.toThrow('Unable to parse variables from provided data.')
+})
+
+test('set-environment-variables - JSON stdin - not array', async () => {
   setCurrentOrgId('good')
   setStore({
     cloudmanager_programid: '4',
@@ -323,7 +355,23 @@ test('set-environment-variables - stdin - not array', async () => {
   await expect(runResult).rejects.toThrow('Provided variables input was not an array.')
 })
 
-test('set-environment-variables - stdin - secret and variable', async () => {
+test('set-environment-variables - YAML stdin - not array', async () => {
+  setCurrentOrgId('good')
+  setStore({
+    cloudmanager_programid: '4',
+  })
+
+  getPipedData.mockResolvedValue('test: true')
+
+  expect.assertions(2)
+
+  const runResult = SetEnvironmentVariablesCommand.run(['1', '--yamlStdin'])
+
+  await expect(runResult instanceof Promise).toBeTruthy()
+  await expect(runResult).rejects.toThrow('Provided variables input was not an array.')
+})
+
+test('set-environment-variables - JSON stdin - secret and variable', async () => {
   setCurrentOrgId('good')
   setStore({
     cloudmanager_programid: '4',
@@ -496,7 +544,7 @@ test('set-environment-variables - both jsonStdin and jsonFile', async () => {
 
   await expect(runResult instanceof Promise).toBeTruthy()
 
-  await expect(runResult).rejects.toThrow('--jsonStdin= cannot also be provided when using --jsonFile=')
+  await expect(runResult).rejects.toThrow('--jsonFile= cannot also be provided when using --jsonStdin=')
 })
 
 test('set-environment-variables - author variable', async () => {

test/data/variables.json

@@ -0,0 +1,7 @@
+[
+  {
+    "name" : "mltest",
+    "value" : "abc\ndef\nghi"
+  }
+
+]

test/data/variables.yaml

@@ -0,0 +1,5 @@
+- name: mltest
+  value: |-
+    abc
+    def
+    ghi