add support for ENCRYPTED Variables in pipeline run cmd (#813)

eti-codefresh · web-flow · commit 657e6552fdf2 · 2023-05-08T11:40:15.000+01:00
diff --git a/codefresh-release.yml b/codefresh-release.yml
@@ -445,14 +445,13 @@ steps:
   update_documentation:
     stage: documentation
     title: "Update documentation http://cli.codefresh.io"
-    image: docker:18.01
+    image: codefresh/build-cli
     commands:
-      - "apk update && apk add git nodejs"
-      - "npm install"
+      - "yarn"
       - "echo cleaning previous public dir and recreating worktree"
-      - "rm -rf public && git worktree prune &&   git worktree add -B gh-pages public origin/gh-pages"
+      - "rm -rf public && git worktree prune && git worktree add -B gh-pages public origin/gh-pages"
       - "echo Building public docs"
-      - "npm run build-public-docs"
+      - "yarn run build-public-docs"
       - "echo Push new docs to gh-pages detached branch"
       - 'git config --global user.email "auto-ci@codefresh.io" && git config --global user.name "Automated CI"'
       - 'cd public && git add --all && git commit -m "Publish new documentation for version ${{PACKAGE_VERSION}}" && git push https://${{GITHUB_TOKEN}}@github.com/codefresh-io/cli.git'
diff --git a/codefresh.yml b/codefresh.yml
@@ -328,7 +328,7 @@ steps:
     type: codefresh-run
     arguments:
       PIPELINE_ID: 'codefresh-io/cli/release'
-      DETACH: true
+      TRIGGER_ID: codefresh-io/cli_1
       BRANCH: master
       VARIABLE:
         - PACKAGE_VERSION=${{PACKAGE_VERSION}}
@@ -342,3 +342,32 @@ steps:
         - name: create_manifest_list
           on:
             - success
+
+  execute_e2e_pipeline:
+    stage: final
+    title: "Execute E2E pipeline for image of this commit"
+    type: codefresh-run
+    arguments:
+      PIPELINE_ID: 'cli-v1-e2e/root'
+      VARIABLE:
+        - CLI_VERSION=${{CF_SHORT_REVISION}}
+    when:
+      steps:
+        - name: push_step_alpine
+          on:
+            - success
+
+  build_documentation:
+    stage: test
+    title: "build documentation http://cli.codefresh.io"
+    image: codefresh/build-cli
+    commands:
+      - "echo Building public docs"
+      - "yarn run build-public-docs"
+    environment:
+      - HUGO_VERSION=0.32.0
+    when:
+      steps:
+        - name: install_dependencies
+          on:
+            - success
diff --git a/docs/content/pipelines/Run Pipeline.md b/docs/content/pipelines/Run Pipeline.md
@@ -15,22 +15,44 @@ The pipeline will be triggered multiple times according to the array length.
 
 #### Variable yaml file with 2 sets of variables
 ```yaml
-- key: value
-  key2: key1
-- key: value
-  key2: key2
+- VARIABLE_A: value_a_for_the_first_build
+  VARIABLE_B: value_b_for_the_first_build
+- VARIABLE_A: value_a_for_the_first_build
+  VARIABLE_B: value_b_for_the_first_build
 ```
 
 #### Variable json file with 2 sets of variables
 ```json
 [
   {
-    "key": "value",
-    "key2": "key1"
+    "VARIABLE_A": "value_a_for_the_first_build",
+    "VARIABLE_B": "value_b_for_the_first_build"
   },
   {
-    "key": "value",
-    "key2": "key2"
+    "VARIABLE_A": "value_a_for_the_first_build",
+    "VARIABLE_B": "value_b_for_the_first_build"
+  }
+]
+```
+### Use encrypted variables in Codefresh build runs
+#### Variable yaml file with single variable set with encrypted variables
+```yaml
+- key:
+    val: value
+    encrypted: true
+  key2: val2
+
+```
+
+#### Variable json file single variable set with encrypted variables
+```json
+[
+  {
+    "key": {
+      "val": "value",
+      "encrypted": true
+    },
+    "key2": "key1"
   }
 ]
 ```
diff --git a/lib/interface/cli/commands/annotation/create.cmd.js b/lib/interface/cli/commands/annotation/create.cmd.js
@@ -31,7 +31,7 @@ const command = new Command({
         .example('codefresh create annotation image 2dfacdaad466 coverage=75%', 'Annotate entity with a single label')
         .example('codefresh create annotation image 2dfacdaad466 coverage=75% tests_passed=true', 'Annotate entity with multiple labels')
         // eslint-disable-next-line max-len
-        .example('codefresh create annotation image 2dfacdaad466 coverage=75% tests_passed=true --display coverage', 'Annotate entity with multiple labels and display selection'),
+        .example('codefresh create annotation workflow 643d807b85bbe35931ae2282 ENV=prod tests_passed=true --display ENV', 'Annotate entity with multiple labels and display selection'),
     handler: async (argv) => {
         const { entityType, entityId, labels, display } = argv;
 
diff --git a/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js b/lib/interface/cli/commands/pipeline/pipeline.sdk.spec.js
@@ -1,3 +1,6 @@
+const yaml = require('js-yaml');
+const request = require('requestretry');
+const fs = require('fs');
 const DEFAULTS = require('../../defaults');
 const getCmd = require('./get.cmd').toCommand();
 const deleteCmd = require('./delete.cmd').toCommand();
@@ -10,18 +13,21 @@ jest.mock('../../helpers/validation'); // eslint-disable-line
 jest.mock('../../../../../check-version');
 jest.mock('../../completion/helpers', () => { // eslint-disable-line
     return {
-        authContextWrapper: func => func,
+        authContextWrapper: (func) => func,
     };
 });
 
+jest.mock('../../helpers/general', () => ({
+    ...jest.requireActual('../../helpers/general'),
+    isCompatibleApiVersion: () => true,
+}));
+
 jest.mock('../../../../logic/entities/Pipeline', () => { // eslint-disable-line
     return {
-        fromResponse: res => res,
+        fromResponse: (res) => res,
     };
 });
 
-const request = require('requestretry');
-
 const DEFAULT_RESPONSE = request.__defaultResponse();
 
 describe('pipeline', () => {
@@ -57,11 +63,11 @@ describe('pipeline', () => {
             });
 
             it('should return default limit', async () => {
-                expect(_getLimit(undefined,false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS);
+                expect(_getLimit(undefined, false)).toEqual(DEFAULTS.GET_LIMIT_RESULTS);
             });
 
             it('should return `unlimited` value', async () => {
-                expect(_getLimit(undefined,true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT);
+                expect(_getLimit(undefined, true)).toEqual(DEFAULTS.GET_ALL_PIPELINES_LIMIT);
             });
         });
 
@@ -84,6 +90,69 @@ describe('pipeline', () => {
                     });
                 });
 
+                describe('run', () => {
+                    it('should handle running pipeline with encrypted variables', async () => {
+                        const argv = { name: 'some name',
+                            detach: true,
+                            annotation: [],
+                            variable: [
+                                'secret=secret',
+                                'VAR1=VAL1',
+                            ],
+                            encrypted: ['secret'],
+                        };
+                        const pip = new CfPipeline(argv);
+                        await pip.run();
+                        expect(pip.executionRequests[0].options.variables).toEqual([
+                            {
+                                key: 'secret',
+                                value: 'secret',
+                                encrypted: true,
+                            },
+                            {
+                                key: 'VAR1',
+                                value: 'VAL1',
+                            },
+                        ]);
+                        await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+                    });
+
+                    it('should handle running pipeline with encrypted variables passing inside json file', async () => {
+                        const rawFile = fs.readFileSync('lib/interface/cli/commands/pipeline/test-files/var.json', 'utf8');
+
+                        const argv = { name: 'some name',
+                            detach: true,
+                            annotation: [],
+                            'var-file': JSON.parse(rawFile),
+                        };
+                        const pip = new CfPipeline(argv);
+                        await pip.run();
+                        expect(pip.executionRequests[0].options.variables).toEqual(
+                            [{ key: 'help6', value: '85858' },
+                                { key: 'should_be_encrepted', value: '0000' },
+                                { encrypted: true, key: 'help7', value: 'test' }],
+                        );
+                        await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+                    });
+
+                    it('should handle running pipeline with encrypted variables passing inside yaml file', async () => {
+                        const rawFile = fs.readFileSync('lib/interface/cli/commands/pipeline//test-files/var.yml', 'utf8');
+
+                        const argv = { name: 'some name',
+                            detach: true,
+                            annotation: [],
+                            'var-file': yaml.safeLoad(rawFile),
+                        };
+                        const pip = new CfPipeline(argv);
+                        await pip.run();
+                        expect(pip.executionRequests[0].options.variables).toEqual(
+                            [{ key: 'VAR1', value: 'VAL1' },
+                                { encrypted: true, key: 'VAR2', value: 'VAL2' }],
+                        );
+                        await verifyResponsesReturned([DEFAULT_RESPONSE]); // eslint-disable-line
+                    });
+                });
+
                 describe('runImpl', () => {
                     it('should handle running pipeline', async () => {
                         const argv = { name: 'some name', detach: true };
diff --git a/lib/interface/cli/commands/pipeline/run.base.js b/lib/interface/cli/commands/pipeline/run.base.js
@@ -1,10 +1,16 @@
 const _ = require('lodash');
 const Promise = require('bluebird');
-const { prepareKeyValueFromCLIEnvOption } = require('../../helpers/general');
+const CFError = require('cf-errors');
+const { prepareKeyValueFromCLIEnvOption,
+    markEncryptedFlagOnRequestedVariables,
+    prepareKeyValueObjectsFromEnvFileOption,
+    prepareKeyValueObjectsFromCLIEnvOption,
+    isCompatibleApiVersion,
+} = require('../../helpers/general');
 const { validatePipelineYaml } = require('../../helpers/validation');
 const { printResult } = require('../root/validate.cmd');
-const CFError = require('cf-errors');
 const { sdk } = require('../../../../logic');
+const defaults = require('../../defaults');
 
 class RunBaseCommand {
     constructor(argv) {
@@ -55,22 +61,34 @@ class RunBaseCommand {
                 packName,
             },
         };
-
+        const encryptedVarsSupported = await isCompatibleApiVersion({
+            supportedVersion: defaults.MIN_API_VERSION_FOR_ENCRYPTED_VARS_SUPPORT_IN_RUN_CMD,
+        });
         if (variablesFromFile) {
             _.forEach(variablesFromFile, (variables) => {
                 const request = _.cloneDeep(executionRequestTemplate);
-                request.options.variables = variables;
+                if (encryptedVarsSupported) {
+                    request.options.variables = prepareKeyValueObjectsFromEnvFileOption(variables);
+                } else {
+                    request.options.variables = variables;
+                }
                 this.executionRequests.push(request);
             });
         } else {
-            const variables = prepareKeyValueFromCLIEnvOption(this.argv.variable);
+            let variables;
+            if (encryptedVarsSupported) {
+                const varsArr = prepareKeyValueObjectsFromCLIEnvOption(this.argv.variable);
+                variables = markEncryptedFlagOnRequestedVariables(varsArr, this.argv.encrypted);
+            } else {
+                variables = prepareKeyValueFromCLIEnvOption(this.argv.variable);
+            }
             const request = _.cloneDeep(executionRequestTemplate);
             request.options.variables = variables;
             request.options.contexts = contexts;
             this.executionRequests.push(request);
         }
 
-        const results = await Promise.all(this.executionRequests.map(request => this.runImpl(request)));
+        const results = await Promise.all(this.executionRequests.map((request) => this.runImpl(request)));
         const findMaxReducer = (accumulator, currentValue) => (currentValue > accumulator ? currentValue : accumulator);
         const exitCode = results.reduce(findMaxReducer);
         await this.postRunRequest();
diff --git a/lib/interface/cli/commands/pipeline/run.cmd.js b/lib/interface/cli/commands/pipeline/run.cmd.js
@@ -1,8 +1,9 @@
 const debug = require('debug')('codefresh:cli:run:pipeline');
 const Command = require('../../Command');
-const { crudFilenameOption } = require('../../helpers/general');
+const { crudFilenameOption, isCompatibleApiVersion } = require('../../helpers/general');
 const RunLocalCommand = require('./run.local');
 const RunExternalCommand = require('./run.cf');
+const defaults = require('../../defaults');
 
 function getCommandFlavor(argv) {
     if (argv.local) {
@@ -86,6 +87,22 @@ const run = new Command({
                 default: [],
                 alias: 'v',
             })
+            .option('encrypted', {
+                array: true,
+                alias: 'e',
+                describe: 'Variable names to encrypt',
+                default: [],
+            })
+            .check(async (argv) => {
+                const encryptedVarsSupported = await isCompatibleApiVersion({
+                    supportedVersion: defaults.MIN_API_VERSION_FOR_ENCRYPTED_VARS_SUPPORT_IN_RUN_CMD,
+                });
+                if (!encryptedVarsSupported && argv.encrypted.length > 0) {
+                    throw new Error('The "encrypted" option is only supported in API versions '
+                        + `equal to or greater than ${defaults.MIN_API_VERSION_FOR_ENCRYPTED_VARS_SUPPORT_IN_RUN_CMD}.`);
+                }
+                return true;
+            })
             .option('detach', {
                 alias: 'd',
                 describe: 'Run pipeline and print build ID',
@@ -126,6 +143,7 @@ const run = new Command({
             .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master', 'Defining the source control context using a branch')
             .example('codefresh run PIPELINE_ID | PIPELINE_NAME -s=52b992e783d2f84dd0123c70ac8623b4f0f938d1', 'Defining the source control context using a commit')
             .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2', 'Setting variables through the command')
+            .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2 -e key1', 'Setting variables through the command with encrypted option')
             .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master --var-file ./var_file.yml', 'Settings variables through a yml file')
             .example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master --context context', 'Inject contexts to the pipeline execution')
             .example('codefresh run PIPELINE_ID | PIPELINE_NAME --skip step1 step2 step3', 'Skip specific steps');
diff --git a/lib/interface/cli/commands/pipeline/test-files/var.json b/lib/interface/cli/commands/pipeline/test-files/var.json
@@ -0,0 +1,12 @@
+{
+  "build1": {
+    "help6": "85858",
+    "should_be_encrepted": "0000",
+    "help7": {
+      "value": "test",
+      "encrypted": true
+    }
+  }
+}
+
+
diff --git a/lib/interface/cli/commands/pipeline/test-files/var.yml b/lib/interface/cli/commands/pipeline/test-files/var.yml
@@ -0,0 +1,5 @@
+build1:
+  VAR1: 'VAL1'
+  VAR2:
+    value: VAL2
+    encrypted: true
diff --git a/lib/interface/cli/commands/project/apply.cmd.js b/lib/interface/cli/commands/project/apply.cmd.js
@@ -4,7 +4,7 @@ const _ = require('lodash');
 const { sdk } = require('../../../../logic');
 
 const applyRoot = require('../root/apply.cmd');
-const { prepareKeyValueObjectsFromCLIEnvOption, ignoreHttpError } = require('../../helpers/general');
+const { prepareKeyValueObjectsFromCLIEnvOption, ignoreHttpError, markEncryptedFlagOnRequestedVariables } = require('../../helpers/general');
 
 const command = new Command({
     command: 'project [id|name]',
@@ -61,14 +61,7 @@ const command = new Command({
             encrypted,
         } = argv;
 
-        const variableMap = _.reduce(variables, (acc, v) => _.assign(acc, { [v.key]: v }), {});
-        _.forEach(encrypted, (varName) => {
-            const variable = variableMap[varName];
-            if (!variable) {
-                throw new CFError(`Variable is not provided: "${varName}"`);
-            }
-            variable.encrypted = true;
-        });
+        const requestedProjectVariables = markEncryptedFlagOnRequestedVariables(variables, encrypted);
 
         let project = await sdk.projects.get({ id }).catch(ignoreHttpError);
         project = project || await sdk.projects.getByName({ name }).catch(ignoreHttpError);
@@ -81,7 +74,7 @@ const command = new Command({
         const updatePayload = _.pickBy({
             projectName,
             tags: tags || existingTags,
-            variables: variables || existingVariables,
+            variables: requestedProjectVariables || existingVariables,
         }, _.identity);
 
         await sdk.projects.patch({ id: project.id }, updatePayload);
diff --git a/lib/interface/cli/commands/project/create.cmd.js b/lib/interface/cli/commands/project/create.cmd.js
diff --git a/lib/interface/cli/defaults.js b/lib/interface/cli/defaults.js
diff --git a/lib/interface/cli/helpers/general.js b/lib/interface/cli/helpers/general.js
diff --git a/package.json b/package.json
