feat: ✨ Introduce dataFromWithOptions (#846)

dataFromWithOptions allows to get values in bulk of a specific version

Author: adutchak-x

README.md

@@ -512,7 +512,28 @@ spec:
     - hello-service/credentials
 ```
 
-`data` and `dataFrom` can of course be combined, any naming conflicts will use the last defined, with `data` overriding `dataFrom`
+`dataFrom` by default retrieves the latest (`AWSCURRENT`) version of the backend secret, if you want to get values in bulk of a specific version, you can use `dataFromWithOptions`:
+
+```yml
+apiVersion: kubernetes-client.io/v1
+kind: ExternalSecret
+metadata:
+  name: hello-service
+spec:
+  backendType: secretsManager
+  # optional: specify role to assume when retrieving the data
+  roleArn: arn:aws:iam::123456789012:role/test-role
+  # optional: specify region
+  region: us-east-1
+  dataFromWithOptions:
+    - key: hello-service/credentials
+      versionStage: AWSPREVIOUS
+    - key: hello-service/credentials
+      versionId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+```
+
+`data`, `dataFrom` and `dataFromWithOptions` can of course be combined, any naming conflicts will use the last defined.   
+In the below example `data` takes precedence over `dataFromWithOptions` and `dataFrom`.
 
 ```yml
 apiVersion: kubernetes-client.io/v1
@@ -527,6 +548,9 @@ spec:
   region: us-east-1
   dataFrom:
     - hello-service/credentials
+  dataFromWithOptions:
+    - key: hello-service/credentials
+      versionId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   data:
     - key: hello-service/migration-credentials
       name: password

charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml

@@ -73,6 +73,31 @@ spec:
                   type: array
                   items:
                     type: string
+                dataFromWithOptions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      key:
+                        description: Secret key in backend
+                        type: string
+                      isBinary:
+                        description: >-
+                          Whether the backend secret shall be treated as binary data
+                          represented by a base64-encoded string. You must set this to true
+                          for any base64-encoded binary data in the backend - to ensure it
+                          is not encoded in base64 again. Default is false.
+                        type: boolean
+                      versionStage:
+                        description: >-
+                          Used by: alicloudSecretsManager, secretsManager
+                        type: string
+                      versionId:
+                        description: >-
+                          Used by: secretsManager
+                        type: string
+                    required:
+                    - key
                 data:
                   type: array
                   items:
@@ -180,6 +205,8 @@ spec:
                     - data
                 - required:
                     - dataFrom
+                - required:
+                    - dataFromWithOptions
             status:
               type: object
               properties:

lib/backends/kv-backend.js

@@ -112,6 +112,26 @@ class KVBackend extends AbstractBackend {
     }))
   }
 
+  /**
+   * Fetch Kubernetes secret property values with options.
+   * @param {Object[]} dataFromWithOptions - Array of secret keys in the backend, including extra options
+   * @param {string} specOptions - Options set on spec level that might be interesting for the backend
+   * @returns {Promise} Promise object representing secret property values.
+   */
+  _fetchDataFromValuesWithOptions ({ dataFromWithOptions, specOptions }) {
+    return Promise.all(dataFromWithOptions.map(async dataItem => {
+      const { key, ...keyOptions } = dataItem
+      const value = await this._get({ key, specOptions, keyOptions })
+
+      try {
+        return JSON.parse(value)
+      } catch (err) {
+        this._logger.warn(`Failed to JSON.parse value for '${dataItem}',` +
+          ' please verify that your secret value is correctly formatted as JSON.')
+      }
+    }))
+  }
+
   /**
    * Get a secret property value from Key Value backend.
    * @param {string} key - Secret key in the backend.
@@ -162,15 +182,17 @@ class KVBackend extends AbstractBackend {
       properties = [],
       data = properties,
       dataFrom = [],
+      dataFromWithOptions = [],
       ...specOptions
     }
   }) {
-    const [dataFromValues, dataValues] = await Promise.all([
+    const [dataFromValues, dataFromValuesWithOptions, dataValues] = await Promise.all([
       this._fetchDataFromValues({ dataFrom, specOptions }),
+      this._fetchDataFromValuesWithOptions({ dataFromWithOptions, specOptions }),
       this._fetchDataValues({ data, specOptions })
     ])
 
-    const plainValues = dataFromValues.concat(dataValues)
+    const plainValues = dataFromValues.concat(dataFromValuesWithOptions).concat(dataValues)
       .reduce((acc, parsedValue) => ({
         ...acc,
         ...parsedValue

lib/backends/kv-backend.test.js

@@ -267,6 +267,43 @@ describe('kv-backend', () => {
     })
   })
 
+  describe('_fetchDataFromValuesWithOptions', () => {
+    beforeEach(() => {
+      kvBackend._get = sinon.stub()
+    })
+    it('handles secrets with spec and key options', async () => {
+      kvBackend._get.onFirstCall().resolves('{"fakePropertyName1":"fakePropertyValue1"}')
+      kvBackend._get.onSecondCall().resolves('{"fakePropertyName2":"fakePropertyValue2"}')
+      const dataFromValuesWithOptions = await kvBackend._fetchDataFromValuesWithOptions({
+        dataFromWithOptions: [{ versionStage: 'AWSCURRENT', key: 'fakeKey1' },
+          { versionId: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx', key: 'fakeKey2' }],
+        specOptions: { passMeAlong: true }
+      })
+      expect(dataFromValuesWithOptions).to.deep.equal([{ fakePropertyName1: 'fakePropertyValue1' }, { fakePropertyName2: 'fakePropertyValue2' }])
+      expect(kvBackend._get.getCall(0).args[0]).to.deep.equal({
+        key: 'fakeKey1',
+        keyOptions: {
+          versionStage: 'AWSCURRENT'
+        },
+        specOptions: { passMeAlong: true }
+      })
+      expect(kvBackend._get.getCall(1).args[0]).to.deep.equal({
+        key: 'fakeKey2',
+        keyOptions: {
+          versionId: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
+        },
+        specOptions: { passMeAlong: true }
+      })
+    })
+    it('handles invalid JSON objects', async () => {
+      kvBackend._get.onFirstCall().resolves('{')
+      const dataFromValuesWithOptions = await kvBackend._fetchDataFromValuesWithOptions({
+        dataFromWithOptions: [{ key: 'fakeKey1' }]
+      })
+      expect(dataFromValuesWithOptions).to.deep.equal([undefined])
+    })
+  })
+
   describe('_get', () => {
     it('throws an error', () => {
       let error