paginate getParameter call when there are more than 10 parameters (#2480)

* paginate getParameter call when there are more than 10 parameters

* update changeset

* remove file not needed

Author: Amplifiyer

.changeset/famous-boats-wait.md

@@ -0,0 +1,6 @@
+---
+'@aws-amplify/backend-function': patch
+'@aws-amplify/backend': patch
+---
+
+paginate getParameter call when there are more than 10 parameters

package-lock.json

@@ -1,5 +1,5 @@
-import { SSM } from '@aws-sdk/client-ssm';
-import { after, describe, it, mock } from 'node:test';
+import { GetParametersCommandInput, SSM } from '@aws-sdk/client-ssm';
+import { afterEach, describe, it, mock } from 'node:test';
 import { internalAmplifyFunctionResolveSsmParams } from './resolve_ssm_params.js';
 import assert from 'node:assert';
 
@@ -8,7 +8,7 @@ void describe('internalAmplifyFunctionResolveSsmParams', () => {
   const client = new SSM();
 
   // reset process.env after test suite to ensure there are no side effects
-  after(() => {
+  afterEach(() => {
     process.env = originalEnv;
   });
 
@@ -43,4 +43,42 @@ void describe('internalAmplifyFunctionResolveSsmParams', () => {
     assert.equal(mockGetParameters.mock.callCount(), 1);
     assert.equal(process.env[envName], secretValue);
   });
+
+  void it('paginates when there are more than 10 secrets', async () => {
+    const envName = 'TEST_SECRET';
+    const secretPath = '/test/path';
+    let ssmPaths = {};
+    for (let i = 0; i < 100; i++) {
+      ssmPaths = Object.assign(ssmPaths, {
+        [secretPath + i]: {
+          name: envName + i,
+          sharedSecretPath: '/test/shared/path',
+        },
+      });
+    }
+    process.env.AMPLIFY_SSM_ENV_CONFIG = JSON.stringify(ssmPaths);
+
+    // Let's return the value same as args
+    const mockGetParameters = mock.method(
+      client,
+      'getParameters',
+      (args: GetParametersCommandInput) => {
+        const parameters: unknown[] = [];
+        args.Names?.forEach((name) => {
+          parameters.push({
+            Name: name,
+            Value: name,
+          });
+        });
+        return Promise.resolve({
+          Parameters: parameters,
+        });
+      }
+    );
+    await internalAmplifyFunctionResolveSsmParams(client);
+    assert.equal(mockGetParameters.mock.callCount(), 10);
+    for (let i = 0; i < 100; i++) {
+      assert.equal(process.env[envName + i], secretPath + i);
+    }
+  });
 });

packages/backend-function/src/lambda-shims/resolve_ssm_params.test.ts

@@ -2,7 +2,7 @@
  * This code loads environment values from SSM and places them in their corresponding environment variables.
  * If there are no SSM environment values for this function, this is a noop.
  */
-import type { SSM } from '@aws-sdk/client-ssm';
+import type { GetParametersCommandOutput, SSM } from '@aws-sdk/client-ssm';
 import type { SsmEnvVars } from '../function_env_translator.js';
 
 /**
@@ -27,11 +27,36 @@ export const internalAmplifyFunctionResolveSsmParams = async (client?: SSM) => {
     actualSsmClient = new ssmSdk.SSM();
   }
 
+  const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+      chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+  };
+
   const resolveSecrets = async (paths: string[]) => {
-    const response = await actualSsmClient.getParameters({
-      Names: paths,
-      WithDecryption: true,
-    });
+    const response = (
+      await Promise.all(
+        chunkArray(paths, 10).map(
+          async (chunkedPaths) =>
+            await actualSsmClient.getParameters({
+              Names: chunkedPaths,
+              WithDecryption: true,
+            })
+        )
+      )
+    ).reduce(
+      (accumulator, res: GetParametersCommandOutput) => {
+        accumulator.Parameters?.push(...(res.Parameters ?? []));
+        accumulator.InvalidParameters?.push(...(res.InvalidParameters ?? []));
+        return accumulator;
+      },
+      {
+        Parameters: [],
+        InvalidParameters: [],
+      } as Partial<GetParametersCommandOutput>
+    );
 
     if (response.Parameters && response.Parameters.length > 0) {
       for (const parameter of response.Parameters) {

packages/backend-function/src/lambda-shims/resolve_ssm_params.ts

@@ -1,4 +1,4 @@
-import aws from 'aws-sdk';
+import aws, { SSM } from 'aws-sdk';
 import { after, describe, it, mock } from 'node:test';
 import { internalAmplifyFunctionResolveSsmParams } from './resolve_ssm_params_sdk_v2.js';
 import assert from 'node:assert';
@@ -45,4 +45,45 @@ void describe('internalAmplifyFunctionResolveSsmParams', () => {
     assert.equal(mockGetParameters.mock.callCount(), 1);
     assert.equal(process.env[envName], secretValue);
   });
+
+  void it('paginates when there are more than 10 secrets', async () => {
+    const envName = 'TEST_SECRET';
+    const secretPath = '/test/path';
+    let ssmPaths = {};
+    for (let i = 0; i < 100; i++) {
+      ssmPaths = Object.assign(ssmPaths, {
+        [secretPath + i]: {
+          name: envName + i,
+          sharedSecretPath: '/test/shared/path',
+        },
+      });
+    }
+    process.env.AMPLIFY_SSM_ENV_CONFIG = JSON.stringify(ssmPaths);
+
+    // Let's return the value same as args
+    const mockGetParameters = mock.method(
+      client,
+      'getParameters',
+      (args: SSM.Types.GetParametersRequest) => {
+        const parameters: unknown[] = [];
+        args.Names?.forEach((name) => {
+          parameters.push({
+            Name: name,
+            Value: name,
+          });
+        });
+        return {
+          promise: () =>
+            Promise.resolve({
+              Parameters: parameters,
+            }),
+        };
+      }
+    );
+    await internalAmplifyFunctionResolveSsmParams(client);
+    assert.equal(mockGetParameters.mock.callCount(), 10);
+    for (let i = 0; i < 100; i++) {
+      assert.equal(process.env[envName + i], secretPath + i);
+    }
+  });
 });

packages/backend-function/src/lambda-shims/resolve_ssm_params_sdk_v2.test.ts

@@ -36,13 +36,40 @@ export const internalAmplifyFunctionResolveSsmParams = async (
     return;
   }
 
+  const chunkArray = <T>(array: T[], chunkSize: number): T[][] => {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+      chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+  };
+
   const resolveSecrets = async (paths: string[]) => {
-    const response = await client
-      .getParameters({
-        Names: paths,
-        WithDecryption: true,
-      })
-      .promise();
+    const response = (
+      await Promise.all(
+        chunkArray(paths, 10).map(
+          async (chunkedPaths) =>
+            await client
+              .getParameters({
+                Names: chunkedPaths,
+                WithDecryption: true,
+              })
+              .promise()
+        )
+      )
+    ).reduce(
+      (accumulator, response) => {
+        accumulator.Parameters?.push(...(response.Parameters ?? []));
+        accumulator.InvalidParameters?.push(
+          ...(response.InvalidParameters ?? [])
+        );
+        return accumulator;
+      },
+      {
+        Parameters: [],
+        InvalidParameters: [],
+      } as SSM.GetParametersResult
+    );
 
     if (response.Parameters && response.Parameters.length > 0) {
       for (const parameter of response.Parameters) {