fix: delete temporary resources (#55)

fix: delete temporary resources

Refs: #54

---------

Signed-off-by: seven <[email protected]>

Author: Blankll

src/common/constants.ts

@@ -1 +1,2 @@
 export const CODE_ZIP_SIZE_LIMIT = 15 * 1000 * 1000;
+export const OSS_DEPLOYMENT_TIMEOUT = 3000; // in seconds

src/common/rosAssets.ts

@@ -65,7 +65,16 @@ const zipAssets = async (assetsPath: string) => {
   return zipPath;
 };
 
-const constructAssets = async ({ files, rootPath }: CdkAssets, region: string) => {
+type ConstructedAsset = {
+  bucketName: string;
+  source: string;
+  objectKey: string;
+};
+
+export const constructAssets = async (
+  { files, rootPath }: CdkAssets,
+  region: string,
+): Promise<Array<ConstructedAsset> | undefined> => {
   const assets = await Promise.all(
     Object.entries(files)
       .filter(([, fileItem]) => !fileItem.source.path.endsWith('.template.json'))
@@ -103,15 +112,16 @@ const ensureBucketExits = async (bucketName: string, ossClient: OSS) =>
     }
   });
 
-export const publishAssets = async (assets: CdkAssets, context: ActionContext) => {
-  const constructedAssets = await constructAssets(assets, context.region);
-
-  if (!constructedAssets?.length) {
+export const publishAssets = async (
+  assets: Array<ConstructedAsset> | undefined,
+  context: ActionContext,
+) => {
+  if (!assets?.length) {
     logger.info('No assets to publish, skipped!');
     return;
   }
 
-  const bucketName = constructedAssets[0].bucketName;
+  const bucketName = assets[0].bucketName;
 
   const client = new OSS({
     region: `oss-${context.region}`,
@@ -129,11 +139,40 @@ export const publishAssets = async (assets: CdkAssets, context: ActionContext) =
   } as OSS.PutObjectOptions;
 
   await Promise.all(
-    constructedAssets.map(async ({ source, objectKey }) => {
+    assets.map(async ({ source, objectKey }) => {
       await client.put(objectKey, path.normalize(source), { headers });
-      logger.info(`Upload file: ${source}) to bucket: ${bucketName} successfully!`);
+      logger.debug(`Upload file: ${source} to bucket: ${bucketName} successfully!`);
     }),
   );
 
   return bucketName;
 };
+
+export const cleanupAssets = async (
+  assets: Array<ConstructedAsset> | undefined,
+  context: ActionContext,
+) => {
+  if (!assets?.length) {
+    logger.info('No assets to cleanup, skipped!');
+    return;
+  }
+
+  const bucketName = assets[0].bucketName;
+
+  const client = new OSS({
+    region: `oss-${context.region}`,
+    accessKeyId: context.accessKeyId,
+    accessKeySecret: context.accessKeySecret,
+    bucket: bucketName,
+  });
+
+  await Promise.all(
+    assets.map(async ({ objectKey }) => {
+      await client.delete(objectKey);
+      logger.debug(`Cleanup file: ${objectKey} from bucket: ${bucketName} successfully!`);
+    }),
+  );
+  // delete the bucket
+  await client.deleteBucket(bucketName);
+  logger.debug(`Cleanup bucket: ${bucketName} successfully!`);
+};

src/common/rosClient.ts

@@ -190,7 +190,7 @@ export const rosStackDelete = async ({
     });
     await client.deleteStack(deleteStackRequest);
     await getStackActionResult(stackInfo.stackId as string, region);
-    logger.info(`Stack: ${stackName} deleted! ♻️`);
+    logger.info(`Stack: ${stackName} deleted!🗑 `);
   } catch (err) {
     logger.error(`Stack: ${stackName} delete failed! ❌, error: ${JSON.stringify(err)}`);
     throw new Error(JSON.stringify(err));

src/stack/deploy.ts

@@ -2,7 +2,14 @@ import * as ros from '@alicloud/ros-cdk-core';
 import fs from 'node:fs';
 
 import { ActionContext, ServerlessIac } from '../types';
-import { logger, ProviderEnum, publishAssets, rosStackDeploy } from '../common';
+import {
+  cleanupAssets,
+  constructAssets,
+  logger,
+  ProviderEnum,
+  publishAssets,
+  rosStackDeploy,
+} from '../common';
 import { prepareBootstrapStack, RosStack } from './rosStack';
 import { RfsStack } from './rfsStack';
 import { get } from 'lodash';
@@ -50,10 +57,25 @@ export const deployStack = async (
   const { template, assets } = generateRosStackTemplate(stackName, iac, context);
   await prepareBootstrapStack(context);
   logger.info(`Deploying stack, publishing assets...`);
-  await publishAssets(assets, context);
-  logger.info(`Assets published! 🎉`);
-  await rosStackDeploy(stackName, template, context);
-  logger.info(`Stack deployed! 🎉`);
+  const constructedAssets = await constructAssets(assets, context.region);
+  try {
+    await publishAssets(constructedAssets, context);
+    logger.info(`Assets published! 🎉`);
+    await rosStackDeploy(stackName, template, context);
+  } catch (e) {
+    logger.error(`Failed to deploy stack: ${e}`);
+    throw e;
+  } finally {
+    try {
+      logger.info(`Cleaning up temporary Assets...`);
+      await cleanupAssets(constructedAssets, context);
+      logger.info(`Assets cleaned up!♻️`);
+    } catch (e) {
+      logger.error(
+        `Failed to cleanup assets, it wont affect the deployment result, but to avoid potential cost, you can delete the temporary bucket : ${constructedAssets?.[0].bucketName}, error details:${e}`,
+      );
+    }
+  }
 };
 
 export const generateStackTemplate = (

src/stack/rosStack/bucket.ts

@@ -1,7 +1,13 @@
 import { ActionContext, BucketAccessEnum, BucketDomain } from '../../types';
 import * as oss from '@alicloud/ros-cdk-oss';
 import * as ros from '@alicloud/ros-cdk-core';
-import { encodeBase64ForRosId, getAssets, replaceReference, splitDomain } from '../../common';
+import {
+  encodeBase64ForRosId,
+  getAssets,
+  OSS_DEPLOYMENT_TIMEOUT,
+  replaceReference,
+  splitDomain,
+} from '../../common';
 import * as ossDeployment from '@alicloud/ros-cdk-ossdeployment';
 import * as dns from '@alicloud/ros-cdk-dns';
 import path from 'node:path';
@@ -80,7 +86,7 @@ export const resolveBuckets = (
           sources: getAssets(filePath),
           destinationBucket: ossBucket.attrName,
           roleArn: siAutoOssDeploymentBucketRole!.attrArn,
-          timeout: 3000,
+          timeout: OSS_DEPLOYMENT_TIMEOUT,
           logMonitoring: false,
           retainOnCreate: false,
         },

src/stack/rosStack/function.ts

@@ -9,6 +9,7 @@ import {
   CODE_ZIP_SIZE_LIMIT,
   encodeBase64ForRosId,
   getFileSource,
+  OSS_DEPLOYMENT_TIMEOUT,
   readCodeSize,
   replaceReference,
   resolveCode,
@@ -143,7 +144,7 @@ export const resolveFunctions = (
       {
         sources: fileSources!.map(({ source }) => source),
         destinationBucket: destinationBucketName,
-        timeout: 3000,
+        timeout: OSS_DEPLOYMENT_TIMEOUT,
         logMonitoring: false,
         retainOnCreate: false,
       },

tests/common/rosAssets.test.ts

@@ -1,19 +1,25 @@
-import { getAssets, publishAssets } from '../../src/common';
+import { cleanupAssets, constructAssets, getAssets, publishAssets } from '../../src/common';
 import { Stats } from 'node:fs';
 import { assetsFixture } from '../fixtures/assetsFixture';
-import { ActionContext, CdkAssets } from '../../src/types';
+import { ActionContext } from '../../src/types';
+import { defaultContext } from '../fixtures/deployFixture';
 
 const mockedBucketPut = jest.fn();
+const mockedDeleteBucket = jest.fn();
+const mockedDelete = jest.fn();
 const mockedReaddirSync = jest.fn();
 const mockedLstatSync = jest.fn();
 const mockedExistsSync = jest.fn();
 const mockedGenerateAsync = jest.fn();
 const mockedInfoLogger = jest.fn();
+const mockedDebugLogger = jest.fn();
 
 jest.mock('ali-oss', () =>
   jest.fn().mockImplementation(() => ({
     getBucketInfo: jest.fn().mockResolvedValue({}),
     put: (...args: unknown[]) => mockedBucketPut(...args),
+    delete: (...args: unknown[]) => mockedDelete(...args),
+    deleteBucket: (...args: unknown[]) => mockedDeleteBucket(...args),
   })),
 );
 
@@ -43,14 +49,15 @@ jest.mock('jszip', () =>
 jest.mock('../../src/common/logger', () => ({
   logger: {
     info: (...args: unknown[]) => mockedInfoLogger(...args),
+    debug: (...args: unknown[]) => mockedDebugLogger(...args),
   },
 }));
 
 describe('Unit test for rosAssets', () => {
   beforeEach(() => {
     jest.clearAllMocks();
   });
-  describe('Unite test for getAssets', () => {
+  describe('Unit test for getAssets', () => {
     it('should return assets from the specified location', () => {
       const mockLocation = 'mock/location';
       mockedExistsSync.mockReturnValue(true);
@@ -78,7 +85,7 @@ describe('Unit test for rosAssets', () => {
     });
   });
 
-  describe('publishAssets', () => {
+  describe('Unit test for publishAssets', () => {
     const mockContext = {
       region: 'mock-region',
       accessKeyId: 'mock-access-key-id',
@@ -93,7 +100,10 @@ describe('Unit test for rosAssets', () => {
         .mockReturnValueOnce({ isFile: () => true } as Stats);
       mockedGenerateAsync.mockResolvedValue(Buffer.from('mock-zip-content'));
 
-      const bucketName = await publishAssets(assetsFixture, mockContext);
+      const bucketName = await publishAssets(
+        await constructAssets(assetsFixture, 'mock-region'),
+        mockContext,
+      );
 
       expect(bucketName).toBe('cdk-ajmywduza-assets-mock-region');
       expect(mockedBucketPut.mock.calls).toEqual([
@@ -112,21 +122,40 @@ describe('Unit test for rosAssets', () => {
         [
           'Folder compressed to: path/to/asset.55d1d2dd5d6c1b083a04c15431f70da1f2840b9de06383411cbf7eda2a512efe.zip',
         ],
-        [
-          'Upload file: path/to/asset.55d1d2dd5d6c1b083a04c15431f70da1f2840b9de06383411cbf7eda2a512efe.zip) to bucket: cdk-ajmywduza-assets-mock-region successfully!',
-        ],
-        [
-          'Upload file: path/to/asset.c6a72ed7e7e83f01a000b75885758088fa050298a31a1e95d37ac88f08e42315.zip) to bucket: cdk-ajmywduza-assets-mock-region successfully!',
-        ],
       ]);
     });
 
     it('should log and skip if no assets to publish', async () => {
-      const emptyAssets = { files: {} } as unknown as CdkAssets;
-
-      await publishAssets(emptyAssets, mockContext);
+      await publishAssets([], mockContext);
 
       expect(mockedInfoLogger).toHaveBeenCalledWith('No assets to publish, skipped!');
     });
   });
+
+  describe('Unit test for cleanupAssets', () => {
+    it('should cleanup and delete the bucket when given assets is valid', async () => {
+      mockedExistsSync.mockReturnValue(true);
+      mockedReaddirSync.mockReturnValueOnce(['file1', 'file2']);
+      mockedLstatSync
+        .mockReturnValueOnce({ isFile: () => true } as Stats)
+        .mockReturnValueOnce({ isFile: () => true } as Stats);
+      mockedGenerateAsync.mockResolvedValue(Buffer.from('mock-zip-content'));
+
+      await cleanupAssets(await constructAssets(assetsFixture, 'mock-region'), defaultContext);
+
+      expect(mockedDelete).toHaveBeenCalledTimes(2);
+      expect(mockedDelete.mock.calls).toEqual([
+        ['55d1d2dd5d6c1b083a04c15431f70da1f2840b9de06383411cbf7eda2a512efe.zip'],
+        ['c6a72ed7e7e83f01a000b75885758088fa050298a31a1e95d37ac88f08e42315.zip'],
+      ]);
+      expect(mockedDeleteBucket).toHaveBeenCalledTimes(1);
+      expect(mockedDeleteBucket).toHaveBeenCalledWith('cdk-ajmywduza-assets-mock-region');
+    });
+
+    it('should skip the cleanupAssets when there is no assets', async () => {
+      await cleanupAssets([], defaultContext);
+
+      expect(mockedInfoLogger).toHaveBeenCalledWith('No assets to cleanup, skipped!');
+    });
+  });
 });

tests/common/rosClient.test.ts

@@ -102,7 +102,7 @@ describe('Unit test for rosClient', () => {
       await rosStackDelete(context);
 
       expect(logger.info).toHaveBeenCalledWith('stack status: DELETE_COMPLETE');
-      expect(logger.info).toHaveBeenCalledWith('Stack: testStack deleted! ♻️');
+      expect(logger.info).toHaveBeenCalledWith('Stack: testStack deleted!🗑 ');
     });
 
     it('should throw an error when the stack does not exist', async () => {

tests/fixtures/deployFixture.ts

@@ -1476,7 +1476,7 @@ export const defaultContext = {
   iacLocation: expect.stringContaining('tests/fixtures/serverless-insight.yml'),
   parameters: [],
   region: 'cn-hangzhou',
-  provider: 'aliyun',
+  provider: ProviderEnum.ALIYUN,
   securityToken: 'account id',
   stackName: 'my-demo-stack',
   stage: 'default',

tests/stack/deploy.test.ts

@@ -30,14 +30,17 @@ import { cloneDeep, set } from 'lodash';
 const mockedRosStackDeploy = jest.fn();
 const mockedResolveCode = jest.fn();
 const mockedPublishAssets = jest.fn();
+const mockedCleanupAssets = jest.fn();
 const mockedGetIamInfo = jest.fn();
 
 jest.mock('../../src/common', () => ({
   ...jest.requireActual('../../src/common'),
   rosStackDeploy: (...args: unknown[]) => mockedRosStackDeploy(...args),
   publishAssets: (...args: unknown[]) => mockedPublishAssets(...args),
+  cleanupAssets: (...args: unknown[]) => mockedCleanupAssets(...args),
   resolveCode: (path: string) => mockedResolveCode(path),
   getIamInfo: (...args: unknown[]) => mockedGetIamInfo(...args),
+  logger: { info: jest.fn(), debug: jest.fn() },
 }));
 
 describe('Unit tests for stack deployment', () => {
@@ -51,6 +54,7 @@ describe('Unit tests for stack deployment', () => {
     mockedResolveCode.mockRestore();
     mockedPublishAssets.mockRestore();
     mockedGetIamInfo.mockRestore();
+    mockedCleanupAssets.mockRestore();
   });
 
   it('should deploy generated stack when minimum fields provided', async () => {
@@ -133,25 +137,23 @@ describe('Unit tests for stack deployment', () => {
       { stackName } as ActionContext,
     );
 
+    const expectedAssets = Array(2).fill({
+      bucketName: expect.any(String),
+      objectKey: expect.stringContaining('.zip'),
+      source: expect.stringContaining('.zip'),
+    });
     expect(mockedPublishAssets).toHaveBeenCalledTimes(1);
     expect(mockedRosStackDeploy).toHaveBeenCalledTimes(2);
-    expect(mockedPublishAssets).toHaveBeenCalledWith(
-      expect.objectContaining({
-        dockerImages: {},
-        files: expect.any(Object),
-        rootPath: expect.any(String),
-        version: '7.0.0',
-      }),
-
-      { stackName },
-    );
+    expect(mockedPublishAssets).toHaveBeenCalledWith(expectedAssets, { stackName });
     expect(mockedRosStackDeploy.mock.calls[1]).toEqual([
       stackName,
       largeCodeRos,
       {
         stackName,
       },
     ]);
+    expect(mockedCleanupAssets).toHaveBeenCalledTimes(1);
+    expect(mockedCleanupAssets).toHaveBeenCalledWith(expectedAssets, { stackName });
   });
 
   describe('unit test for deploy of events', () => {
@@ -211,6 +213,7 @@ describe('Unit tests for stack deployment', () => {
 
       expect(mockedRosStackDeploy).toHaveBeenCalledTimes(2);
       expect(mockedPublishAssets).toHaveBeenCalledTimes(1);
+      expect(mockedCleanupAssets).toHaveBeenCalledTimes(1);
       expect(mockedRosStackDeploy.mock.calls[1]).toEqual([
         stackName,
         bucketWithWebsiteRos,