diff --git a/src/common/constants.ts b/src/common/constants.ts index 5dbf14e..dfd9af3 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1 +1,2 @@ export const CODE_ZIP_SIZE_LIMIT = 15 * 1000 * 1000; +export const OSS_DEPLOYMENT_TIMEOUT = 3000; // in seconds diff --git a/src/common/rosAssets.ts b/src/common/rosAssets.ts index a49c693..8e77e83 100644 --- a/src/common/rosAssets.ts +++ b/src/common/rosAssets.ts @@ -65,7 +65,16 @@ const zipAssets = async (assetsPath: string) => { return zipPath; }; -const constructAssets = async ({ files, rootPath }: CdkAssets, region: string) => { +type ConstructedAsset = { + bucketName: string; + source: string; + objectKey: string; +}; + +export const constructAssets = async ( + { files, rootPath }: CdkAssets, + region: string, +): Promise | undefined> => { const assets = await Promise.all( Object.entries(files) .filter(([, fileItem]) => !fileItem.source.path.endsWith('.template.json')) @@ -103,15 +112,16 @@ const ensureBucketExits = async (bucketName: string, ossClient: OSS) => } }); -export const publishAssets = async (assets: CdkAssets, context: ActionContext) => { - const constructedAssets = await constructAssets(assets, context.region); - - if (!constructedAssets?.length) { +export const publishAssets = async ( + assets: Array | undefined, + context: ActionContext, +) => { + if (!assets?.length) { logger.info('No assets to publish, skipped!'); return; } - const bucketName = constructedAssets[0].bucketName; + const bucketName = assets[0].bucketName; const client = new OSS({ region: `oss-${context.region}`, @@ -129,11 +139,40 @@ export const publishAssets = async (assets: CdkAssets, context: ActionContext) = } as OSS.PutObjectOptions; await Promise.all( - constructedAssets.map(async ({ source, objectKey }) => { + assets.map(async ({ source, objectKey }) => { await client.put(objectKey, path.normalize(source), { headers }); - logger.info(`Upload file: ${source}) to bucket: ${bucketName} successfully!`); + logger.debug(`Upload file: ${source} to bucket: ${bucketName} successfully!`); }), ); return bucketName; }; + +export const cleanupAssets = async ( + assets: Array | undefined, + context: ActionContext, +) => { + if (!assets?.length) { + logger.info('No assets to cleanup, skipped!'); + return; + } + + const bucketName = assets[0].bucketName; + + const client = new OSS({ + region: `oss-${context.region}`, + accessKeyId: context.accessKeyId, + accessKeySecret: context.accessKeySecret, + bucket: bucketName, + }); + + await Promise.all( + assets.map(async ({ objectKey }) => { + await client.delete(objectKey); + logger.debug(`Cleanup file: ${objectKey} from bucket: ${bucketName} successfully!`); + }), + ); + // delete the bucket + await client.deleteBucket(bucketName); + logger.debug(`Cleanup bucket: ${bucketName} successfully!`); +}; diff --git a/src/common/rosClient.ts b/src/common/rosClient.ts index 586ae75..52d4acf 100644 --- a/src/common/rosClient.ts +++ b/src/common/rosClient.ts @@ -190,7 +190,7 @@ export const rosStackDelete = async ({ }); await client.deleteStack(deleteStackRequest); await getStackActionResult(stackInfo.stackId as string, region); - logger.info(`Stack: ${stackName} deleted! ♻️`); + logger.info(`Stack: ${stackName} deleted!🗑 `); } catch (err) { logger.error(`Stack: ${stackName} delete failed! ❌, error: ${JSON.stringify(err)}`); throw new Error(JSON.stringify(err)); diff --git a/src/stack/deploy.ts b/src/stack/deploy.ts index 06200bb..87e32e3 100644 --- a/src/stack/deploy.ts +++ b/src/stack/deploy.ts @@ -2,7 +2,14 @@ import * as ros from '@alicloud/ros-cdk-core'; import fs from 'node:fs'; import { ActionContext, ServerlessIac } from '../types'; -import { logger, ProviderEnum, publishAssets, rosStackDeploy } from '../common'; +import { + cleanupAssets, + constructAssets, + logger, + ProviderEnum, + publishAssets, + rosStackDeploy, +} from '../common'; import { prepareBootstrapStack, RosStack } from './rosStack'; import { RfsStack } from './rfsStack'; import { get } from 'lodash'; @@ -50,10 +57,25 @@ export const deployStack = async ( const { template, assets } = generateRosStackTemplate(stackName, iac, context); await prepareBootstrapStack(context); logger.info(`Deploying stack, publishing assets...`); - await publishAssets(assets, context); - logger.info(`Assets published! 🎉`); - await rosStackDeploy(stackName, template, context); - logger.info(`Stack deployed! 🎉`); + const constructedAssets = await constructAssets(assets, context.region); + try { + await publishAssets(constructedAssets, context); + logger.info(`Assets published! 🎉`); + await rosStackDeploy(stackName, template, context); + } catch (e) { + logger.error(`Failed to deploy stack: ${e}`); + throw e; + } finally { + try { + logger.info(`Cleaning up temporary Assets...`); + await cleanupAssets(constructedAssets, context); + logger.info(`Assets cleaned up!♻️`); + } catch (e) { + logger.error( + `Failed to cleanup assets, it wont affect the deployment result, but to avoid potential cost, you can delete the temporary bucket : ${constructedAssets?.[0].bucketName}, error details:${e}`, + ); + } + } }; export const generateStackTemplate = ( diff --git a/src/stack/rosStack/bucket.ts b/src/stack/rosStack/bucket.ts index 87e0a7e..e8ce41b 100644 --- a/src/stack/rosStack/bucket.ts +++ b/src/stack/rosStack/bucket.ts @@ -1,7 +1,13 @@ import { ActionContext, BucketAccessEnum, BucketDomain } from '../../types'; import * as oss from '@alicloud/ros-cdk-oss'; import * as ros from '@alicloud/ros-cdk-core'; -import { encodeBase64ForRosId, getAssets, replaceReference, splitDomain } from '../../common'; +import { + encodeBase64ForRosId, + getAssets, + OSS_DEPLOYMENT_TIMEOUT, + replaceReference, + splitDomain, +} from '../../common'; import * as ossDeployment from '@alicloud/ros-cdk-ossdeployment'; import * as dns from '@alicloud/ros-cdk-dns'; import path from 'node:path'; @@ -80,7 +86,7 @@ export const resolveBuckets = ( sources: getAssets(filePath), destinationBucket: ossBucket.attrName, roleArn: siAutoOssDeploymentBucketRole!.attrArn, - timeout: 3000, + timeout: OSS_DEPLOYMENT_TIMEOUT, logMonitoring: false, retainOnCreate: false, }, diff --git a/src/stack/rosStack/function.ts b/src/stack/rosStack/function.ts index ac78371..5a9cb18 100644 --- a/src/stack/rosStack/function.ts +++ b/src/stack/rosStack/function.ts @@ -9,6 +9,7 @@ import { CODE_ZIP_SIZE_LIMIT, encodeBase64ForRosId, getFileSource, + OSS_DEPLOYMENT_TIMEOUT, readCodeSize, replaceReference, resolveCode, @@ -143,7 +144,7 @@ export const resolveFunctions = ( { sources: fileSources!.map(({ source }) => source), destinationBucket: destinationBucketName, - timeout: 3000, + timeout: OSS_DEPLOYMENT_TIMEOUT, logMonitoring: false, retainOnCreate: false, }, diff --git a/tests/common/rosAssets.test.ts b/tests/common/rosAssets.test.ts index 2d5b09e..a5bb0eb 100644 --- a/tests/common/rosAssets.test.ts +++ b/tests/common/rosAssets.test.ts @@ -1,19 +1,25 @@ -import { getAssets, publishAssets } from '../../src/common'; +import { cleanupAssets, constructAssets, getAssets, publishAssets } from '../../src/common'; import { Stats } from 'node:fs'; import { assetsFixture } from '../fixtures/assetsFixture'; -import { ActionContext, CdkAssets } from '../../src/types'; +import { ActionContext } from '../../src/types'; +import { defaultContext } from '../fixtures/deployFixture'; const mockedBucketPut = jest.fn(); +const mockedDeleteBucket = jest.fn(); +const mockedDelete = jest.fn(); const mockedReaddirSync = jest.fn(); const mockedLstatSync = jest.fn(); const mockedExistsSync = jest.fn(); const mockedGenerateAsync = jest.fn(); const mockedInfoLogger = jest.fn(); +const mockedDebugLogger = jest.fn(); jest.mock('ali-oss', () => jest.fn().mockImplementation(() => ({ getBucketInfo: jest.fn().mockResolvedValue({}), put: (...args: unknown[]) => mockedBucketPut(...args), + delete: (...args: unknown[]) => mockedDelete(...args), + deleteBucket: (...args: unknown[]) => mockedDeleteBucket(...args), })), ); @@ -43,6 +49,7 @@ jest.mock('jszip', () => jest.mock('../../src/common/logger', () => ({ logger: { info: (...args: unknown[]) => mockedInfoLogger(...args), + debug: (...args: unknown[]) => mockedDebugLogger(...args), }, })); @@ -50,7 +57,7 @@ describe('Unit test for rosAssets', () => { beforeEach(() => { jest.clearAllMocks(); }); - describe('Unite test for getAssets', () => { + describe('Unit test for getAssets', () => { it('should return assets from the specified location', () => { const mockLocation = 'mock/location'; mockedExistsSync.mockReturnValue(true); @@ -78,7 +85,7 @@ describe('Unit test for rosAssets', () => { }); }); - describe('publishAssets', () => { + describe('Unit test for publishAssets', () => { const mockContext = { region: 'mock-region', accessKeyId: 'mock-access-key-id', @@ -93,7 +100,10 @@ describe('Unit test for rosAssets', () => { .mockReturnValueOnce({ isFile: () => true } as Stats); mockedGenerateAsync.mockResolvedValue(Buffer.from('mock-zip-content')); - const bucketName = await publishAssets(assetsFixture, mockContext); + const bucketName = await publishAssets( + await constructAssets(assetsFixture, 'mock-region'), + mockContext, + ); expect(bucketName).toBe('cdk-ajmywduza-assets-mock-region'); expect(mockedBucketPut.mock.calls).toEqual([ @@ -112,21 +122,40 @@ describe('Unit test for rosAssets', () => { [ 'Folder compressed to: path/to/asset.55d1d2dd5d6c1b083a04c15431f70da1f2840b9de06383411cbf7eda2a512efe.zip', ], - [ - 'Upload file: path/to/asset.55d1d2dd5d6c1b083a04c15431f70da1f2840b9de06383411cbf7eda2a512efe.zip) to bucket: cdk-ajmywduza-assets-mock-region successfully!', - ], - [ - 'Upload file: path/to/asset.c6a72ed7e7e83f01a000b75885758088fa050298a31a1e95d37ac88f08e42315.zip) to bucket: cdk-ajmywduza-assets-mock-region successfully!', - ], ]); }); it('should log and skip if no assets to publish', async () => { - const emptyAssets = { files: {} } as unknown as CdkAssets; - - await publishAssets(emptyAssets, mockContext); + await publishAssets([], mockContext); expect(mockedInfoLogger).toHaveBeenCalledWith('No assets to publish, skipped!'); }); }); + + describe('Unit test for cleanupAssets', () => { + it('should cleanup and delete the bucket when given assets is valid', async () => { + mockedExistsSync.mockReturnValue(true); + mockedReaddirSync.mockReturnValueOnce(['file1', 'file2']); + mockedLstatSync + .mockReturnValueOnce({ isFile: () => true } as Stats) + .mockReturnValueOnce({ isFile: () => true } as Stats); + mockedGenerateAsync.mockResolvedValue(Buffer.from('mock-zip-content')); + + await cleanupAssets(await constructAssets(assetsFixture, 'mock-region'), defaultContext); + + expect(mockedDelete).toHaveBeenCalledTimes(2); + expect(mockedDelete.mock.calls).toEqual([ + ['55d1d2dd5d6c1b083a04c15431f70da1f2840b9de06383411cbf7eda2a512efe.zip'], + ['c6a72ed7e7e83f01a000b75885758088fa050298a31a1e95d37ac88f08e42315.zip'], + ]); + expect(mockedDeleteBucket).toHaveBeenCalledTimes(1); + expect(mockedDeleteBucket).toHaveBeenCalledWith('cdk-ajmywduza-assets-mock-region'); + }); + + it('should skip the cleanupAssets when there is no assets', async () => { + await cleanupAssets([], defaultContext); + + expect(mockedInfoLogger).toHaveBeenCalledWith('No assets to cleanup, skipped!'); + }); + }); }); diff --git a/tests/common/rosClient.test.ts b/tests/common/rosClient.test.ts index 45f36a4..1bd64d9 100644 --- a/tests/common/rosClient.test.ts +++ b/tests/common/rosClient.test.ts @@ -102,7 +102,7 @@ describe('Unit test for rosClient', () => { await rosStackDelete(context); expect(logger.info).toHaveBeenCalledWith('stack status: DELETE_COMPLETE'); - expect(logger.info).toHaveBeenCalledWith('Stack: testStack deleted! ♻️'); + expect(logger.info).toHaveBeenCalledWith('Stack: testStack deleted!🗑 '); }); it('should throw an error when the stack does not exist', async () => { diff --git a/tests/fixtures/deployFixture.ts b/tests/fixtures/deployFixture.ts index 92cb257..09a2a2d 100644 --- a/tests/fixtures/deployFixture.ts +++ b/tests/fixtures/deployFixture.ts @@ -1476,7 +1476,7 @@ export const defaultContext = { iacLocation: expect.stringContaining('tests/fixtures/serverless-insight.yml'), parameters: [], region: 'cn-hangzhou', - provider: 'aliyun', + provider: ProviderEnum.ALIYUN, securityToken: 'account id', stackName: 'my-demo-stack', stage: 'default', diff --git a/tests/stack/deploy.test.ts b/tests/stack/deploy.test.ts index 4fbd42a..8ffd4bd 100644 --- a/tests/stack/deploy.test.ts +++ b/tests/stack/deploy.test.ts @@ -30,14 +30,17 @@ import { cloneDeep, set } from 'lodash'; const mockedRosStackDeploy = jest.fn(); const mockedResolveCode = jest.fn(); const mockedPublishAssets = jest.fn(); +const mockedCleanupAssets = jest.fn(); const mockedGetIamInfo = jest.fn(); jest.mock('../../src/common', () => ({ ...jest.requireActual('../../src/common'), rosStackDeploy: (...args: unknown[]) => mockedRosStackDeploy(...args), publishAssets: (...args: unknown[]) => mockedPublishAssets(...args), + cleanupAssets: (...args: unknown[]) => mockedCleanupAssets(...args), resolveCode: (path: string) => mockedResolveCode(path), getIamInfo: (...args: unknown[]) => mockedGetIamInfo(...args), + logger: { info: jest.fn(), debug: jest.fn() }, })); describe('Unit tests for stack deployment', () => { @@ -51,6 +54,7 @@ describe('Unit tests for stack deployment', () => { mockedResolveCode.mockRestore(); mockedPublishAssets.mockRestore(); mockedGetIamInfo.mockRestore(); + mockedCleanupAssets.mockRestore(); }); it('should deploy generated stack when minimum fields provided', async () => { @@ -133,18 +137,14 @@ describe('Unit tests for stack deployment', () => { { stackName } as ActionContext, ); + const expectedAssets = Array(2).fill({ + bucketName: expect.any(String), + objectKey: expect.stringContaining('.zip'), + source: expect.stringContaining('.zip'), + }); expect(mockedPublishAssets).toHaveBeenCalledTimes(1); expect(mockedRosStackDeploy).toHaveBeenCalledTimes(2); - expect(mockedPublishAssets).toHaveBeenCalledWith( - expect.objectContaining({ - dockerImages: {}, - files: expect.any(Object), - rootPath: expect.any(String), - version: '7.0.0', - }), - - { stackName }, - ); + expect(mockedPublishAssets).toHaveBeenCalledWith(expectedAssets, { stackName }); expect(mockedRosStackDeploy.mock.calls[1]).toEqual([ stackName, largeCodeRos, @@ -152,6 +152,8 @@ describe('Unit tests for stack deployment', () => { stackName, }, ]); + expect(mockedCleanupAssets).toHaveBeenCalledTimes(1); + expect(mockedCleanupAssets).toHaveBeenCalledWith(expectedAssets, { stackName }); }); describe('unit test for deploy of events', () => { @@ -211,6 +213,7 @@ describe('Unit tests for stack deployment', () => { expect(mockedRosStackDeploy).toHaveBeenCalledTimes(2); expect(mockedPublishAssets).toHaveBeenCalledTimes(1); + expect(mockedCleanupAssets).toHaveBeenCalledTimes(1); expect(mockedRosStackDeploy.mock.calls[1]).toEqual([ stackName, bucketWithWebsiteRos,