fix: upload code zip file to bucket issue (#22)

fix: upload code zip file to bucket issue
- use oss_deployment to upload code to the oss bucket for code size >
15MB


Refs: #5

---------

Signed-off-by: seven <[email protected]>

Author: Blankll

package-lock.json

@@ -59,6 +59,7 @@
     "@alicloud/ros-cdk-ram": "^1.4.0",
     "@alicloud/ros20190910": "^3.5.2",
     "ajv": "^8.17.1",
+    "ali-oss": "^6.22.0",
     "chalk": "^5.3.0",
     "commander": "^12.1.0",
     "i18n": "^0.15.1",
@@ -68,6 +69,7 @@
     "yaml": "^2.6.1"
   },
   "devDependencies": {
+    "@types/ali-oss": "^6.16.11",
     "@types/i18n": "^0.13.12",
     "@types/jest": "^29.5.14",
     "@types/lodash": "^4.17.13",

package.json

@@ -10,9 +10,12 @@ import ROS20190910, {
   UpdateStackRequestParameters,
 } from '@alicloud/ros20190910';
 import { Config } from '@alicloud/openapi-client';
-import { ActionContext } from '../types';
+import OSS from 'ali-oss';
+import { ActionContext, CdkAssets } from '../types';
 import { logger } from './logger';
 import { lang } from '../lang';
+import path from 'node:path';
+import { get, isEmpty } from 'lodash';
 
 const client = new ROS20190910(
   new Config({
@@ -196,3 +199,68 @@ export const rosStackDelete = async ({
     throw new Error(JSON.stringify(err));
   }
 };
+
+const ensureBucketExits = async (bucketName: string, ossClient: OSS) =>
+  await ossClient.getBucketInfo(bucketName).catch((err) => {
+    if (err.code === 'NoSuchBucket') {
+      logger.info(`Bucket: ${bucketName} not exists, creating...`);
+      return ossClient.putBucket(bucketName, {
+        storageClass: 'Standard',
+        acl: 'private',
+        dataRedundancyType: 'LRS',
+      } as OSS.PutBucketOptions);
+    } else {
+      throw err;
+    }
+  });
+
+const getZipAssets = ({ files, rootPath }: CdkAssets, region: string) => {
+  const zipAssets = Object.entries(files)
+    .filter(([, fileItem]) => fileItem.source.path.endsWith('zip'))
+    .map(([, fileItem]) => ({
+      bucketName: get(
+        fileItem,
+        'destinations.current_account-current_region.bucketName',
+        '',
+      ).replace('${ALIYUN::Region}', region),
+      source: `${rootPath}/${fileItem.source.path}`,
+      objectKey: get(fileItem, 'destinations.current_account-current_region.objectKey'),
+    }));
+
+  return !isEmpty(zipAssets) ? zipAssets : undefined;
+};
+
+export const publishAssets = async (assets: CdkAssets, context: ActionContext) => {
+  const zipAssets = getZipAssets(assets, context.region);
+
+  if (!zipAssets) {
+    logger.info('No assets to publish, skipped!');
+    return;
+  }
+
+  const bucketName = zipAssets[0].bucketName;
+
+  const client = new OSS({
+    region: `oss-${context.region}`,
+    accessKeyId: context.accessKeyId,
+    accessKeySecret: context.accessKeySecret,
+    bucket: bucketName,
+  });
+
+  await ensureBucketExits(bucketName, client);
+
+  const headers = {
+    'x-oss-storage-class': 'Standard',
+    'x-oss-object-acl': 'private',
+    'x-oss-forbid-overwrite': 'false',
+  } as OSS.PutObjectOptions;
+
+  await Promise.all(
+    zipAssets.map(async ({ source, objectKey }) => {
+      await client.put(objectKey, path.normalize(source), { headers });
+      logger.info(`Upload file: ${source}) to bucket: ${bucketName} successfully!`);
+    }),
+  );
+
+  return bucketName;
+};

src/common/rosClient.ts

@@ -1,9 +1,11 @@
 import * as ros from '@alicloud/ros-cdk-core';
+import fs from 'node:fs';
 
 import { ActionContext, ServerlessIac } from '../types';
-import { logger, ProviderEnum, rosStackDeploy } from '../common';
+import { logger, ProviderEnum, publishAssets, rosStackDeploy } from '../common';
 import { RosStack } from './rosStack';
 import { RfsStack } from './rfsStack';
+import { get } from 'lodash';
 
 export const generateRosStackTemplate = (
   stackName: string,
@@ -14,9 +16,17 @@ export const generateRosStackTemplate = (
   new RosStack(app, iac, context);
 
   const assembly = app.synth();
-  const stackArtifact = assembly.getStackByName(stackName);
 
-  return { template: stackArtifact.template };
+  const { template } = assembly.getStackByName(stackName);
+
+  const assetFolderPath = get(assembly.tryGetArtifact(`${stackName}.assets`), 'file', '');
+  const assetsFileBody = fs.readFileSync(assetFolderPath);
+  const assets = {
+    rootPath: assembly.directory,
+    ...JSON.parse(assetsFileBody.toString('utf-8').trim()),
+  };
+
+  return { template, assets };
 };
 
 export const generateRfsStackTemplate = (
@@ -37,8 +47,10 @@ export const deployStack = async (
   iac: ServerlessIac,
   context: ActionContext,
 ) => {
-  const { template } = generateRosStackTemplate(stackName, iac, context);
-
+  const { template, assets } = generateRosStackTemplate(stackName, iac, context);
+  logger.info(`Deploying stack, publishing assets...`);
+  await publishAssets(assets, context);
+  logger.info(`Assets published! 🎉`);
   await rosStackDeploy(stackName, template, context);
   logger.info(`Stack deployed! 🎉`);
 };

src/stack/deploy.ts

@@ -6,7 +6,6 @@ import {
   replaceReference,
   resolveCode,
 } from '../../common';
-import { RosFunction } from '@alicloud/ros-cdk-fc3/lib/fc3.generated';
 import * as fc from '@alicloud/ros-cdk-fc3';
 import * as oss from '@alicloud/ros-cdk-oss';
 import { isEmpty } from 'lodash';
@@ -49,15 +48,16 @@ export const resolveFunctions = (
       {
         sources: fileSources!.map(({ source }) => source),
         destinationBucket,
-        timeout: 300,
-        logMonitoring: false, // 是否开启日志监控，设为false则不开启
+        timeout: 3000,
+        logMonitoring: false,
       },
       true,
     );
+    artifactsDeployment.addDependency(destinationBucket);
   }
   functions?.forEach((fnc) => {
     const storeInBucket = readCodeSize(fnc.code) > CODE_ZIP_SIZE_LIMIT;
-    let code: RosFunction.CodeProperty = {
+    let code: fc.RosFunction.CodeProperty = {
       zipFile: resolveCode(fnc.code),
     };
     if (storeInBucket) {
@@ -68,7 +68,7 @@ export const resolveFunctions = (
         )?.objectKey,
       };
     }
-    const fcn = new fc.RosFunction(
+    new fc.RosFunction(
       scope,
       fnc.key,
       {
@@ -82,9 +82,5 @@ export const resolveFunctions = (
       },
       true,
     );
-    if (storeInBucket) {
-      fcn.addDependsOn(destinationBucket as unknown as ros.RosResource);
-      fcn.addDependsOn(artifactsDeployment as unknown as ros.RosResource);
-    }
   });
 };

src/stack/rosStack/function.ts

@@ -0,0 +1,19 @@
+export type CdkAssets = {
+  version: string;
+  rootPath: string;
+  dockerImages: Record<string, unknown>;
+  files: {
+    [key: string]: {
+      source: {
+        path: string;
+        packaging: string;
+      };
+      destinations: {
+        [key: string]: {
+          bucketName: string;
+          objectKey: string;
+        };
+      };
+    };
+  };
+};

src/types/assets.ts

@@ -12,6 +12,8 @@ export * from './domains/tag';
 export * from './domains/vars';
 export * from './domains/context';
 
+export * from './assets';
+
 export type ServerlessIacRaw = {
   version: string;
   provider: Provider;

src/types/index.ts

@@ -481,10 +481,11 @@ export const largeCodeRos = {
         ServiceName: {
           'Fn::GetAtt': ['FCServiceFormy-demo-service_artifacts_code_deployment', 'ServiceName'],
         },
-        Timeout: 300,
+        Timeout: 3000,
       },
       Type: 'ALIYUN::FC::Function',
     },
+
     'FCRoleFormy-demo-service_artifacts_code_deployment': {
       Properties: {
         AssumeRolePolicyDocument: {
@@ -682,6 +683,7 @@ export const largeCodeRos = {
       Type: 'ALIYUN::OSS::Bucket',
     },
     'my-demo-service_artifacts_code_deployment': {
+      DependsOn: ['my-demo-service_artifacts_bucket'],
       Properties: {
         Parameters: {
           destinationBucket: {
@@ -690,7 +692,7 @@ export const largeCodeRos = {
           retainOnCreate: false,
           sources: [
             {
-              bucket: { 'Fn::Sub': expect.stringContaining('assets-${ALIYUN::Region}') },
+              bucket: { 'Fn::Sub': expect.any(String) },
               fileName: 'hello_fn/43cb4c356149762dbe507fc1baede172-large-artifact.zip',
               objectKey: '2bfeafed8d3df0d44c235271cdf2aa7d908a3c2757af14a67d33d102847f46fd.zip',
             },
@@ -699,7 +701,7 @@ export const largeCodeRos = {
         ServiceToken: {
           'Fn::GetAtt': ['FCFunctionFormy-demo-service_artifacts_code_deployment', 'ARN'],
         },
-        Timeout: 300,
+        Timeout: 3000,
       },
       Type: 'ALIYUN::ROS::CustomResource',
     },

tests/fixtures/deployFixture.ts

@@ -19,19 +19,24 @@ import { cloneDeep, set } from 'lodash';
 
 const mockedRosStackDeploy = jest.fn();
 const mockedResolveCode = jest.fn();
+const mockedPublishAssets = jest.fn();
+
 jest.mock('../../src/common', () => ({
   ...jest.requireActual('../../src/common'),
   rosStackDeploy: (...args: unknown[]) => mockedRosStackDeploy(...args),
+  publishAssets: (...args: unknown[]) => mockedPublishAssets(...args),
   resolveCode: (path: string) => mockedResolveCode(path),
 }));
 
 describe('Unit tests for stack deployment', () => {
   beforeEach(() => {
     mockedResolveCode.mockReturnValueOnce('resolved-code');
+    mockedPublishAssets.mockResolvedValueOnce('published-assets-bucket');
   });
   afterEach(() => {
     mockedRosStackDeploy.mockRestore();
     mockedResolveCode.mockRestore();
+    mockedPublishAssets.mockRestore();
   });
 
   it('should deploy generated stack when iac is valid', async () => {
@@ -95,6 +100,7 @@ describe('Unit tests for stack deployment', () => {
       options,
     );
   });
+
   it('should evaluate service name as pure string when it reference ${ctx.stage}', async () => {
     const options = { stackName: 'my-demo-stack-fc-with-stage-1', stage: 'dev' };
     mockedRosStackDeploy.mockResolvedValueOnce(options.stackName);
@@ -109,11 +115,11 @@ describe('Unit tests for stack deployment', () => {
     );
   });
 
-  it('should create bucket and store code artifact to bucket when code size > 15MB', () => {
+  it('should create bucket and store code artifact to bucket when code size > 15MB', async () => {
     const stackName = 'my-large-code-stack';
     mockedRosStackDeploy.mockResolvedValueOnce(stackName);
 
-    deployStack(
+    await deployStack(
       stackName,
       set(
         cloneDeep(oneFcOneGatewayIac),
@@ -123,16 +129,27 @@ describe('Unit tests for stack deployment', () => {
       { stackName } as ActionContext,
     );
 
-    expect(mockedResolveCode).toHaveBeenCalledTimes(1);
+    expect(mockedPublishAssets).toHaveBeenCalledTimes(1);
+    expect(mockedRosStackDeploy).toHaveBeenCalledTimes(1);
+    expect(mockedPublishAssets).toHaveBeenCalledWith(
+      expect.objectContaining({
+        dockerImages: {},
+        files: expect.any(Object),
+        rootPath: expect.any(String),
+        version: '7.0.0',
+      }),
+
+      { stackName },
+    );
     expect(mockedRosStackDeploy).toHaveBeenCalledWith(stackName, largeCodeRos, { stackName });
   });
 
   describe('unit test for deploy of databases', () => {
-    it('should deploy elasticsearch serverless when database minimum fields provided', () => {
+    it('should deploy elasticsearch serverless when database minimum fields provided', async () => {
       const stackName = 'my-demo-es-serverless-stack';
       mockedRosStackDeploy.mockResolvedValueOnce(stackName);
 
-      deployStack(stackName, esServerlessMinimumIac, { stackName } as ActionContext);
+      await deployStack(stackName, esServerlessMinimumIac, { stackName } as ActionContext);
 
       expect(mockedRosStackDeploy).toHaveBeenCalledTimes(1);
       expect(mockedRosStackDeploy).toHaveBeenCalledWith(stackName, esServerlessMinimumRos, {