Add support for upgrading/removing BoilingData AWS Lambda Layer for selected Lambda functions (prefix)

package.json

@@ -60,6 +60,7 @@
     "@aws-sdk/client-cognito-identity": "^3.533.0",
     "@aws-sdk/client-iam": "^3.533.0",
     "@aws-sdk/client-sts": "^3.533.0",
+    "@aws-sdk/client-lambda": "^3.533.0",
     "@aws-sdk/credential-provider-cognito-identity": "^3.533.0",
     "@boilingdata/node-boilingdata": "^1.0.20",
     "amazon-cognito-identity-js": "^6.3.4",

src/bdcli/commands/account/bdcli-account-register.ts

@@ -27,6 +27,7 @@ async function show(options: any, _command: cmd.Command): Promise<void> {
       await registerToBoilingData(region, environment, email, password, logger);
     }
     if (!(await hasValidConfig())) return spinnerError("No valid config, was registration successful?");
+    updateSpinnerText("Registering to BoilingData. Check your email for confirmation code and use --confirm <code>.");
     spinnerSuccess();
   } catch (err: any) {
     spinnerError(err?.message);

src/bdcli/commands/api/bdcli-api-query.ts

@@ -20,7 +20,8 @@ async function query(options: any, _command: cmd.Command): Promise<void> {
     spinnerSuccess();
 
     updateSpinnerText("Sending Query to Boiling API");
-    const results = await runBoilingQuery(options.sql, token, region, logger);
+    const spinnerText = "Query run through Boiling API";
+    const results = await runBoilingQuery(options.sql, token, region, logger, spinnerText);
     spinnerSuccess();
     await outputResults(results, options.disableSpinner);
   } catch (err: any) {

src/bdcli/commands/aws/bdcli-aws-lambda-layers.ts

@@ -0,0 +1,71 @@
+import * as cmd from "commander";
+import * as sts from "@aws-sdk/client-sts";
+import { ELogLevel, getLogger } from "../../utils/logger_util.js";
+import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js";
+import { addGlobalOptions } from "../../utils/options_util.js";
+import { getIdToken } from "../../utils/auth_util.js";
+import { combineOptsWithSettings } from "../../utils/config_util.js";
+import { BDAccount } from "../../../integration/boilingdata/account.js";
+import { BoilingLambdaFunctions, ILambdaLayerOpts } from "../../../integration/bdLambdas.js";
+import { LambdaClient } from "@aws-sdk/client-lambda";
+
+const logger = getLogger("bdcli-aws-lambda-layers");
+logger.setLogLevel(ELogLevel.WARN);
+
+async function lambdalayers(options: any, _command: cmd.Command): Promise<void> {
+  try {
+    options = await combineOptsWithSettings(options, logger);
+
+    if (!options.remove && !options.upgrade) {
+      spinnerWarn("Please specify either --remove or --upgrade");
+      return;
+    }
+    if (options.remove && (options.denyCwLogs || options.removeCwLogs)) {
+      spinnerWarn("Can not use --deny-cw-logs or --remove-cw-logs when removing layer");
+      return;
+    }
+
+    updateSpinnerText("Authenticating");
+    const { idToken: token, cached, region } = await getIdToken(logger);
+    updateSpinnerText(cached ? "Authenticating: cached" : "Authenticating: success");
+    spinnerSuccess();
+
+    updateSpinnerText(`NOT_IMPLEMENTED_YET: ${options.remove ? "Removing" : "Upgrading"} BoilingData AWS Lambda Layer`);
+    if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
+    const bdAccount = new BDAccount({ logger, authToken: token });
+    const stsClient = new sts.STSClient({ region });
+    const lambdaClient = new LambdaClient({ region });
+    const bdLambdas = new BoilingLambdaFunctions({ logger, bdAccount, stsClient, lambdaClient });
+    const opts: ILambdaLayerOpts = { prefix: options.functionPrefix };
+    if (options.remove) await bdLambdas.removeLambdaLayers(opts);
+    else if (options.upgrade) await bdLambdas.upgradeLambdaLayers(opts);
+    spinnerSuccess();
+
+    if (options.upgrade && options.denyCwLogs) {
+      updateSpinnerText("NOT_IMPLEMENTED_YET: Denying AWS Lambda CW logging.");
+      await bdLambdas.denyCWLogging(opts);
+      spinnerSuccess();
+    }
+    if (options.upgrade && options.removeCwLogs) {
+      updateSpinnerText("NOT_IMPLEMENTED_YET: Removing respective AWS Lambda CW log groups.");
+      await bdLambdas.removeCWLogs(opts);
+      spinnerSuccess();
+    }
+  } catch (err: any) {
+    spinnerError(err?.message);
+  }
+}
+
+const program = new cmd.Command("bdcli aws taps-iam")
+  .addOption(new cmd.Option("-r, --region <region>", "AWS region"))
+  .addOption(new cmd.Option("-p, --function-prefix <prefix>", "AWS Lambda function name prefix").makeOptionMandatory())
+  .addOption(new cmd.Option("--upgrade", "Add/Upgrade the layer if not latest"))
+  .addOption(new cmd.Option("--remove", "Remove the layer"))
+  .addOption(new cmd.Option("--deny-cw-logs", "Update the AWS Lambda IAM Role default policy to deny CW Logs"))
+  .addOption(new cmd.Option("--remove-cw-logs", "After succesull layer setup, remove AWS Lambda CW Logs"))
+  .action(async (options, command) => await lambdalayers(options, command));
+
+(async () => {
+  await addGlobalOptions(program, logger);
+  await program.parseAsync(process.argv);
+})();

src/bdcli/commands/aws/bdcli-aws-s3-iam.ts

@@ -45,7 +45,7 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
       assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
       assumeCondExternalId: await bdAccount.getExtId(),
     });
-    const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources, stsClient });
+    const bdIntegration = new BDIntegration({ logger, bdAccount, bdDataSources, stsClient });
     const policyDocument = await bdIntegration.getS3PolicyDocument();
     const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
     updateSpinnerText(`Creating S3 IAM Role: ${iamRoleArn}`);

src/bdcli/commands/aws/bdcli-aws-taps-iam.ts

@@ -42,7 +42,7 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
       assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
       assumeCondExternalId: await bdAccount.getExtId(),
     });
-    const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, stsClient });
+    const bdIntegration = new BDIntegration({ logger, bdAccount, stsClient });
     const policyDocument = await bdIntegration.getTapsPolicyDocument();
     const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
     updateSpinnerText(`Creating TAPS IAM Role: ${iamRoleArn}`);

src/bdcli/commands/bdcli-aws.ts

@@ -3,6 +3,7 @@ import { Command } from "commander";
 const program = new Command("bdcli aws")
   .executableDir("aws")
   .command("s3-iam", "Setup IAM Role into your AWS account for accessing S3")
-  .command("taps-iam", "Setup deployer and service IAM Roles into your AWS account for creating and running Data Taps");
+  .command("taps-iam", "Setup deployer and service IAM Roles into your AWS account for creating and running Data Taps")
+  .command("lambda-layers", "Maintain BoilingData AWS Lambda Layer for your Lambda functions");
 
 program.parse(process.argv);

src/bdcli/commands/domain/bdcli-domain-setup.ts

@@ -1,14 +1,8 @@
-import * as iam from "@aws-sdk/client-iam";
-import * as sts from "@aws-sdk/client-sts";
 import * as cmd from "commander";
 import { ELogLevel, getLogger } from "../../utils/logger_util.js";
 import { spinnerError, spinnerSuccess, spinnerWarn, updateSpinnerText } from "../../utils/spinner_util.js";
 import { addGlobalOptions } from "../../utils/options_util.js";
 import { authSpinnerWithConfigCheck, getIdToken } from "../../utils/auth_util.js";
-import { BDIamRole } from "../../../integration/aws/iam_role.js";
-import { BDAccount } from "../../../integration/boilingdata/account.js";
-import { BDDataSourceConfig } from "../../../integration/boilingdata/dataset.js";
-import { BDIntegration } from "../../../integration/bdIntegration.js";
 import { combineOptsWithSettings } from "../../utils/config_util.js";
 
 const logger = getLogger("bdcli-domain");
@@ -25,36 +19,13 @@ async function iamrole(options: any, _command: cmd.Command): Promise<void> {
     }
 
     if (!authSpinnerWithConfigCheck()) return;
-    const { idToken: token, cached, region } = await getIdToken(logger);
+    const { cached, region } = await getIdToken(logger);
+    if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
     updateSpinnerText(cached ? "Authenticating: cached" : "Authenticating: success");
     spinnerSuccess();
 
-    updateSpinnerText("Creating IAM Role");
-    if (!region) throw new Error("Pass --region parameter or set AWS_REGION env");
-    const bdAccount = new BDAccount({ logger, authToken: token });
-    const bdDataSources = new BDDataSourceConfig({ logger });
-    await bdDataSources.readConfig(options.config);
-    const stsClient = new sts.STSClient({ region });
-    const bdRole = new BDIamRole({
-      ...options,
-      logger,
-      iamClient: new iam.IAMClient({ region }),
-      stsClient: new sts.STSClient({ region }),
-      assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
-      assumeCondExternalId: await bdAccount.getExtId(),
-    });
-    const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources, stsClient });
-    const policyDocument = await bdIntegration.getS3PolicyDocument();
-    const iamRoleArn = await bdRole.upsertRole(JSON.stringify(policyDocument));
-    updateSpinnerText(`Creating IAM Role: ${iamRoleArn}`);
+    updateSpinnerText("TODO: Manage domain");
     spinnerSuccess();
-
-    if (!options.createRoleOnly) {
-      updateSpinnerText(`Registering IAM Role: ${iamRoleArn}`);
-      const datasourcesConfig = bdDataSources.getDatasourcesConfig();
-      await bdAccount.setS3IamRoleWithPayload(iamRoleArn, { datasourcesConfig });
-      spinnerSuccess();
-    }
   } catch (err: any) {
     spinnerError(err?.message);
   }

src/bdcli/commands/sandbox/bdcli-sandbox-create-role.ts

@@ -46,7 +46,7 @@ async function show(options: any, _command: cmd.Command): Promise<void> {
       assumeAwsAccount: await bdAccount.getAssumeAwsAccount(),
       assumeCondExternalId: await bdAccount.getExtId(),
     });
-    const bdIntegration = new BDIntegration({ logger, bdAccount, bdRole, bdDataSources, stsClient });
+    const bdIntegration = new BDIntegration({ logger, bdAccount, bdDataSources, stsClient });
     const policyDocument = await bdIntegration.getS3PolicyDocument(options.listBucketsPermission);
     if (options.dryRun) {
       updateSpinnerText(`Creating IAM Role (dry-run)`);

src/integration/aws/lambda.ts

@@ -0,0 +1,23 @@
+import {
+  FunctionConfiguration,
+  LambdaClient,
+  ListFunctionsCommand,
+  ListFunctionsCommandOutput,
+} from "@aws-sdk/client-lambda";
+import { ILogger } from "../../bdcli/utils/logger_util.js";
+
+export async function getAllLambdaFunctions(
+  lambdaClient: LambdaClient,
+  logger?: ILogger,
+): Promise<FunctionConfiguration[]> {
+  let Marker;
+  const funcs: FunctionConfiguration[] = [];
+  do {
+    const res: ListFunctionsCommandOutput = await lambdaClient.send(new ListFunctionsCommand({ Marker }));
+    logger?.debug(res?.NextMarker);
+    logger?.debug(res?.Functions?.length);
+    Marker = res?.NextMarker;
+    if (Array.isArray(res.Functions)) funcs.push(...res.Functions);
+  } while (Marker);
+  return funcs;
+}

src/integration/bdIntegration.test.ts

@@ -1,6 +1,5 @@
 import { getIdToken } from "../bdcli/utils/auth_util.js";
 import { ELogLevel, getLogger } from "../bdcli/utils/logger_util.js";
-import { BDIamRole, ERoleType } from "./aws/iam_role.js";
 import { BDIntegration } from "./bdIntegration.js";
 import { BDAccount } from "./boilingdata/account.js";
 import { BDDataSourceConfig } from "./boilingdata/dataset.js";
@@ -14,15 +13,12 @@ iamMock.resolves({});
 
 const accountLogger = getLogger("bd-account");
 const dssLogger = getLogger("bd-datasets");
-const roleLogger = getLogger("bd-role");
 const accessLogger = getLogger("bd-access");
 accountLogger.setLogLevel(ELogLevel.DEBUG);
 // dssLogger.setLogLevel(ELogLevel.DEBUG);
-// roleLogger.setLogLevel(ELogLevel.DEBUG);
 accessLogger.setLogLevel(ELogLevel.DEBUG);
 
 const region = "eu-west-1";
-const iamClient = new IAMClient({ region });
 const stsClient = new STSClient({ region });
 const bdDataSets = new BDDataSourceConfig({ logger: dssLogger });
 let bdAccount: BDAccount;
@@ -35,24 +31,10 @@ describe("BDIntegration", () => {
 
   it.skip("getGroupedBuckets", async () => {
     bdDataSets.readConfig("./example_datasource_config.yaml");
-    const assumeCondExternalId = await bdAccount.getExtId(); // FIXME: This calls real API
-    const assumeAwsAccount = await bdAccount.getAssumeAwsAccount();
-    const username = await bdAccount.getUsername();
-    const bdRole = new BDIamRole({
-      logger: roleLogger,
-      region,
-      roleType: ERoleType.S3,
-      iamClient,
-      stsClient,
-      username,
-      assumeAwsAccount,
-      assumeCondExternalId,
-    });
     const bdIntegration = new BDIntegration({
       logger: accessLogger,
       bdAccount,
       bdDataSources: bdDataSets,
-      bdRole,
       stsClient,
     });
     stsMock.on(GetCallerIdentityCommand).resolves({ Account: "123123123123" });
@@ -88,24 +70,10 @@ describe("BDIntegration", () => {
 
   it.skip("PolicyDocument", async () => {
     bdDataSets.readConfig("./example_datasource_config.yaml");
-    const assumeCondExternalId = await bdAccount.getExtId(); // FIXME: This calls real API
-    const assumeAwsAccount = await bdAccount.getAssumeAwsAccount();
-    const username = await bdAccount.getUsername();
-    const bdRole = new BDIamRole({
-      logger: roleLogger,
-      region,
-      roleType: ERoleType.S3,
-      iamClient,
-      stsClient,
-      username,
-      assumeAwsAccount,
-      assumeCondExternalId,
-    });
     const bdIntegration = new BDIntegration({
       logger: accessLogger,
       bdAccount,
       bdDataSources: bdDataSets,
-      bdRole,
       stsClient,
     });
     stsMock.on(GetCallerIdentityCommand).resolves({ Account: "123123123123" });

src/integration/bdIntegration.ts

@@ -1,6 +1,5 @@
 import { GetCallerIdentityCommand, STSClient } from "@aws-sdk/client-sts";
 import { ILogger } from "../bdcli/utils/logger_util.js";
-import { BDIamRole } from "./aws/iam_role.js";
 import { BDAccount } from "./boilingdata/account.js";
 import { GRANT_PERMISSION, IStatement, IStatementExt } from "./boilingdata/dataset.interface.js";
 import { BDDataSourceConfig } from "./boilingdata/dataset.js";
@@ -19,7 +18,6 @@ export interface IBDIntegration {
   logger: ILogger;
   stsClient: STSClient;
   bdAccount: BDAccount;
-  bdRole: BDIamRole;
   bdDataSources?: BDDataSourceConfig;
 }
 

src/integration/bdLambdas.ts

@@ -0,0 +1,67 @@
+import { FunctionConfiguration, LambdaClient } from "@aws-sdk/client-lambda";
+import { ILogger } from "../bdcli/utils/logger_util.js";
+import { BDAccount } from "./boilingdata/account.js";
+import { STSClient } from "@aws-sdk/client-sts";
+import { getAllLambdaFunctions } from "./aws/lambda.js";
+
+export interface ILambdaLayerOpts {
+  prefix: string; // Lambda Function name prefix
+}
+
+export interface IBoilingLambdaFunctions {
+  logger: ILogger;
+  lambdaClient: LambdaClient;
+  bdAccount?: BDAccount;
+  stsClient?: STSClient;
+}
+
+export class BoilingLambdaFunctions {
+  private logger: ILogger;
+  private lambda: LambdaClient;
+  // private bdAccount: BDAccount;
+  // private stsClient: STSClient;
+  private allLambdas: FunctionConfiguration[] = [];
+  private boilingLambdas: string[] = [];
+  private boilingLambdasCachedPrefix = "";
+
+  constructor(private params: IBoilingLambdaFunctions) {
+    this.logger = this.params.logger;
+    this.logger.debug(params);
+    this.lambda = this.params.lambdaClient;
+    // this.bdAccount = this.params.bdAccount;
+    // this.stsClient = this.params.stsClient;
+  }
+
+  private async populateLambdaFunctionsCache(prefix: string): Promise<void> {
+    this.logger.debug(prefix);
+    if (
+      (this.boilingLambdasCachedPrefix === "" || this.boilingLambdasCachedPrefix != prefix) &&
+      Array.isArray(this.allLambdas) &&
+      this.allLambdas.length <= 0
+    ) {
+      this.allLambdas = await getAllLambdaFunctions(this.lambda, this.logger);
+    }
+    this.boilingLambdas = <string[]>this.allLambdas
+      .filter(f => f.FunctionName?.startsWith(prefix))
+      .map(f => f.FunctionName)
+      .filter(f => !!f);
+    this.boilingLambdasCachedPrefix = prefix;
+    this.logger.debug(this.boilingLambdas);
+  }
+
+  public async removeLambdaLayers(opts: ILambdaLayerOpts): Promise<void> {
+    await this.populateLambdaFunctionsCache(opts.prefix);
+  }
+
+  public async upgradeLambdaLayers(opts: ILambdaLayerOpts): Promise<void> {
+    await this.populateLambdaFunctionsCache(opts.prefix);
+  }
+
+  public async denyCWLogging(opts: ILambdaLayerOpts): Promise<void> {
+    this.logger.debug(opts);
+  }
+
+  public async removeCWLogs(opts: ILambdaLayerOpts): Promise<void> {
+    await this.populateLambdaFunctionsCache(opts.prefix);
+  }
+}

src/integration/boilingdata/query.ts

@@ -1,8 +1,15 @@
 import { BoilingData } from "@boilingdata/node-boilingdata";
 import { ILogger } from "../../bdcli/utils/logger_util.js";
 import { BDAWSRegion } from "@boilingdata/node-boilingdata/dist/cjs/boilingdata/boilingdata.js";
+import { updateSpinnerText } from "../../bdcli/utils/spinner_util.js";
 
-export async function runBoilingQuery(sql: string, idToken: string, region: string, logger: ILogger): Promise<any[]> {
+export async function runBoilingQuery(
+  sql: string,
+  idToken: string,
+  region: string,
+  logger: ILogger,
+  spinnerText?: string,
+): Promise<any[]> {
   try {
     logger.debug({ sql, idToken, region });
     const bdInstance = new BoilingData({
@@ -17,7 +24,8 @@ export async function runBoilingQuery(sql: string, idToken: string, region: stri
     const stop = Date.now();
     const parsedRows = JSON.parse(JSON.stringify(rows));
     logger.debug(JSON.parse(JSON.stringify(rows)));
-    logger.debug("Query time measured e2e (ms):", stop - start);
+    logger.debug(`Query time measured e2e: ${stop - start}ms`);
+    if (spinnerText) updateSpinnerText(`${spinnerText}: ${stop - start}ms`);
     await bdInstance.close();
     return parsedRows;
   } catch (err) {