feat: custom runtimes, optional VPC, python 3.11 (#74)

* database bootstrapper lambda is configurable. As a result, we had to merge the `database` and `bootstrapper` constructs together, thus the `bootstrapper` construct is no longer available.

* Switched to `Function` and `FromDockerBuild` for the `Code` for all lambdas to allow for more flexible configurability

* made VPC optional for lambdas to allow for low-security deployments that do not necessitate a NAT gateway

* harmonized pgstac to 0.7.10 in default runtimes.

* harmonized python to 3.11 in default runtimes.

BREAKING CHANGE: the `bootstrapper` construct was deleted and is no longer available. In addition, we switched from `PythonFunction` to `Function` for all lambdas.

Author: emileten

lib/bootstrapper/index.ts

@@ -1,13 +1,15 @@
-FROM lambci/lambda:build-python3.8
+ARG PYTHON_VERSION
+FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
 
 ARG PGSTAC_VERSION
-RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"
+RUN echo "PGSTAC_VERSION: ${PGSTAC_VERSION}"
+RUN echo "PYTHON_VERSION: ${PYTHON_VERSION}"
 
 WORKDIR /tmp
 
 RUN pip install httpx psycopg[binary,pool] pypgstac==${PGSTAC_VERSION} -t /asset
 
-COPY runtime/handler.py /asset/handler.py
+COPY bootstrapper_runtime/handler.py /asset/handler.py
 
 # https://stackoverflow.com/a/61746719
 # Tip from eoAPI: turns out, asyncio is part of python

lib/bootstrapper/runtime/Dockerfile renamed to lib/database/bootstrapper_runtime/Dockerfile

@@ -1,12 +1,26 @@
 import {
   Stack,
   aws_rds as rds,
+  aws_ec2 as ec2,
   aws_secretsmanager as secretsmanager,
+  aws_lambda,
+  CustomResource,
+  RemovalPolicy,
+  Duration,
+  aws_logs,
+
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
-import { BootstrapPgStac, BootstrapPgStacProps } from "../bootstrapper";
+import { CustomLambdaFunctionProps } from "../utils";
 
 const instanceSizes: Record<string, number> = require("./instance-memory.json");
+const DEFAULT_PGSTAC_VERSION = "0.7.10";
+
+function hasVpc(
+  instance: rds.DatabaseInstance | rds.IDatabaseInstance
+): instance is rds.DatabaseInstance {
+  return (instance as rds.DatabaseInstance).vpc !== undefined;
+}
 
 /**
  * An RDS instance with pgSTAC installed. This is a wrapper around the
@@ -45,17 +59,76 @@ export class PgStacDatabase extends Construct {
       ...props,
     });
 
-    const bootstrap = new BootstrapPgStac(this, "bootstrap-pgstac-instance", {
-      vpc: props.vpc,
-      database: this.db,
-      dbSecret: this.db.secret!,
-      pgstacDbName: props.pgstacDbName,
-      pgstacVersion: props.pgstacVersion,
-      pgstacUsername: props.pgstacUsername,
-      secretsPrefix: props.secretsPrefix,
+    const handler = new aws_lambda.Function(this, "lambda", {
+      // defaults for configurable properties
+      runtime: aws_lambda.Runtime.PYTHON_3_11,
+      handler: "handler.handler",
+      memorySize: 128,
+      logRetention: aws_logs.RetentionDays.ONE_WEEK,
+      timeout: Duration.minutes(2),
+      code: aws_lambda.Code.fromDockerBuild(__dirname, {
+        file: "bootstrapper_runtime/Dockerfile",
+        buildArgs: {PGSTAC_VERSION: DEFAULT_PGSTAC_VERSION, PYTHON_VERSION: "3.11"}
+      }),
+      // overwrites defaults with user-provided configurable properties
+      ...props.bootstrapperLambdaFunctionOptions,
+      // Non configurable properties that are going to be overwritten even if provided by the user
+      vpc: hasVpc(this.db) ? this.db.vpc : props.vpc,
+      allowPublicSubnet: true
+    });
+
+    this.pgstacSecret = new secretsmanager.Secret(this, "bootstrappersecret", {
+      secretName: [
+        props.secretsPrefix || "pgstac",
+        id,
+        this.node.addr.slice(-8),
+      ].join("/"),
+      generateSecretString: {
+        secretStringTemplate: JSON.stringify({
+          dbname: props.pgstacDbName || "pgstac",
+          engine: "postgres",
+          port: 5432,
+          host: this.db.instanceEndpoint.hostname,
+          username: props.pgstacUsername || "pgstac_user",
+        }),
+        generateStringKey: "password",
+        excludePunctuation: true,
+      },
+      description: `PgSTAC database bootstrapped by ${
+        Stack.of(this).stackName
+      }`,
+    });
+
+    // Allow lambda to...
+    // read new user secret
+    this.pgstacSecret.grantRead(handler);
+    // read database secret
+    this.db.secret!.grantRead(handler);
+    // connect to database
+    this.db.connections.allowFrom(handler, ec2.Port.tcp(5432));
+
+    let customResourceProperties : { [key: string]: any} = {};
+
+    // if customResourceProperties are provided, fill in the values. 
+    if (props.customResourceProperties) {
+      Object.assign(customResourceProperties, props.customResourceProperties);
+    }
+
+    // update properties
+    customResourceProperties["conn_secret_arn"] = this.db.secret!.secretArn;
+    customResourceProperties["new_user_secret_arn"] = this.pgstacSecret.secretArn;
+
+    // if props.lambdaFunctionOptions doesn't have 'code' defined, update pgstac_version (needed for default runtime)
+    if (!props.bootstrapperLambdaFunctionOptions?.code) {
+      customResourceProperties["pgstac_version"] = DEFAULT_PGSTAC_VERSION;
+    }
+    // this.connections = props.database.connections;
+    new CustomResource(this, "bootstrapper", {
+      serviceToken: handler.functionArn,
+      properties: customResourceProperties,
+      removalPolicy: RemovalPolicy.RETAIN, // This retains the custom resource (which doesn't really exist), not the database
     });
 
-    this.pgstacSecret = bootstrap.secret;
   }
 
   public getParameters(
@@ -99,10 +172,43 @@ export class PgStacDatabase extends Construct {
 }
 
 export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps {
-  readonly pgstacDbName?: BootstrapPgStacProps["pgstacDbName"];
-  readonly pgstacVersion?: BootstrapPgStacProps["pgstacVersion"];
-  readonly pgstacUsername?: BootstrapPgStacProps["pgstacUsername"];
-  readonly secretsPrefix?: BootstrapPgStacProps["secretsPrefix"];
+  /**
+   * Name of database that is to be created and onto which pgSTAC will be installed.
+   *
+   * @default pgstac
+   */
+  readonly pgstacDbName?: string;
+
+    /**
+   * Prefix to assign to the generated `secrets_manager.Secret`
+   *
+   * @default pgstac
+   */
+    readonly secretsPrefix?: string;
+
+  /**
+   * Name of user that will be generated for connecting to the pgSTAC database.
+   *
+   * @default pgstac_user
+   */
+  readonly pgstacUsername?: string;
+
+  /**
+   * Lambda function Custom Resource properties. A custom resource property is going to be created
+   * to trigger the boostrapping lambda function. This parameter allows the user to specify additional properties
+   * on top of the defaults ones. 
+   *
+   */
+  readonly customResourceProperties?: {
+    [key: string]: any;
+}
+
+  /**
+   * Optional settings for the bootstrapper lambda function. Can be anything that can be configured on the lambda function, but some will be overwritten by values defined here. 
+   *
+   * @default - defined in the construct.
+   */
+  readonly bootstrapperLambdaFunctionOptions?: CustomLambdaFunctionProps;
 }
 
 export interface DatabaseParameters {

lib/bootstrapper/runtime/handler.py renamed to lib/database/bootstrapper_runtime/handler.py

@@ -1,8 +1,8 @@
 export * from "./bastion-host";
-export * from "./bootstrapper";
 export * from "./database";
 export * from "./ingestor-api";
 export * from "./stac-api";
 export * from "./titiler-pgstac-api";
 export * from "./stac-browser";
 export * from "./tipg-api";
+export * from "./utils";