feat: custom runtimes option for titiler and ingestor (#66)

* custom runtimes option for titiler and ingestor

* make those optional

* switch to pythonFunction for titiler runtime, remove boto3 from requirements, a coupl adjustments accordingly, fix optional env vars

* remove ref to duration

* put back the duration

* wrap the python lambda settings that are optional in an interface to make things clearer and more configurable, add bundling

Author: emileten

lib/ingestor-api/index.ts

@@ -11,7 +11,7 @@ import {
   RemovalPolicy,
   Stack,
 } from "aws-cdk-lib";
-import { PythonFunction } from "@aws-cdk/aws-lambda-python-alpha";
+import { PythonFunction, PythonFunctionProps } from "@aws-cdk/aws-lambda-python-alpha";
 import { Construct } from "constructs";
 
 export class StacIngestor extends Construct {
@@ -55,6 +55,7 @@ export class StacIngestor extends Construct {
       dbVpc: props.vpc,
       dbSecurityGroup: props.stacDbSecurityGroup,
       subnetSelection: props.subnetSelection,
+      apiCode: props.apiCode,
     });
 
     this.buildApiEndpoint({
@@ -72,6 +73,7 @@ export class StacIngestor extends Construct {
       dbVpc: props.vpc,
       dbSecurityGroup: props.stacDbSecurityGroup,
       subnetSelection: props.subnetSelection,
+      ingestorCode: props.ingestorCode,
     });
 
     this.registerSsmParameter({
@@ -108,11 +110,17 @@ export class StacIngestor extends Construct {
     dbVpc: ec2.IVpc;
     dbSecurityGroup: ec2.ISecurityGroup;
     subnetSelection: ec2.SubnetSelection
+    apiCode?: ApiCode;
   }): PythonFunction {
 
-    const handler = new PythonFunction(this, "api-handler", {
+    const apiCode = props.apiCode || {
       entry: `${__dirname}/runtime`,
       index: "src/handler.py",
+      handler: "handler",
+    };
+
+    const handler = new PythonFunction(this, "api-handler", {
+      ...apiCode,
       runtime: lambda.Runtime.PYTHON_3_9,
       timeout: Duration.seconds(30),
       environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env },
@@ -145,10 +153,19 @@ export class StacIngestor extends Construct {
     dbVpc: ec2.IVpc;
     dbSecurityGroup: ec2.ISecurityGroup;
     subnetSelection: ec2.SubnetSelection;
+    ingestorCode?: IngestorCode;
   }): PythonFunction {
-    const handler = new PythonFunction(this, "stac-ingestor", {
+
+
+
+    const ingestorCode = props.ingestorCode || {
       entry: `${__dirname}/runtime`,
       index: "src/ingestor.py",
+      handler: "handler",
+    };
+
+    const handler = new PythonFunction(this, "stac-ingestor", {
+      ...ingestorCode,
       runtime: lambda.Runtime.PYTHON_3_9,
       timeout: Duration.seconds(180),
       environment: { DB_SECRET_ARN: props.dbSecret.secretArn, ...props.env },
@@ -290,4 +307,56 @@ export interface StacIngestorProps {
    * Custom Domain Name Options for Ingestor API
    */
    readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions;
+
+  /**
+   * Custom code for the ingestor api.
+   *
+   * @default - default in the runtime folder. 
+   */
+   readonly apiCode?: ApiCode;
+
+   /**
+   * Custom code for the ingestor.
+   *
+   * @default - default in the runtime folder. 
+   */
+   readonly ingestorCode?: IngestorCode;
 }
+
+export interface ApiCode {
+
+  /**
+   * Path to the source of the function or the location for dependencies, for the api lambda. 
+   */
+  readonly entry: PythonFunctionProps["entry"];
+
+  /**
+   * Path to the index file containing the exported handler, relative to `api_lambda_entry`. 
+   */
+  readonly index: PythonFunctionProps["index"];
+
+  /**
+   * The name of the exported handler in the `api_lambda_index` file. 
+   */
+  readonly handler: PythonFunctionProps["handler"];
+
+}
+
+export interface IngestorCode {
+
+  /**
+   * Path to the source of the function or the location for dependencies, for the ingestor lambda. 
+   */
+  readonly entry: PythonFunctionProps["entry"];
+
+  /**
+   * Path to the index file containing the exported handler, relative to `ingestor_lambda_entry`. 
+   */
+  readonly index: PythonFunctionProps["index"];
+
+  /**
+   * The name of the exported handler in the `ingestor_lambda_index` file.
+   */
+  readonly handler: PythonFunctionProps["handler"];
+
+}

lib/tipg-api/index.ts

@@ -87,9 +87,9 @@ import {
     readonly dbSecret: secretsmanager.ISecret;
 
     /**
-     * Custom code to run for fastapi-pgstac.
+     * Custom code to run for the application.
      *
-     * @default - simplified version of fastapi-pgstac
+     * @default - simplified version of tipg.
      */
     readonly apiCode?: TiPgApiEntrypoint;
 

lib/titiler-pgstac-api/index.ts

@@ -8,51 +8,61 @@ import {
     CfnOutput,
     Duration,
     aws_logs,
+    BundlingOptions
   } from "aws-cdk-lib";
+  import { Runtime } from 'aws-cdk-lib/aws-lambda';
+  import {PythonFunction} from "@aws-cdk/aws-lambda-python-alpha";
   import { IDomainName, HttpApi } from "@aws-cdk/aws-apigatewayv2-alpha";
   import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha";
   import { Construct } from "constructs";
 
+
+  // default settings that can be overridden by the user-provided environment. 
+  let defaultTitilerPgstacEnv :{ [key: string]: any } = {
+    "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff",
+    "GDAL_CACHEMAX": "200", 
+    "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+    "GDAL_INGESTED_BYTES_AT_OPEN": "32768",
+    "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
+    "GDAL_HTTP_MULTIPLEX": "YES",
+    "GDAL_HTTP_VERSION": "2",
+    "PYTHONWARNINGS": "ignore",
+    "VSI_CACHE": "TRUE",
+    "VSI_CACHE_SIZE": "5000000", 
+    "DB_MIN_CONN_SIZE": "1",
+    "DB_MAX_CONN_SIZE": "1"
+  }
+
   export class TitilerPgstacApiLambda extends Construct {
     readonly url: string;
     public titilerPgstacLambdaFunction: lambda.Function;
 
     constructor(scope: Construct, id: string, props: TitilerPgStacApiLambdaProps) {
       super(scope, id);
-  
-      const titilerPgstacEnv = {
-          "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff",
-          "GDAL_CACHEMAX": "200", 
-          "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
-          "GDAL_INGESTED_BYTES_AT_OPEN": "32768",
-          "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
-          "GDAL_HTTP_MULTIPLEX": "YES",
-          "GDAL_HTTP_VERSION": "2",
-          "PYTHONWARNINGS": "ignore",
-          "VSI_CACHE": "TRUE",
-          "VSI_CACHE_SIZE": "5000000", 
-          "DB_MIN_CONN_SIZE": "1",
-          "DB_MAX_CONN_SIZE": "1",
-          "PGSTAC_SECRET_ARN": props.dbSecret.secretArn,
-      }
-    
-      
-      this.titilerPgstacLambdaFunction = new lambda.Function(this, "lambda", {
-        handler: "handler.handler",
+
+
+      // if user provided environment variables, merge them with the defaults.
+      const apiEnv = props.apiEnv ? { ...defaultTitilerPgstacEnv, ...props.apiEnv, "PGSTAC_SECRET_ARN": props.dbSecret.secretArn } : defaultTitilerPgstacEnv;
+
+      const pythonLambdaOptions: TitilerPgstacPythonLambdaOptions = props.pythonLambdaOptions ?? {
         runtime: lambda.Runtime.PYTHON_3_10,
-        code: lambda.Code.fromDockerBuild(__dirname, {
-          file: "runtime/Dockerfile",
-          buildArgs: { PYTHON_VERSION: '3.10' },
-        }),
-        timeout: Duration.seconds(30),
+        entry: `${__dirname}/runtime`,
+        index: "src/handler.py",
+        handler: "handler",
+        memorySize: 3008,
+        architecture: lambda.Architecture.X86_64
+      }
+
+      this.titilerPgstacLambdaFunction = new PythonFunction(this, "titiler-pgstac-api", {
+        ...pythonLambdaOptions,
+        environment: apiEnv,
         vpc: props.vpc,
         vpcSubnets: props.subnetSelection,
         allowPublicSubnet: true,
-        memorySize: 3008,
         logRetention: aws_logs.RetentionDays.ONE_WEEK,
-        environment: titilerPgstacEnv,
-      });
-
+        timeout: Duration.seconds(30)
+      })
+      
       // grant access to buckets using addToRolePolicy
       if (props.buckets) {
         props.buckets.forEach(bucket => {
@@ -105,8 +115,9 @@ import {
     readonly dbSecret: secretsmanager.ISecret;
 
     /**
-     * Customized environment variables to send to titiler-pgstac runtime.
-     */
+     * Customized environment variables to send to titiler-pgstac runtime. These will be merged with `defaultTitilerPgstacEnv`.
+     * The database secret arn is automatically added to the environment variables at deployment. 
+    /*/
     readonly apiEnv?: Record<string, string>;
 
     /**
@@ -116,6 +127,57 @@ import {
 
     /**
      * Custom Domain Name Options for Titiler Pgstac API,
+     * 
+     * @default - undefined. 
      */
     readonly titilerPgstacApiDomainName?: IDomainName;
+
+    /**
+     * Optional settings for the titiler-pgstac python lambda function.
+     *
+     * @default - defined in the construct.
+     */
+    readonly pythonLambdaOptions?: TitilerPgstacPythonLambdaOptions;
+
+  }
+
+
+  export interface TitilerPgstacPythonLambdaOptions {
+
+    /**
+     * Path to the source of the function or the location for dependencies.
+     */
+    readonly entry: string;
+    /**
+     * The runtime environment. Only runtimes of the Python family are
+     * supported.
+     */
+    readonly runtime: Runtime;
+
+    /**
+     * The path (relative to entry) to the index file containing the exported handler.
+     *
+     */
+    readonly index: string;
+    /**
+     * The name of the exported handler in the index file.
+     */
+    readonly handler: string;
+
+    /**
+     * Bundling options to use for this function. Use this to specify custom bundling options like
+     * the bundling Docker image, asset hash type, custom hash, architecture, etc.
+     */
+    readonly bundling?: BundlingOptions;
+
+    /**
+     * The amount of memory, in MB, that is allocated to your Lambda function.
+     */
+    readonly memorySize: number;
+
+    /**
+     * The system architectures compatible with this lambda function.
+     */
+    readonly architecture: lambda.Architecture;
+
   }

lib/titiler-pgstac-api/runtime/Dockerfile

@@ -1 +1,2 @@
-uvicorn
+uvicorn
+boto3>=1.26.139

lib/titiler-pgstac-api/runtime/dev_requirements.txt

@@ -1,3 +1,3 @@
 titiler.pgstac==0.5.1
-boto3>=1.26.139
 psycopg[binary, pool]
+mangum>=0.14,<0.15

lib/titiler-pgstac-api/runtime/requirements.txt

@@ -5,7 +5,7 @@
 import asyncio
 import os
 from mangum import Mangum
-from utils import get_secret_dict
+from src.utils import get_secret_dict
 
 pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"]