feat(titiler-pgstac-api): add titiler-pgstac endpoint (#42)

* added a new feature, deploying a titiler-pgstac service with access to the pgstac database, and exposing its API.

Author: emileten

lib/index.ts

@@ -3,3 +3,4 @@ export * from "./bootstrapper";
 export * from "./database";
 export * from "./ingestor-api";
 export * from "./stac-api";
+export * from "./titiler-pgstac-api";

lib/titiler-pgstac-api/index.ts

@@ -0,0 +1,114 @@
+import {
+    Stack,
+    aws_iam as iam,
+    aws_ec2 as ec2,
+    aws_rds as rds,
+    aws_lambda as lambda,
+    aws_secretsmanager as secretsmanager,
+    CfnOutput,
+    Duration,
+    aws_logs,
+  } from "aws-cdk-lib";
+  import { HttpApi } from "@aws-cdk/aws-apigatewayv2-alpha";
+  import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha";
+  import { Construct } from "constructs";
+  
+  export class TitilerPgstacApiLambda extends Construct {
+    readonly url: string;
+    public titilerPgstacLambdaFunction: lambda.Function;
+  
+    constructor(scope: Construct, id: string, props: TitilerPgStacApiLambdaProps) {
+      super(scope, id);
+  
+      const titilerPgstacEnv = {
+          "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff",
+          "GDAL_CACHEMAX": "200", 
+          "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+          "GDAL_INGESTED_BYTES_AT_OPEN": "32768",
+          "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
+          "GDAL_HTTP_MULTIPLEX": "YES",
+          "GDAL_HTTP_VERSION": "2",
+          "PYTHONWARNINGS": "ignore",
+          "VSI_CACHE": "TRUE",
+          "VSI_CACHE_SIZE": "5000000", 
+          "DB_MIN_CONN_SIZE": "1",
+          "DB_MAX_CONN_SIZE": "1",
+          "PGSTAC_SECRET_ARN": props.dbSecret.secretArn,
+      }
+    
+      
+      this.titilerPgstacLambdaFunction = new lambda.Function(this, "lambda", {
+        handler: "handler.handler",
+        runtime: lambda.Runtime.PYTHON_3_8,
+        code: lambda.Code.fromDockerBuild(__dirname, {
+          file: "runtime/Dockerfile",
+          buildArgs: { PYTHON_VERSION: '3.10' },
+        }),
+        timeout: Duration.seconds(30),
+        vpc: props.vpc,
+        vpcSubnets: props.subnetSelection,
+        allowPublicSubnet: true,
+        memorySize: 3008,
+        logRetention: aws_logs.RetentionDays.ONE_WEEK,
+        environment: titilerPgstacEnv,
+      });
+
+      // grant access to buckets using addToRolePolicy
+      if (props.buckets) {
+        props.buckets.forEach(bucket => {
+          this.titilerPgstacLambdaFunction.addToRolePolicy(new iam.PolicyStatement({
+            actions: ["s3:GetObject"],
+            resources: [`arn:aws:s3:::${bucket}/*`],
+          }));
+        });
+      }
+      
+      props.dbSecret.grantRead(this.titilerPgstacLambdaFunction);
+      this.titilerPgstacLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432), "allow connections from titiler");
+  
+      const stacApi = new HttpApi(this, `${Stack.of(this).stackName}-titiler-pgstac-api`, {
+        defaultIntegration: new HttpLambdaIntegration("integration", this.titilerPgstacLambdaFunction),
+      });
+  
+      this.url = stacApi.url!;
+  
+      new CfnOutput(this, "titiler-pgstac-api-output", {
+        exportName: `${Stack.of(this).stackName}-titiler-pgstac-url`,
+        value: this.url,
+      });
+    }
+  }
+  
+  export interface TitilerPgStacApiLambdaProps {
+
+    /**
+     * VPC into which the lambda should be deployed.
+     */
+    readonly vpc: ec2.IVpc;
+  
+    /**
+     * RDS Instance with installed pgSTAC.
+     */
+    readonly db: rds.IDatabaseInstance;
+  
+    /**
+     * Subnet into which the lambda should be deployed.
+     */
+    readonly subnetSelection: ec2.SubnetSelection;
+  
+    /**
+     * Secret containing connection information for pgSTAC database.
+     */
+    readonly dbSecret: secretsmanager.ISecret;
+
+    /**
+     * Customized environment variables to send to titiler-pgstac runtime.
+     */
+    readonly apiEnv?: Record<string, string>;
+
+    /**
+     * list of buckets the lambda will be granted access to. 
+     */
+    readonly buckets?: string[];
+
+  }

lib/titiler-pgstac-api/runtime/Dockerfile

@@ -0,0 +1,20 @@
+ARG PYTHON_VERSION
+
+FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
+
+WORKDIR /tmp
+RUN python -m pip install pip -U
+
+COPY runtime/requirements.txt requirements.txt
+RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset
+
+# Reduce package size and remove useless files
+RUN find /asset -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
+RUN find /asset -type d -name '__pycache__' -print0 | xargs -0 rm -rf
+RUN find /asset -type f -name '*.py' -print0 | xargs -0 rm -f
+RUN find /asset -type d -name 'tests' -print0 | xargs -0 rm -rf
+RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc
+
+COPY runtime/src/*.py /asset
+
+CMD ["echo", "hello world"]

lib/titiler-pgstac-api/runtime/dev_requirements.txt

@@ -0,0 +1 @@
+uvicorn

lib/titiler-pgstac-api/runtime/requirements.txt

@@ -0,0 +1,3 @@
+titiler.pgstac==0.3.3
+boto3>=1.26.139
+psycopg[binary, pool]

lib/titiler-pgstac-api/runtime/src/__init__.py

@@ -0,0 +1,3 @@
+"""eoapi.raster."""
+
+__version__ = "0.1.0"

lib/titiler-pgstac-api/runtime/src/handler.py

@@ -0,0 +1,23 @@
+"""
+Handler for AWS Lambda.
+"""
+
+import os
+from mangum import Mangum
+from utils import get_secret_dict
+from titiler.pgstac.main import app
+
+pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"]
+
+secret = get_secret_dict(pgstac_secret_arn)
+os.environ.update(
+    {
+        "postgres_host": secret["host"],
+        "postgres_dbname": secret["dbname"],
+        "postgres_user": secret["username"],
+        "postgres_pass": secret["password"],
+        "postgres_port": str(secret["port"]),
+    }
+)
+
+handler = Mangum(app)

lib/titiler-pgstac-api/runtime/src/utils.py

@@ -0,0 +1,26 @@
+import base64
+import json
+import boto3
+
+
+def get_secret_dict(secret_name: str):
+    """Retrieve secrets from AWS Secrets Manager
+
+    Args:
+        secret_name (str): name of aws secrets manager secret containing database connection secrets
+        profile_name (str, optional): optional name of aws profile for use in debugger only
+
+    Returns:
+        secrets (dict): decrypted secrets in dict
+    """
+
+    # Create a Secrets Manager client
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager")
+
+    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+
+    if "SecretString" in get_secret_value_response:
+        return json.loads(get_secret_value_response["SecretString"])
+    else:
+        return json.loads(base64.b64decode(get_secret_value_response["SecretBinary"]))