feat: update runtimes and add pgstac customization options (#100)

* feat: update runtimes and add pgstac customization options

* Update lib/tipg-api/runtime/requirements.txt

Co-authored-by: Emile Tenezakis  <[email protected]>

* change defaults

* revert version updates

* Revert "revert version updates"

This reverts commit 0d675e2.

* back to 0.7.1

* (fix): fix database bootstrap (#102)

* (fix): fix database bootstrap

* fix handler

* update to pgstac 0.8.5

---------

Co-authored-by: Emile Tenezakis <[email protected]>

Author: vincentsarago

lib/database/bootstrapper_runtime/Dockerfile

@@ -17,4 +17,4 @@ RUN rm -rf /asset/asyncio*
 
 # A command must be present avoid the following error on CDK deploy:
 # Error response from daemon: No command specified
-CMD [ "echo", "ready to go!" ]
+CMD [ "echo", "ready to go!" ]

lib/database/bootstrapper_runtime/handler.py

@@ -4,6 +4,7 @@
 """
 
 import json
+import logging
 
 import boto3
 import httpx
@@ -13,6 +14,8 @@
 from pypgstac.db import PgstacDB
 from pypgstac.migrate import Migrate
 
+logger = logging.getLogger("eoapi-bootstrap")
+
 
 def send(
     event,
@@ -36,10 +39,6 @@ def send(
     It isn't available for source code that's stored in Amazon S3 buckets.
     For code in buckets, you must write your own functions to send responses.
     """
-    responseUrl = event["ResponseURL"]
-
-    print(responseUrl)
-
     responseBody = {}
     responseBody["Status"] = responseStatus
     responseBody["Reason"] = (
@@ -53,21 +52,21 @@ def send(
     responseBody["Data"] = responseData
 
     json_responseBody = json.dumps(responseBody)
-
-    print("Response body:\n" + json_responseBody)
-
-    headers = {"content-type": "", "content-length": str(len(json_responseBody))}
+    print("Response body:\n     " + json_responseBody)
 
     try:
         response = httpx.put(
-            responseUrl,
+            event["ResponseURL"],
             data=json_responseBody,
-            headers=headers,
+            headers={"content-type": "", "content-length": str(len(json_responseBody))},
             timeout=30,
         )
-        print("Status code: " + response.status_code)
+        print("Status code: ", response.status_code)
+        logger.debug(f"OK - Status code: {response.status_code}")
+
     except Exception as e:
         print("send(..) failed executing httpx.put(..): " + str(e))
+        logger.debug(f"NOK - failed executing PUT requests:  {e}")
 
 
 def get_secret(secret_name):
@@ -84,9 +83,9 @@ def create_db(cursor, db_name: str) -> None:
         sql.SQL("SELECT 1 FROM pg_catalog.pg_database " "WHERE datname = %s"), [db_name]
     )
     if cursor.fetchone():
-        print(f"database {db_name} exists, not creating DB")
+        print(f"    database {db_name} exists, not creating DB")
     else:
-        print(f"database {db_name} not found, creating...")
+        print(f"    database {db_name} not found, creating...")
         cursor.execute(
             sql.SQL("CREATE DATABASE {db_name}").format(db_name=sql.Identifier(db_name))
         )
@@ -114,8 +113,8 @@ def create_user(cursor, username: str, password: str) -> None:
     )
 
 
-def create_permissions(cursor, db_name: str, username: str) -> None:
-    """Add permissions."""
+def update_user_permissions(cursor, db_name: str, username: str) -> None:
+    """Update eoAPI user permissions."""
     cursor.execute(
         sql.SQL(
             "GRANT CONNECT ON DATABASE {db_name} TO {username};"
@@ -140,6 +139,33 @@ def register_extensions(cursor) -> None:
     cursor.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS postgis;"))
 
 
+###############################################################################
+# PgSTAC Customization
+###############################################################################
+def customization(cursor, params) -> None:
+    """
+    CUSTOMIZED YOUR PGSTAC DATABASE
+
+    ref: https://github.com/stac-utils/pgstac/blob/main/docs/src/pgstac.md
+
+    """
+    if str(params.get("context", "FALSE")).upper() == "TRUE":
+        # Add CONTEXT=ON
+        pgstac_settings = """
+        INSERT INTO pgstac_settings (name, value)
+        VALUES ('context', 'on')
+        ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;"""
+        cursor.execute(sql.SQL(pgstac_settings))
+
+    if str(params.get("mosaic_index", "TRUE")).upper() == "TRUE":
+        # Create index of searches with `mosaic`` type
+        cursor.execute(
+            sql.SQL(
+                "CREATE INDEX IF NOT EXISTS searches_mosaic ON searches ((true)) WHERE metadata->>'type'='mosaic';"
+            )
+        )
+
+
 def handler(event, context):
     """Lambda Handler."""
     print(f"Handling {event}")
@@ -149,88 +175,90 @@ def handler(event, context):
 
     try:
         params = event["ResourceProperties"]
-        connection_params = get_secret(params["conn_secret_arn"])
-        user_params = get_secret(params["new_user_secret_arn"])
-
-        print("Connecting to admin DB...")
-        admin_db_conninfo = make_conninfo(
-            dbname=connection_params.get("dbname", "postgres"),
-            user=connection_params["username"],
-            password=connection_params["password"],
-            host=connection_params["host"],
-            port=connection_params["port"],
+
+        # Admin (AWS RDS) user/password/dbname parameters
+        admin_params = get_secret(params["conn_secret_arn"])
+
+        # Custom eoAPI user/password/dbname parameters
+        eoapi_params = get_secret(params["new_user_secret_arn"])
+
+        print("Connecting to RDS...")
+        rds_conninfo = make_conninfo(
+            dbname=admin_params.get("dbname", "postgres"),
+            user=admin_params["username"],
+            password=admin_params["password"],
+            host=admin_params["host"],
+            port=admin_params["port"],
         )
-        with psycopg.connect(admin_db_conninfo, autocommit=True) as conn:
+        with psycopg.connect(rds_conninfo, autocommit=True) as conn:
             with conn.cursor() as cur:
-                print("Creating database...")
+                print(f"Creating eoAPI '{eoapi_params['dbname']}' database...")
                 create_db(
                     cursor=cur,
-                    db_name=user_params["dbname"],
+                    db_name=eoapi_params["dbname"],
                 )
 
-                print("Creating user...")
+                print(f"Creating eoAPI '{eoapi_params['username']}' user...")
                 create_user(
                     cursor=cur,
-                    username=user_params["username"],
-                    password=user_params["password"],
+                    username=eoapi_params["username"],
+                    password=eoapi_params["password"],
                 )
 
-        # Install extensions on the user DB with
-        # superuser permissions, since they will
-        # otherwise fail to install when run as
-        # the non-superuser within the pgstac
-        # migrations.
-        print("Connecting to STAC DB...")
-        stac_db_conninfo = make_conninfo(
-            dbname=user_params["dbname"],
-            user=connection_params["username"],
-            password=connection_params["password"],
-            host=connection_params["host"],
-            port=connection_params["port"],
+        # Install postgis and pgstac on the eoapi database with
+        # superuser permissions
+        print(f"Connecting to eoAPI '{eoapi_params['dbname']}' database...")
+        eoapi_db_admin_conninfo = make_conninfo(
+            dbname=eoapi_params["dbname"],
+            user=admin_params["username"],
+            password=admin_params["password"],
+            host=admin_params["host"],
+            port=admin_params["port"],
         )
-        with psycopg.connect(stac_db_conninfo, autocommit=True) as conn:
+        with psycopg.connect(eoapi_db_admin_conninfo, autocommit=True) as conn:
             with conn.cursor() as cur:
-                print("Registering PostGIS ...")
+                print(
+                    f"Registering Extension in '{eoapi_params['dbname']}' database..."
+                )
                 register_extensions(cursor=cur)
 
-        stac_db_admin_dsn = (
-            "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(
-                dbname=user_params.get("dbname", "postgres"),
-                user=connection_params["username"],
-                password=connection_params["password"],
-                host=connection_params["host"],
-                port=connection_params["port"],
-            )
-        )
-
-        pgdb = PgstacDB(dsn=stac_db_admin_dsn, debug=True)
-        print(f"Current {pgdb.version=}")
+            print("Starting PgSTAC Migration ")
+            with PgstacDB(connection=conn, debug=True) as pgdb:
+                print(f"Current PgSTAC Version: {pgdb.version}")
 
-        # As admin, run migrations
-        print("Running migrations...")
-        Migrate(pgdb).run_migration(params["pgstac_version"])
-
-        # Assign appropriate permissions to user (requires pgSTAC migrations to have run)
-        with psycopg.connect(admin_db_conninfo, autocommit=True) as conn:
-            with conn.cursor() as cur:
-                print("Setting permissions...")
-                create_permissions(
-                    cursor=cur,
-                    db_name=user_params["dbname"],
-                    username=user_params["username"],
-                )
+                print(f"Running migrations to PgSTAC {params['pgstac_version']}...")
+                Migrate(pgdb).run_migration(params["pgstac_version"])
 
-        print("Adding mosaic index...")
         with psycopg.connect(
-            stac_db_admin_dsn,
+            eoapi_db_admin_conninfo,
             autocommit=True,
             options="-c search_path=pgstac,public -c application_name=pgstac",
         ) as conn:
-            conn.execute(
-                sql.SQL(
-                    "CREATE INDEX IF NOT EXISTS searches_mosaic ON searches ((true)) WHERE metadata->>'type'='mosaic';"
+            print("Customize PgSTAC database...")
+            # Update permissions to eoAPI user to assume pgstac_* roles
+            with conn.cursor() as cur:
+                print(f"Update '{eoapi_params['username']}' permissions...")
+                update_user_permissions(
+                    cursor=cur,
+                    db_name=eoapi_params["dbname"],
+                    username=eoapi_params["username"],
                 )
+
+                customization(cursor=cur, params=params)
+
+        # Make sure the user can access the database
+        eoapi_user_dsn = (
+            "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(
+                dbname=eoapi_params["dbname"],
+                user=eoapi_params["username"],
+                password=eoapi_params["password"],
+                host=admin_params["host"],
+                port=admin_params["port"],
             )
+        )
+        print("Checking eoAPI user access to the PgSTAC database...")
+        with PgstacDB(dsn=eoapi_user_dsn, debug=True) as pgdb:
+            print(f"    OK - User has access to pgstac db, pgstac schema version: {pgdb.version}")
 
     except Exception as e:
         print(f"Unable to bootstrap database with exception={e}")

lib/database/index.ts

@@ -14,7 +14,12 @@ import { Construct } from "constructs";
 import { CustomLambdaFunctionProps } from "../utils";
 
 const instanceSizes: Record<string, number> = require("./instance-memory.json");
-const DEFAULT_PGSTAC_VERSION = "0.7.10";
+const DEFAULT_PGSTAC_VERSION = "0.8.5";
+
+let defaultPgSTACCustomOptions :{ [key: string]: any } = {
+  "context": "FALSE",
+  "mosaic_index": "TRUE"
+}
 
 function hasVpc(
   instance: rds.DatabaseInstance | rds.IDatabaseInstance
@@ -106,12 +111,7 @@ export class PgStacDatabase extends Construct {
     // connect to database
     this.db.connections.allowFrom(handler, ec2.Port.tcp(5432));
 
-    let customResourceProperties : { [key: string]: any} = {};
-
-    // if customResourceProperties are provided, fill in the values. 
-    if (props.customResourceProperties) {
-      Object.assign(customResourceProperties, props.customResourceProperties);
-    }
+    let customResourceProperties : { [key: string]: any} = props.customResourceProperties ? { ...defaultPgSTACCustomOptions, ...props.customResourceProperties } : defaultPgSTACCustomOptions;
 
     // update properties
     customResourceProperties["conn_secret_arn"] = this.db.secret!.secretArn;
@@ -195,7 +195,7 @@ export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps {
   /**
    * Lambda function Custom Resource properties. A custom resource property is going to be created
    * to trigger the boostrapping lambda function. This parameter allows the user to specify additional properties
-   * on top of the defaults ones. 
+   * on top of the defaults ones.
    *
    */
   readonly customResourceProperties?: {

lib/ingestor-api/runtime/requirements.txt

@@ -5,7 +5,7 @@ orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
 pydantic_ssm_settings>=0.2.0
 pydantic>=1.9.0
-pypgstac==0.7.10
+pypgstac==0.8.5
 requests>=2.27.1
 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116
 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1

lib/tipg-api/runtime/Dockerfile

@@ -4,7 +4,8 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
 WORKDIR /tmp
 RUN python -m pip install pip -U
 
-RUN python -m pip install tipg==0.3.1 "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
+COPY runtime/requirements.txt requirements.txt
+RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
 
 # Reduce package size and remove useless files
 RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
@@ -14,4 +15,4 @@ RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
 
 COPY runtime/src/*.py /asset/
 
-CMD ["echo", "hello world"]
+CMD ["echo", "hello world"]

lib/tipg-api/runtime/requirements.txt

@@ -0,0 +1 @@
+tipg==0.6.3

lib/titiler-pgstac-api/runtime/requirements.txt

@@ -1,2 +1,2 @@
-titiler.pgstac==0.5.1
-psycopg[binary, pool]
+titiler.pgstac==1.2.2
+psycopg[binary, pool]