From 34f3a12747c5ba8e2d682527e98f3d22f8079926 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 09:55:08 -0600 Subject: [PATCH 01/11] update stac-fastapi-pgstac to 4.0 --- lib/stac-api/runtime/requirements.txt | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/stac-api/runtime/requirements.txt b/lib/stac-api/runtime/requirements.txt index cbc95bb..aa4c612 100644 --- a/lib/stac-api/runtime/requirements.txt +++ b/lib/stac-api/runtime/requirements.txt @@ -1,7 +1,2 @@ -stac-fastapi.api==2.4.8 -stac-fastapi.extensions==2.4.8 -stac-fastapi.pgstac==2.4.8 -stac-fastapi.types==2.4.8 -# https://github.com/stac-utils/stac-fastapi/pull/466 -pygeoif==0.7 -starlette_cramjam +stac-fastapi-pgstac>=4.0.2,<4.1 +starlette-cramjam>=0.4,<0.5 From ee2f91c31376128d082aeea11fbc5c49c1b50fd9 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 09:59:29 -0600 Subject: [PATCH 02/11] upgrade to titiler-pgstac==1.7.0 --- lib/titiler-pgstac-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/titiler-pgstac-api/runtime/requirements.txt b/lib/titiler-pgstac-api/runtime/requirements.txt index 229e4b7..e8b7026 100644 --- a/lib/titiler-pgstac-api/runtime/requirements.txt +++ b/lib/titiler-pgstac-api/runtime/requirements.txt @@ -1,2 +1,2 @@ -titiler.pgstac==1.2.2 +titiler.pgstac==1.7.0 psycopg[binary, pool] From 00a93f6a78bc2dac322efef031b9b036af08c0af Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 10:02:22 -0600 Subject: [PATCH 03/11] upgrade to tipg==0.9.0 --- lib/tipg-api/runtime/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tipg-api/runtime/requirements.txt b/lib/tipg-api/runtime/requirements.txt index f30e289..c7094c7 100644 --- a/lib/tipg-api/runtime/requirements.txt +++ b/lib/tipg-api/runtime/requirements.txt @@ -1 +1 @@ -tipg==0.6.3 +tipg==0.9.0 From c7f3ffab9465c34a5092af88363a33bab944a9cf Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 10:23:45 -0600 Subject: [PATCH 04/11] update default pgstac to 0.9.5 --- lib/database/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/database/index.ts b/lib/database/index.ts index 1510cb9..47e60d2 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -14,7 +14,7 @@ import { CustomLambdaFunctionProps } from "../utils"; import { PgBouncer } from "./PgBouncer"; const instanceSizes: Record = require("./instance-memory.json"); -const DEFAULT_PGSTAC_VERSION = "0.8.5"; +const DEFAULT_PGSTAC_VERSION = "0.9.5"; let defaultPgSTACCustomOptions: { [key: string]: any } = { context: "FALSE", From 9805c22235c06ce105d684eb86fc2ca7746fbfbd Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 14:38:54 -0600 Subject: [PATCH 05/11] work out some changes to stac-api and titiler-pgstac handlers --- lib/stac-api/runtime/src/app.py | 51 ++++++++----- lib/stac-api/runtime/src/config.py | 73 +++++-------------- lib/stac-api/runtime/src/handler.py | 16 ++++ lib/titiler-pgstac-api/runtime/Dockerfile | 6 ++ lib/titiler-pgstac-api/runtime/src/handler.py | 22 +++++- 5 files changed, 95 insertions(+), 73 deletions(-) diff --git a/lib/stac-api/runtime/src/app.py b/lib/stac-api/runtime/src/app.py index 0f27bec..fb2d724 100644 --- a/lib/stac-api/runtime/src/app.py +++ b/lib/stac-api/runtime/src/app.py @@ -2,40 +2,57 @@ FastAPI application using PGStac. """ +from brotli_asgi import BrotliMiddleware +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import ORJSONResponse from stac_fastapi.api.app import StacApi +from stac_fastapi.api.middleware import ProxyHeaderMiddleware +from stac_fastapi.pgstac.app import ( + application_extensions, + collections_get_request_model, + get_request_model, + items_get_request_model, + post_request_model, +) from stac_fastapi.pgstac.core import CoreCrudClient from stac_fastapi.pgstac.db import close_db_connection, connect_to_db -from starlette_cramjam.middleware import CompressionMiddleware +from starlette.middleware import Middleware from .config import ApiSettings -from .config import extensions as PgStacExtensions -from .config import get_request_model as GETModel -from .config import post_request_model as POSTModel -api_settings = ApiSettings() +settings = ApiSettings() api = StacApi( - title=api_settings.name, - api_version=api_settings.version, - description=api_settings.description or api_settings.name, - settings=api_settings.load_postgres_settings(), - extensions=PgStacExtensions, - client=CoreCrudClient(post_request_model=POSTModel), - search_get_request_model=GETModel, - search_post_request_model=POSTModel, + app=FastAPI( + openapi_url=settings.openapi_url, + docs_url=settings.docs_url, + redoc_url=None, + root_path=settings.root_path, + title=settings.stac_fastapi_title, + version=settings.stac_fastapi_version, + description=settings.stac_fastapi_description, + ), + settings=settings, + extensions=application_extensions, + client=CoreCrudClient(pgstac_search_model=post_request_model), response_class=ORJSONResponse, - middlewares=[CompressionMiddleware], + items_get_request_model=items_get_request_model, + search_get_request_model=get_request_model, + search_post_request_model=post_request_model, + collections_get_request_model=collections_get_request_model, + middlewares=[ + Middleware(BrotliMiddleware), + Middleware(ProxyHeaderMiddleware), + ], ) - app = api.app # Set all CORS enabled origins -if api_settings.cors_origins: +if settings.cors_origins: app.add_middleware( CORSMiddleware, - allow_origins=api_settings.cors_origins, + allow_origins=settings.cors_origins, allow_credentials=True, allow_methods=["GET", "POST", "OPTIONS"], allow_headers=["*"], diff --git a/lib/stac-api/runtime/src/config.py b/lib/stac-api/runtime/src/config.py index 948dc8a..e0198db 100644 --- a/lib/stac-api/runtime/src/config.py +++ b/lib/stac-api/runtime/src/config.py @@ -3,23 +3,14 @@ import base64 import json -from typing import Optional +from typing import Any, Optional import boto3 -import pydantic -from stac_fastapi.api.models import create_get_request_model, create_post_request_model # from stac_fastapi.pgstac.extensions import QueryExtension -from stac_fastapi.extensions.core import ( - ContextExtension, - FieldsExtension, - FilterExtension, - QueryExtension, - SortExtension, - TokenPaginationExtension, -) +from pydantic import model_validator from stac_fastapi.pgstac.config import Settings -from stac_fastapi.pgstac.types.search import PgstacSearch +from stac_fastapi.pgstac.version import __version__ as stac_fastapi_pgstac_version def get_secret_dict(secret_name: str): @@ -27,7 +18,6 @@ def get_secret_dict(secret_name: str): Args: secret_name (str): name of aws secrets manager secret containing database connection secrets - profile_name (str, optional): optional name of aws profile for use in debugger only Returns: secrets (dict): decrypted secrets in dict @@ -45,53 +35,30 @@ def get_secret_dict(secret_name: str): return json.loads(base64.b64decode(get_secret_value_response["SecretBinary"])) -class ApiSettings(pydantic.BaseSettings): +class ApiSettings(Settings): """API settings""" - name: str = "asdi-stac-api" - version: str = "0.1" + name: str = "stac-fastapi-pgstac" + version: str = stac_fastapi_pgstac_version description: Optional[str] = None - cors_origins: str = "*" cachecontrol: str = "public, max-age=3600" debug: bool = False pgstac_secret_arn: Optional[str] - @pydantic.validator("cors_origins") - def parse_cors_origin(cls, v): - """Parse CORS origins.""" - return [origin.strip() for origin in v.split(",")] - - def load_postgres_settings(self) -> "Settings": - """Load postgres connection params from AWS secret""" - - if self.pgstac_secret_arn: - secret = get_secret_dict(self.pgstac_secret_arn) - - return Settings( - postgres_host_reader=secret["host"], - postgres_host_writer=secret["host"], - postgres_dbname=secret["dbname"], - postgres_user=secret["username"], - postgres_pass=secret["password"], - postgres_port=secret["port"], + @model_validator(mode="before") + def get_postgres_setting(cls, data: Any) -> Any: + if arn := data.get("pgstac_secret_arn"): + secret = get_secret_dict(arn) + data.update( + { + "postgres_host_reader": secret["host"], + "postgres_host_writer": secret["host"], + "postgres_dbname": secret["dbname"], + "postgres_user": secret["username"], + "postgres_pass": secret["password"], + "postgres_port": secret["port"], + } ) - else: - return Settings() - - class Config: - """model config""" - - env_file = ".env" - -extensions = [ - FilterExtension(), - QueryExtension(), - SortExtension(), - FieldsExtension(), - TokenPaginationExtension(), - ContextExtension(), -] -post_request_model = create_post_request_model(extensions, base_model=PgstacSearch) -get_request_model = create_get_request_model(extensions) + return data diff --git a/lib/stac-api/runtime/src/handler.py b/lib/stac-api/runtime/src/handler.py index e4a4745..335707e 100644 --- a/lib/stac-api/runtime/src/handler.py +++ b/lib/stac-api/runtime/src/handler.py @@ -7,6 +7,22 @@ from mangum import Mangum +from .config import get_secret_dict + +pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"] + +secret = get_secret_dict(pgstac_secret_arn) +os.environ.update( + { + "postgres_host_reader": secret["host"], + "postgres_host_writer": secret["host"], + "postgres_dbname": secret["dbname"], + "postgres_user": secret["username"], + "postgres_pass": secret["password"], + "postgres_port": str(secret["port"]), + } +) + from .app import app handler = Mangum(app, lifespan="off") diff --git a/lib/titiler-pgstac-api/runtime/Dockerfile b/lib/titiler-pgstac-api/runtime/Dockerfile index d0722e3..e62a549 100644 --- a/lib/titiler-pgstac-api/runtime/Dockerfile +++ b/lib/titiler-pgstac-api/runtime/Dockerfile @@ -20,6 +20,12 @@ RUN find /asset -type f -name '*.py' -print0 | xargs -0 rm -f RUN find /asset -type d -name 'tests' -print0 | xargs -0 rm -rf RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc +# Strip debug symbols from compiled C/C++ code (except for numpy.libs!) +RUN cd /asset && \ + find . -type f -name '*.so*' \ + -not -path "./numpy.libs/*" \ + -exec strip --strip-unneeded {} \; + COPY runtime/src/*.py /asset/ CMD ["echo", "hello world"] diff --git a/lib/titiler-pgstac-api/runtime/src/handler.py b/lib/titiler-pgstac-api/runtime/src/handler.py index 8f8880c..81e478a 100644 --- a/lib/titiler-pgstac-api/runtime/src/handler.py +++ b/lib/titiler-pgstac-api/runtime/src/handler.py @@ -5,8 +5,7 @@ import asyncio import os -from mangum import Mangum -from utils import get_secret_dict +from .utils import get_secret_dict pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"] @@ -21,14 +20,31 @@ } ) +from mangum import Mangum from titiler.pgstac.db import connect_to_db # noqa: E402 from titiler.pgstac.main import app # noqa: E402 +from titiler.pgstac.settings import PostgresSettings +from utils import get_secret_dict @app.on_event("startup") async def startup_event() -> None: """Connect to database on startup.""" - await connect_to_db(app) + pgstac_secret_arn = os.getenv("PGSTAC_SECRET_ARN") + if not pgstac_secret_arn: + raise ValueError("PGSTAC_SECRET_ARN is not set!") + + secret = get_secret_dict(pgstac_secret_arn) + await connect_to_db( + app, + settings=PostgresSettings( + postgres_user=secret["username"], + postgres_pass=secret["password"], + postgres_host=secret["host"], + postgres_port=secret["port"], + postgres_dbname=secret["dbname"], + ), + ) handler = Mangum(app, lifespan="off") From ce1d56aca96307b05f526df8778727d0ee9fcde6 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 15:13:51 -0600 Subject: [PATCH 06/11] fix imports --- lib/titiler-pgstac-api/runtime/src/handler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/titiler-pgstac-api/runtime/src/handler.py b/lib/titiler-pgstac-api/runtime/src/handler.py index 81e478a..e4038ee 100644 --- a/lib/titiler-pgstac-api/runtime/src/handler.py +++ b/lib/titiler-pgstac-api/runtime/src/handler.py @@ -5,7 +5,8 @@ import asyncio import os -from .utils import get_secret_dict +from mangum import Mangum +from utils import get_secret_dict pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"] @@ -19,12 +20,9 @@ "postgres_port": str(secret["port"]), } ) - -from mangum import Mangum from titiler.pgstac.db import connect_to_db # noqa: E402 from titiler.pgstac.main import app # noqa: E402 from titiler.pgstac.settings import PostgresSettings -from utils import get_secret_dict @app.on_event("startup") From 340299efe6c5c204e68655ea71af715268674420 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 3 Mar 2025 20:32:01 -0600 Subject: [PATCH 07/11] clean up titiler-pgstac handler --- lib/titiler-pgstac-api/runtime/src/handler.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/lib/titiler-pgstac-api/runtime/src/handler.py b/lib/titiler-pgstac-api/runtime/src/handler.py index e4038ee..7234117 100644 --- a/lib/titiler-pgstac-api/runtime/src/handler.py +++ b/lib/titiler-pgstac-api/runtime/src/handler.py @@ -4,13 +4,22 @@ import asyncio import os +from functools import lru_cache from mangum import Mangum from utils import get_secret_dict -pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"] -secret = get_secret_dict(pgstac_secret_arn) +@lru_cache(maxsize=1) +def get_db_credentials(): + """Cache the credentials to avoid repeated Secrets Manager calls.""" + pgstac_secret_arn = os.environ.get("PGSTAC_SECRET_ARN") + if not pgstac_secret_arn: + raise ValueError("PGSTAC_SECRET_ARN is not set!") + return get_secret_dict(pgstac_secret_arn) + + +secret = get_db_credentials() os.environ.update( { "postgres_host": secret["host"], @@ -20,29 +29,15 @@ "postgres_port": str(secret["port"]), } ) + from titiler.pgstac.db import connect_to_db # noqa: E402 from titiler.pgstac.main import app # noqa: E402 -from titiler.pgstac.settings import PostgresSettings @app.on_event("startup") async def startup_event() -> None: """Connect to database on startup.""" - pgstac_secret_arn = os.getenv("PGSTAC_SECRET_ARN") - if not pgstac_secret_arn: - raise ValueError("PGSTAC_SECRET_ARN is not set!") - - secret = get_secret_dict(pgstac_secret_arn) - await connect_to_db( - app, - settings=PostgresSettings( - postgres_user=secret["username"], - postgres_pass=secret["password"], - postgres_host=secret["host"], - postgres_port=secret["port"], - postgres_dbname=secret["dbname"], - ), - ) + await connect_to_db(app) handler = Mangum(app, lifespan="off") From ea3c44ef32ee17e9ae5bdcb5d8199bbc8b3947ec Mon Sep 17 00:00:00 2001 From: hrodmn Date: Tue, 4 Mar 2025 05:44:39 -0600 Subject: [PATCH 08/11] no more custom runtimes --- lib/stac-api/index.ts | 7 +- lib/stac-api/runtime/Dockerfile | 5 +- lib/stac-api/runtime/src/app.py | 75 ------------------- lib/stac-api/runtime/src/config.py | 64 ---------------- lib/stac-api/runtime/src/handler.py | 26 +++++-- lib/tipg-api/index.ts | 33 ++++---- lib/tipg-api/runtime/Dockerfile | 5 +- lib/tipg-api/runtime/src/handler.py | 13 +++- lib/tipg-api/runtime/src/utils.py | 42 ----------- lib/titiler-pgstac-api/index.ts | 32 ++++---- lib/titiler-pgstac-api/runtime/Dockerfile | 5 +- lib/titiler-pgstac-api/runtime/src/handler.py | 13 +--- .../runtime/src => utils}/utils.py | 11 ++- 13 files changed, 86 insertions(+), 245 deletions(-) delete mode 100644 lib/stac-api/runtime/src/app.py delete mode 100644 lib/stac-api/runtime/src/config.py delete mode 100644 lib/tipg-api/runtime/src/utils.py rename lib/{titiler-pgstac-api/runtime/src => utils}/utils.py (67%) diff --git a/lib/stac-api/index.ts b/lib/stac-api/index.ts index 8556e77..8b18006 100644 --- a/lib/stac-api/index.ts +++ b/lib/stac-api/index.ts @@ -12,6 +12,7 @@ import { IDomainName, HttpApi, ParameterMapping, MappingValue} from "@aws-cdk/aw import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; +import * as path from 'path'; export class PgStacApiLambda extends Construct { readonly url: string; @@ -23,12 +24,12 @@ export class PgStacApiLambda extends Construct { this.stacApiLambdaFunction = new lambda.Function(this, "lambda", { // defaults runtime: lambda.Runtime.PYTHON_3_11, - handler: "src.handler.handler", + handler: "handler.handler", memorySize: 8192, logRetention: aws_logs.RetentionDays.ONE_WEEK, timeout: Duration.seconds(30), - code: lambda.Code.fromDockerBuild(__dirname, { - file: "runtime/Dockerfile", + code: lambda.Code.fromDockerBuild(path.join(__dirname, '..'), { + file: "stac-api/runtime/Dockerfile", buildArgs: { PYTHON_VERSION: '3.11' }, }), vpc: props.vpc, diff --git a/lib/stac-api/runtime/Dockerfile b/lib/stac-api/runtime/Dockerfile index 380eb4e..1199d18 100644 --- a/lib/stac-api/runtime/Dockerfile +++ b/lib/stac-api/runtime/Dockerfile @@ -4,10 +4,11 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} WORKDIR /tmp RUN python -m pip install pip -U -COPY runtime/requirements.txt requirements.txt +COPY stac-api/runtime/requirements.txt requirements.txt RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic RUN mkdir -p /asset/src -COPY runtime/src/*.py /asset/src/ +COPY stac-api/runtime/src/*.py /asset/ +COPY utils/utils.py /asset/ CMD ["echo", "hello world"] diff --git a/lib/stac-api/runtime/src/app.py b/lib/stac-api/runtime/src/app.py deleted file mode 100644 index fb2d724..0000000 --- a/lib/stac-api/runtime/src/app.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -FastAPI application using PGStac. -""" - -from brotli_asgi import BrotliMiddleware -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import ORJSONResponse -from stac_fastapi.api.app import StacApi -from stac_fastapi.api.middleware import ProxyHeaderMiddleware -from stac_fastapi.pgstac.app import ( - application_extensions, - collections_get_request_model, - get_request_model, - items_get_request_model, - post_request_model, -) -from stac_fastapi.pgstac.core import CoreCrudClient -from stac_fastapi.pgstac.db import close_db_connection, connect_to_db -from starlette.middleware import Middleware - -from .config import ApiSettings - -settings = ApiSettings() - -api = StacApi( - app=FastAPI( - openapi_url=settings.openapi_url, - docs_url=settings.docs_url, - redoc_url=None, - root_path=settings.root_path, - title=settings.stac_fastapi_title, - version=settings.stac_fastapi_version, - description=settings.stac_fastapi_description, - ), - settings=settings, - extensions=application_extensions, - client=CoreCrudClient(pgstac_search_model=post_request_model), - response_class=ORJSONResponse, - items_get_request_model=items_get_request_model, - search_get_request_model=get_request_model, - search_post_request_model=post_request_model, - collections_get_request_model=collections_get_request_model, - middlewares=[ - Middleware(BrotliMiddleware), - Middleware(ProxyHeaderMiddleware), - ], -) -app = api.app - -# Set all CORS enabled origins -if settings.cors_origins: - app.add_middleware( - CORSMiddleware, - allow_origins=settings.cors_origins, - allow_credentials=True, - allow_methods=["GET", "POST", "OPTIONS"], - allow_headers=["*"], - ) - - -@app.on_event("startup") -async def startup_event(): - """Connect to database on startup.""" - print("Setting up DB connection...") - await connect_to_db(app) - print("DB connection setup.") - - -@app.on_event("shutdown") -async def shutdown_event(): - """Close database connection.""" - print("Closing up DB connection...") - await close_db_connection(app) - print("DB connection closed.") diff --git a/lib/stac-api/runtime/src/config.py b/lib/stac-api/runtime/src/config.py deleted file mode 100644 index e0198db..0000000 --- a/lib/stac-api/runtime/src/config.py +++ /dev/null @@ -1,64 +0,0 @@ -"""API settings. -Based on https://github.com/developmentseed/eoAPI/tree/master/src/eoapi/stac""" - -import base64 -import json -from typing import Any, Optional - -import boto3 - -# from stac_fastapi.pgstac.extensions import QueryExtension -from pydantic import model_validator -from stac_fastapi.pgstac.config import Settings -from stac_fastapi.pgstac.version import __version__ as stac_fastapi_pgstac_version - - -def get_secret_dict(secret_name: str): - """Retrieve secrets from AWS Secrets Manager - - Args: - secret_name (str): name of aws secrets manager secret containing database connection secrets - - Returns: - secrets (dict): decrypted secrets in dict - """ - - # Create a Secrets Manager client - session = boto3.session.Session() - client = session.client(service_name="secretsmanager") - - get_secret_value_response = client.get_secret_value(SecretId=secret_name) - - if "SecretString" in get_secret_value_response: - return json.loads(get_secret_value_response["SecretString"]) - else: - return json.loads(base64.b64decode(get_secret_value_response["SecretBinary"])) - - -class ApiSettings(Settings): - """API settings""" - - name: str = "stac-fastapi-pgstac" - version: str = stac_fastapi_pgstac_version - description: Optional[str] = None - cachecontrol: str = "public, max-age=3600" - debug: bool = False - - pgstac_secret_arn: Optional[str] - - @model_validator(mode="before") - def get_postgres_setting(cls, data: Any) -> Any: - if arn := data.get("pgstac_secret_arn"): - secret = get_secret_dict(arn) - data.update( - { - "postgres_host_reader": secret["host"], - "postgres_host_writer": secret["host"], - "postgres_dbname": secret["dbname"], - "postgres_user": secret["username"], - "postgres_pass": secret["password"], - "postgres_port": secret["port"], - } - ) - - return data diff --git a/lib/stac-api/runtime/src/handler.py b/lib/stac-api/runtime/src/handler.py index 335707e..f25fe7d 100644 --- a/lib/stac-api/runtime/src/handler.py +++ b/lib/stac-api/runtime/src/handler.py @@ -6,12 +6,10 @@ import os from mangum import Mangum +from utils import get_secret_dict -from .config import get_secret_dict +secret = get_secret_dict(secret_arn_env_var="PGSTAC_SECRET_ARN") -pgstac_secret_arn = os.environ["PGSTAC_SECRET_ARN"] - -secret = get_secret_dict(pgstac_secret_arn) os.environ.update( { "postgres_host_reader": secret["host"], @@ -23,7 +21,25 @@ } ) -from .app import app +from stac_fastapi.pgstac.app import app +from stac_fastapi.pgstac.db import close_db_connection, connect_to_db + + +@app.on_event("startup") +async def startup_event(): + """Connect to database on startup.""" + print("Setting up DB connection...") + await connect_to_db(app) + print("DB connection setup.") + + +@app.on_event("shutdown") +async def shutdown_event(): + """Close database connection.""" + print("Closing up DB connection...") + await close_db_connection(app) + print("DB connection closed.") + handler = Mangum(app, lifespan="off") diff --git a/lib/tipg-api/index.ts b/lib/tipg-api/index.ts index 26673c3..d17aca4 100644 --- a/lib/tipg-api/index.ts +++ b/lib/tipg-api/index.ts @@ -1,17 +1,18 @@ import { - Stack, - aws_ec2 as ec2, - aws_lambda as lambda, - aws_logs as logs, - aws_rds as rds, - aws_secretsmanager as secretsmanager, - CfnOutput, - Duration, - } from "aws-cdk-lib"; - import { IDomainName, HttpApi, ParameterMapping, MappingValue} from "@aws-cdk/aws-apigatewayv2-alpha"; - import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; - import { Construct } from "constructs"; - import { CustomLambdaFunctionProps } from "../utils"; + Stack, + aws_ec2 as ec2, + aws_lambda as lambda, + aws_logs as logs, + aws_rds as rds, + aws_secretsmanager as secretsmanager, + CfnOutput, + Duration, +} from "aws-cdk-lib"; +import { IDomainName, HttpApi, ParameterMapping, MappingValue} from "@aws-cdk/aws-apigatewayv2-alpha"; +import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; +import { Construct } from "constructs"; +import { CustomLambdaFunctionProps } from "../utils"; +import * as path from 'path'; export class TiPgApiLambda extends Construct { readonly url: string; @@ -27,9 +28,9 @@ import { memorySize: 1024, logRetention: logs.RetentionDays.ONE_WEEK, timeout: Duration.seconds(30), - code: lambda.Code.fromDockerBuild(__dirname, { - file: "runtime/Dockerfile", - buildArgs: { PYTHON_VERSION: '3.11' }, + code: lambda.Code.fromDockerBuild(path.join(__dirname, '..'), { + file: "tipg-api/runtime/Dockerfile", + buildArgs: { PYTHON_VERSION: '3.11' }, }), vpc: props.vpc, vpcSubnets: props.subnetSelection, diff --git a/lib/tipg-api/runtime/Dockerfile b/lib/tipg-api/runtime/Dockerfile index 954254b..fc29417 100644 --- a/lib/tipg-api/runtime/Dockerfile +++ b/lib/tipg-api/runtime/Dockerfile @@ -4,7 +4,7 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} WORKDIR /tmp RUN python -m pip install pip -U -COPY runtime/requirements.txt requirements.txt +COPY tipg-api/runtime/requirements.txt requirements.txt RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic # Reduce package size and remove useless files @@ -13,6 +13,7 @@ RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf -COPY runtime/src/*.py /asset/ +COPY tipg-api/runtime/src/*.py /asset/ +COPY utils/utils.py /asset/ CMD ["echo", "hello world"] diff --git a/lib/tipg-api/runtime/src/handler.py b/lib/tipg-api/runtime/src/handler.py index f077b59..1a7fed8 100644 --- a/lib/tipg-api/runtime/src/handler.py +++ b/lib/tipg-api/runtime/src/handler.py @@ -6,9 +6,18 @@ import os from mangum import Mangum -from utils import load_pgstac_secret +from utils import get_secret_dict -load_pgstac_secret(os.environ["PGSTAC_SECRET_ARN"]) # required for the below imports +secret = get_secret_dict(secret_arn_env_var="PGSTAC_SECRET_ARN") +os.environ.update( + { + "postgres_host": secret["host"], + "postgres_dbname": secret["dbname"], + "postgres_user": secret["username"], + "postgres_pass": secret["password"], + "postgres_port": str(secret["port"]), + } +) from tipg.collections import register_collection_catalog # noqa: E402 from tipg.database import connect_to_db # noqa: E402 diff --git a/lib/tipg-api/runtime/src/utils.py b/lib/tipg-api/runtime/src/utils.py deleted file mode 100644 index dadb49e..0000000 --- a/lib/tipg-api/runtime/src/utils.py +++ /dev/null @@ -1,42 +0,0 @@ -import base64 -import json -import os - -import boto3 - - -def load_pgstac_secret(secret_name: str): - """Retrieve secrets from AWS Secrets Manager - - Args: - secret_name (str): name of aws secrets manager secret containing database connection secrets - profile_name (str, optional): optional name of aws profile for use in debugger only - - Returns: - secrets (dict): decrypted secrets in dict - """ - - # Create a Secrets Manager client - session = boto3.session.Session() - client = session.client(service_name="secretsmanager") - - get_secret_value_response = client.get_secret_value(SecretId=secret_name) - - if "SecretString" in get_secret_value_response: - secret = json.loads(get_secret_value_response["SecretString"]) - else: - secret = json.loads(base64.b64decode(get_secret_value_response["SecretBinary"])) - - try: - os.environ.update( - { - "postgres_host": secret["host"], - "postgres_dbname": secret["dbname"], - "postgres_user": secret["username"], - "postgres_pass": secret["password"], - "postgres_port": str(secret["port"]), - } - ) - except Exception as ex: - print("Could not load the pgstac environment variables from the secret") - raise ex diff --git a/lib/titiler-pgstac-api/index.ts b/lib/titiler-pgstac-api/index.ts index 965ceb2..52153e5 100644 --- a/lib/titiler-pgstac-api/index.ts +++ b/lib/titiler-pgstac-api/index.ts @@ -1,19 +1,19 @@ import { - Stack, - aws_iam as iam, - aws_ec2 as ec2, - aws_rds as rds, - aws_lambda as lambda, - aws_secretsmanager as secretsmanager, - CfnOutput, - Duration, - aws_logs - } from "aws-cdk-lib"; - import { IDomainName, HttpApi, ParameterMapping, MappingValue} from "@aws-cdk/aws-apigatewayv2-alpha"; - import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; - import { Construct } from "constructs"; + Stack, + aws_iam as iam, + aws_ec2 as ec2, + aws_rds as rds, + aws_lambda as lambda, + aws_secretsmanager as secretsmanager, + CfnOutput, + Duration, + aws_logs +} from "aws-cdk-lib"; +import { IDomainName, HttpApi, ParameterMapping, MappingValue} from "@aws-cdk/aws-apigatewayv2-alpha"; +import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; +import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; - +import * as path from 'path'; // default settings that can be overridden by the user-provided environment. let defaultTitilerPgstacEnv :{ [key: string]: any } = { @@ -45,8 +45,8 @@ import { CustomLambdaFunctionProps } from "../utils"; memorySize: 3008, logRetention: aws_logs.RetentionDays.ONE_WEEK, timeout: Duration.seconds(30), - code: lambda.Code.fromDockerBuild(__dirname, { - file: "runtime/Dockerfile", + code: lambda.Code.fromDockerBuild(path.join(__dirname, '..'), { + file: "titiler-pgstac-api/runtime/Dockerfile", buildArgs: { PYTHON_VERSION: '3.11' } }), vpc: props.vpc, diff --git a/lib/titiler-pgstac-api/runtime/Dockerfile b/lib/titiler-pgstac-api/runtime/Dockerfile index e62a549..e58ac23 100644 --- a/lib/titiler-pgstac-api/runtime/Dockerfile +++ b/lib/titiler-pgstac-api/runtime/Dockerfile @@ -7,7 +7,7 @@ RUN python -m pip install pip -U # Install system dependencies to compile (numexpr) RUN yum install -y gcc-c++ -COPY runtime/requirements.txt requirements.txt +COPY titiler-pgstac-api/runtime/requirements.txt requirements.txt RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset # Remove system dependencies @@ -26,6 +26,7 @@ RUN cd /asset && \ -not -path "./numpy.libs/*" \ -exec strip --strip-unneeded {} \; -COPY runtime/src/*.py /asset/ +COPY titiler-pgstac-api/runtime/src/*.py /asset/ +COPY utils/utils.py /asset/ CMD ["echo", "hello world"] diff --git a/lib/titiler-pgstac-api/runtime/src/handler.py b/lib/titiler-pgstac-api/runtime/src/handler.py index 7234117..435802e 100644 --- a/lib/titiler-pgstac-api/runtime/src/handler.py +++ b/lib/titiler-pgstac-api/runtime/src/handler.py @@ -4,22 +4,11 @@ import asyncio import os -from functools import lru_cache from mangum import Mangum from utils import get_secret_dict - -@lru_cache(maxsize=1) -def get_db_credentials(): - """Cache the credentials to avoid repeated Secrets Manager calls.""" - pgstac_secret_arn = os.environ.get("PGSTAC_SECRET_ARN") - if not pgstac_secret_arn: - raise ValueError("PGSTAC_SECRET_ARN is not set!") - return get_secret_dict(pgstac_secret_arn) - - -secret = get_db_credentials() +secret = get_secret_dict(secret_arn_env_var="PGSTAC_SECRET_ARN") os.environ.update( { "postgres_host": secret["host"], diff --git a/lib/titiler-pgstac-api/runtime/src/utils.py b/lib/utils/utils.py similarity index 67% rename from lib/titiler-pgstac-api/runtime/src/utils.py rename to lib/utils/utils.py index 5279139..a145d32 100644 --- a/lib/titiler-pgstac-api/runtime/src/utils.py +++ b/lib/utils/utils.py @@ -1,25 +1,28 @@ import base64 import json +import os import boto3 -def get_secret_dict(secret_name: str): +def get_secret_dict(secret_arn_env_var: str): """Retrieve secrets from AWS Secrets Manager Args: - secret_name (str): name of aws secrets manager secret containing database connection secrets - profile_name (str, optional): optional name of aws profile for use in debugger only + secret_arn_env_var (str): environment variable that contains the secret ARN Returns: secrets (dict): decrypted secrets in dict """ + secret_arn = os.environ.get(secret_arn_env_var) + if not secret_arn: + raise ValueError(f"{secret_arn_env_var} is not set!") # Create a Secrets Manager client session = boto3.session.Session() client = session.client(service_name="secretsmanager") - get_secret_value_response = client.get_secret_value(SecretId=secret_name) + get_secret_value_response = client.get_secret_value(SecretId=secret_arn) if "SecretString" in get_secret_value_response: return json.loads(get_secret_value_response["SecretString"]) From 81d50a2daa424376d8d467ac41dbc0cf44651053 Mon Sep 17 00:00:00 2001 From: hrodmn Date: Wed, 5 Mar 2025 20:45:17 -0600 Subject: [PATCH 09/11] specify version of pypgstac installed in bootstrapper --- lib/database/bootstrapper_runtime/Dockerfile | 4 ++-- lib/database/index.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/database/bootstrapper_runtime/Dockerfile b/lib/database/bootstrapper_runtime/Dockerfile index 2184e6d..ee7779c 100644 --- a/lib/database/bootstrapper_runtime/Dockerfile +++ b/lib/database/bootstrapper_runtime/Dockerfile @@ -4,8 +4,8 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} RUN echo "PYTHON_VERSION: ${PYTHON_VERSION}" WORKDIR /tmp - -RUN pip install httpx psycopg[binary,pool] pypgstac -t /asset +ARG PGSTAC_VERSION +RUN pip install httpx psycopg[binary,pool] pypgstac==${PGSTAC_VERSION} -t /asset COPY bootstrapper_runtime/handler.py /asset/handler.py diff --git a/lib/database/index.ts b/lib/database/index.ts index 47e60d2..60ade67 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -69,7 +69,7 @@ export class PgStacDatabase extends Construct { parameterGroup, ...props, }); - + const pgstac_version = props.pgstacVersion || DEFAULT_PGSTAC_VERSION; const handler = new aws_lambda.Function(this, "lambda", { // defaults runtime: aws_lambda.Runtime.PYTHON_3_11, @@ -81,6 +81,7 @@ export class PgStacDatabase extends Construct { file: "bootstrapper_runtime/Dockerfile", buildArgs: { PYTHON_VERSION: "3.11", + PGSTAC_VERSION: pgstac_version, }, }), vpc: hasVpc(this.db) ? this.db.vpc : props.vpc, @@ -131,8 +132,7 @@ export class PgStacDatabase extends Construct { // if props.lambdaFunctionOptions doesn't have 'code' defined, update pgstac_version (needed for default runtime) if (!props.bootstrapperLambdaFunctionOptions?.code) { - customResourceProperties["pgstac_version"] = - props.pgstacVersion || DEFAULT_PGSTAC_VERSION; + customResourceProperties["pgstac_version"] = pgstac_version; } // add timestamp to properties to ensure the Lambda gets re-executed on each deploy From 671f44248c0b86d34fd46e062f43787d17fe8a3b Mon Sep 17 00:00:00 2001 From: hrodmn Date: Thu, 6 Mar 2025 13:28:16 -0600 Subject: [PATCH 10/11] upgrade to stac_fastapi.pgstac 5.0 to get new PostgresSettings --- lib/stac-api/runtime/requirements.txt | 2 +- lib/stac-api/runtime/src/handler.py | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/stac-api/runtime/requirements.txt b/lib/stac-api/runtime/requirements.txt index aa4c612..58067d1 100644 --- a/lib/stac-api/runtime/requirements.txt +++ b/lib/stac-api/runtime/requirements.txt @@ -1,2 +1,2 @@ -stac-fastapi-pgstac>=4.0.2,<4.1 +stac-fastapi-pgstac>=5.0,<5.1 starlette-cramjam>=0.4,<0.5 diff --git a/lib/stac-api/runtime/src/handler.py b/lib/stac-api/runtime/src/handler.py index f25fe7d..00ebe5d 100644 --- a/lib/stac-api/runtime/src/handler.py +++ b/lib/stac-api/runtime/src/handler.py @@ -6,30 +6,27 @@ import os from mangum import Mangum +from stac_fastapi.pgstac.app import app +from stac_fastapi.pgstac.config import PostgresSettings +from stac_fastapi.pgstac.db import close_db_connection, connect_to_db from utils import get_secret_dict secret = get_secret_dict(secret_arn_env_var="PGSTAC_SECRET_ARN") - -os.environ.update( - { - "postgres_host_reader": secret["host"], - "postgres_host_writer": secret["host"], - "postgres_dbname": secret["dbname"], - "postgres_user": secret["username"], - "postgres_pass": secret["password"], - "postgres_port": str(secret["port"]), - } +postgres_settings = PostgresSettings( + postgres_host_reader=secret["host"], + postgres_host_writer=secret["host"], + postgres_dbname=secret["dbname"], + postgres_user=secret["username"], + postgres_pass=secret["password"], + postgres_port=int(secret["port"]), ) -from stac_fastapi.pgstac.app import app -from stac_fastapi.pgstac.db import close_db_connection, connect_to_db - @app.on_event("startup") async def startup_event(): """Connect to database on startup.""" print("Setting up DB connection...") - await connect_to_db(app) + await connect_to_db(app, postgres_settings=postgres_settings) print("DB connection setup.") From d108d2a0ac52d16969486acd88a1b595fd89698a Mon Sep 17 00:00:00 2001 From: hrodmn Date: Mon, 10 Mar 2025 09:53:07 -0500 Subject: [PATCH 11/11] update tipg and titiler-pgstac requirement to allow patch upgrades --- lib/tipg-api/runtime/requirements.txt | 2 +- lib/titiler-pgstac-api/runtime/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tipg-api/runtime/requirements.txt b/lib/tipg-api/runtime/requirements.txt index c7094c7..a1ec08a 100644 --- a/lib/tipg-api/runtime/requirements.txt +++ b/lib/tipg-api/runtime/requirements.txt @@ -1 +1 @@ -tipg==0.9.0 +tipg>=1.0,<1.1 diff --git a/lib/titiler-pgstac-api/runtime/requirements.txt b/lib/titiler-pgstac-api/runtime/requirements.txt index e8b7026..9ae9c3c 100644 --- a/lib/titiler-pgstac-api/runtime/requirements.txt +++ b/lib/titiler-pgstac-api/runtime/requirements.txt @@ -1,2 +1,2 @@ -titiler.pgstac==1.7.0 +titiler.pgstac>=1.7,<1.8 psycopg[binary, pool]