From 50dcd1aa886863d124cff57bc8ce8608197c0647 Mon Sep 17 00:00:00 2001 From: ividito Date: Thu, 16 Oct 2025 20:59:26 -0700 Subject: [PATCH 1/8] build: speed up raster and stac lambda builds --- raster_api/infrastructure/construct.py | 11 ++-- raster_api/runtime/Dockerfile | 77 +++++++++++++++++++++----- stac_api/infrastructure/construct.py | 10 ++-- stac_api/runtime/Dockerfile | 61 +++++++++++++++++--- 4 files changed, 125 insertions(+), 34 deletions(-) diff --git a/raster_api/infrastructure/construct.py b/raster_api/infrastructure/construct.py index aa6682e2..77fe9884 100644 --- a/raster_api/infrastructure/construct.py +++ b/raster_api/infrastructure/construct.py @@ -13,6 +13,7 @@ aws_lambda, aws_logs, ) +from aws_cdk.aws_ecr_assets import Platform from constructs import Construct from .config import veda_raster_settings @@ -37,17 +38,15 @@ def __init__( # TODO config stack_name = Stack.of(self).stack_name - veda_raster_function = aws_lambda.Function( + veda_raster_function = aws_lambda.DockerImageFunction( self, "lambda", - runtime=aws_lambda.Runtime.PYTHON_3_12, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(code_dir), + code=aws_lambda.DockerImageCode.from_image_asset( + directory=os.path.abspath(code_dir), file="raster_api/runtime/Dockerfile", - platform="linux/amd64", + platform=Platform.LINUX_AMD64, ), vpc=vpc, - handler="handler.handler", memory_size=veda_raster_settings.memory, timeout=Duration.seconds(veda_raster_settings.timeout), log_retention=aws_logs.RetentionDays.ONE_WEEK, diff --git a/raster_api/runtime/Dockerfile b/raster_api/runtime/Dockerfile index 514242a8..a197021d 100644 --- a/raster_api/runtime/Dockerfile +++ b/raster_api/runtime/Dockerfile @@ -1,25 +1,74 @@ -FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.12:latest +ARG PYTHON_VERSION=3.12 -RUN dnf install -y gcc-c++ +# Stage 1: application and dependencies +FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +RUN dnf install -y gcc-c++ && dnf clean all WORKDIR /tmp COPY raster_api/runtime /tmp/raster -RUN pip install mangum /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic + +# we could have a `uv export` here, if we converted the project format +RUN uv pip install mangum /tmp/raster["psycopg-binary"] --target /deps --no-binary pydantic + +RUN </dev/null || true +EOF + +# Stage 2: Final runtime stage - minimal Lambda image optimized for container runtime +FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/ +COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/ +COPY infrastructure/aws/lambda/handler.py ${LAMBDA_RUNTIME_DIR}/ +COPY raster_api/runtime/handler.py ${LAMBDA_RUNTIME_DIR}/ + -COPY raster_api/runtime/handler.py /asset/handler.py -RUN dnf remove -y gcc-c++ +RUN <=0.2.1" /tmp/stac -t /asset --no-binary pydantic +# we could have a `uv export` here, if we converted the project format +RUN uv pip install "mangum" "plpygis>=0.2.1" /tmp/stac["psycopg-binary"] --target /deps --no-binary pydantic RUN rm -rf /tmp/stac -# Reduce package size and remove useless files -RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; -RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf -RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f -RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf +WORKDIR /deps +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN </dev/null || true +EOF + +# Stage 2: Final runtime stage - minimal Lambda image optimized for container runtime +FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/ +COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/ +COPY infrastructure/aws/lambda/handler.py ${LAMBDA_RUNTIME_DIR}/ +COPY stac_api/runtime/handler.py ${LAMBDA_RUNTIME_DIR}/ + + +RUN < Date: Wed, 22 Oct 2025 10:36:07 -0700 Subject: [PATCH 2/8] update dockerfile --- app.py | 32 +++++++------- database/infrastructure/construct.py | 66 ++++++++++++++-------------- raster_api/runtime/Dockerfile | 1 - stac_api/runtime/Dockerfile | 1 - 4 files changed, 49 insertions(+), 51 deletions(-) diff --git a/app.py b/app.py index d235b8a1..48cef4aa 100644 --- a/app.py +++ b/app.py @@ -119,22 +119,22 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: git_sha=git_sha, ) -ingest_api = ingest_api_construct( - veda_stack, - "ingest-api", - config=ingestor_config, - db_secret=database.pgstac.secret, - db_vpc=vpc.vpc, -) - -ingestor = ingestor_construct( - veda_stack, - "IngestorConstruct", - config=ingestor_config, - table=ingest_api.table, - db_secret=database.pgstac.secret, - db_vpc=vpc.vpc, -) +# ingest_api = ingest_api_construct( +# veda_stack, +# "ingest-api", +# config=ingestor_config, +# db_secret=database.pgstac.secret, +# db_vpc=vpc.vpc, +# ) +# +# ingestor = ingestor_construct( +# veda_stack, +# "IngestorConstruct", +# config=ingestor_config, +# table=ingest_api.table, +# db_secret=database.pgstac.secret, +# db_vpc=vpc.vpc, +# ) for key, value in { "Project": veda_app_settings.app_name, diff --git a/database/infrastructure/construct.py b/database/infrastructure/construct.py index ff38f615..3518e208 100644 --- a/database/infrastructure/construct.py +++ b/database/infrastructure/construct.py @@ -44,20 +44,20 @@ def __init__( pgstac_version = veda_db_settings.pgstac_version veda_schema_version = veda_db_settings.schema_version - handler = aws_lambda.Function( - self, - "lambda", - handler="handler.handler", - runtime=aws_lambda.Runtime.PYTHON_3_12, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath("./"), - file="database/runtime/Dockerfile", - build_args={"PGSTAC_VERSION": pgstac_version}, - ), - timeout=Duration.minutes(5), - vpc=database.vpc, - log_retention=aws_logs.RetentionDays.ONE_WEEK, - ) + # handler = aws_lambda.Function( + # self, + # "lambda", + # handler="handler.handler", + # runtime=aws_lambda.Runtime.PYTHON_3_12, + # code=aws_lambda.Code.from_docker_build( + # path=os.path.abspath("./"), + # file="database/runtime/Dockerfile", + # build_args={"PGSTAC_VERSION": pgstac_version}, + # ), + # timeout=Duration.minutes(5), + # vpc=database.vpc, + # log_retention=aws_logs.RetentionDays.ONE_WEEK, + # ) self.secret = aws_secretsmanager.Secret( self, @@ -81,28 +81,28 @@ def __init__( # Allow lambda to... # read new user secret - self.secret.grant_read(handler) - # read database secret - database.secret.grant_read(handler) - # connect to database - database.connections.allow_from(handler, port_range=aws_ec2.Port.tcp(5432)) + # self.secret.grant_read(handler) + ## read database secret + # database.secret.grant_read(handler) + ## connect to database + # database.connections.allow_from(handler, port_range=aws_ec2.Port.tcp(5432)) self.connections = database.connections - CustomResource( - scope=scope, - id="bootstrapper", - service_token=handler.function_arn, - properties={ - # By setting pgstac_version in the properties assures - # that Create/Update events will be passed to the service token - "pgstac_version": pgstac_version, - "conn_secret_arn": database.secret.secret_arn, - "new_user_secret_arn": self.secret.secret_arn, - "veda_schema_version": veda_schema_version, - }, - removal_policy=RemovalPolicy.RETAIN, # This retains the custom resource (which doesn't really exist), not the database - ) + # CustomResource( + # scope=scope, + # id="bootstrapper", + # service_token=handler.function_arn, + # properties={ + # # By setting pgstac_version in the properties assures + # # that Create/Update events will be passed to the service token + # "pgstac_version": pgstac_version, + # "conn_secret_arn": database.secret.secret_arn, + # "new_user_secret_arn": self.secret.secret_arn, + # "veda_schema_version": veda_schema_version, + # }, + # removal_policy=RemovalPolicy.RETAIN, # This retains the custom resource (which doesn't really exist), not the database + # ) # https://github.com/developmentseed/eoAPI/blob/master/deployment/cdk/app.py diff --git a/raster_api/runtime/Dockerfile b/raster_api/runtime/Dockerfile index a197021d..191b9922 100644 --- a/raster_api/runtime/Dockerfile +++ b/raster_api/runtime/Dockerfile @@ -56,7 +56,6 @@ ENV PYTHONUNBUFFERED=1 \ COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/ COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/ -COPY infrastructure/aws/lambda/handler.py ${LAMBDA_RUNTIME_DIR}/ COPY raster_api/runtime/handler.py ${LAMBDA_RUNTIME_DIR}/ diff --git a/stac_api/runtime/Dockerfile b/stac_api/runtime/Dockerfile index 9cf15e17..3e102081 100644 --- a/stac_api/runtime/Dockerfile +++ b/stac_api/runtime/Dockerfile @@ -43,7 +43,6 @@ ENV PYTHONUNBUFFERED=1 \ COPY --from=builder /deps ${LAMBDA_RUNTIME_DIR}/ COPY --from=builder /usr/lib64/libexpat.so.1 ${LAMBDA_RUNTIME_DIR}/ -COPY infrastructure/aws/lambda/handler.py ${LAMBDA_RUNTIME_DIR}/ COPY stac_api/runtime/handler.py ${LAMBDA_RUNTIME_DIR}/ From 3529f36d4c0b258a775c114e3e8819b819388aa0 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 22 Oct 2025 10:37:57 -0700 Subject: [PATCH 3/8] Re-add debug lines --- app.py | 32 +++++++------- database/infrastructure/construct.py | 66 ++++++++++++++-------------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/app.py b/app.py index 48cef4aa..56af999e 100644 --- a/app.py +++ b/app.py @@ -119,22 +119,22 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: git_sha=git_sha, ) -# ingest_api = ingest_api_construct( -# veda_stack, -# "ingest-api", -# config=ingestor_config, -# db_secret=database.pgstac.secret, -# db_vpc=vpc.vpc, -# ) -# -# ingestor = ingestor_construct( -# veda_stack, -# "IngestorConstruct", -# config=ingestor_config, -# table=ingest_api.table, -# db_secret=database.pgstac.secret, -# db_vpc=vpc.vpc, -# ) +ingest_api = ingest_api_construct( + veda_stack, + "ingest-api", + config=ingestor_config, + db_secret=database.pgstac.secret, + db_vpc=vpc.vpc, +) + +ingestor = ingestor_construct( + veda_stack, + "IngestorConstruct", + config=ingestor_config, + table=ingest_api.table, + db_secret=database.pgstac.secret, + db_vpc=vpc.vpc, +) for key, value in { "Project": veda_app_settings.app_name, diff --git a/database/infrastructure/construct.py b/database/infrastructure/construct.py index 3518e208..8f912495 100644 --- a/database/infrastructure/construct.py +++ b/database/infrastructure/construct.py @@ -44,20 +44,20 @@ def __init__( pgstac_version = veda_db_settings.pgstac_version veda_schema_version = veda_db_settings.schema_version - # handler = aws_lambda.Function( - # self, - # "lambda", - # handler="handler.handler", - # runtime=aws_lambda.Runtime.PYTHON_3_12, - # code=aws_lambda.Code.from_docker_build( - # path=os.path.abspath("./"), - # file="database/runtime/Dockerfile", - # build_args={"PGSTAC_VERSION": pgstac_version}, - # ), - # timeout=Duration.minutes(5), - # vpc=database.vpc, - # log_retention=aws_logs.RetentionDays.ONE_WEEK, - # ) + handler = aws_lambda.Function( + self, + "lambda", + handler="handler.handler", + runtime=aws_lambda.Runtime.PYTHON_3_12, + code=aws_lambda.Code.from_docker_build( + path=os.path.abspath("./"), + file="database/runtime/Dockerfile", + build_args={"PGSTAC_VERSION": pgstac_version}, + ), + timeout=Duration.minutes(5), + vpc=database.vpc, + log_retention=aws_logs.RetentionDays.ONE_WEEK, + ) self.secret = aws_secretsmanager.Secret( self, @@ -81,28 +81,28 @@ def __init__( # Allow lambda to... # read new user secret - # self.secret.grant_read(handler) - ## read database secret - # database.secret.grant_read(handler) - ## connect to database - # database.connections.allow_from(handler, port_range=aws_ec2.Port.tcp(5432)) + self.secret.grant_read(handler) + # read database secret + database.secret.grant_read(handler) + # connect to database + database.connections.allow_from(handler, port_range=aws_ec2.Port.tcp(5432)) self.connections = database.connections - # CustomResource( - # scope=scope, - # id="bootstrapper", - # service_token=handler.function_arn, - # properties={ - # # By setting pgstac_version in the properties assures - # # that Create/Update events will be passed to the service token - # "pgstac_version": pgstac_version, - # "conn_secret_arn": database.secret.secret_arn, - # "new_user_secret_arn": self.secret.secret_arn, - # "veda_schema_version": veda_schema_version, - # }, - # removal_policy=RemovalPolicy.RETAIN, # This retains the custom resource (which doesn't really exist), not the database - # ) + CustomResource( + scope=scope, + id="bootstrapper", + service_token=handler.function_arn, + properties={ + # By setting pgstac_version in the properties assures + # that Create/Update events will be passed to the service token + "pgstac_version": pgstac_version, + "conn_secret_arn": database.secret.secret_arn, + "new_user_secret_arn": self.secret.secret_arn, + "veda_schema_version": veda_schema_version, + }, + removal_policy=RemovalPolicy.RETAIN, # This retains the custom resource (which doesn't really exist), not the database + ) # https://github.com/developmentseed/eoAPI/blob/master/deployment/cdk/app.py From af5aea44a3102d2898ab664465f07a1439fbc613 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 22 Oct 2025 11:48:32 -0700 Subject: [PATCH 4/8] Fix indent --- database/infrastructure/construct.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/database/infrastructure/construct.py b/database/infrastructure/construct.py index 8f912495..97b16fb6 100644 --- a/database/infrastructure/construct.py +++ b/database/infrastructure/construct.py @@ -81,11 +81,11 @@ def __init__( # Allow lambda to... # read new user secret - self.secret.grant_read(handler) + self.secret.grant_read(handler) # read database secret - database.secret.grant_read(handler) + database.secret.grant_read(handler) # connect to database - database.connections.allow_from(handler, port_range=aws_ec2.Port.tcp(5432)) + database.connections.allow_from(handler, port_range=aws_ec2.Port.tcp(5432)) self.connections = database.connections From a0f563e5c93315990aa8a4610d9a8b687d5d704e Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 22 Oct 2025 11:58:11 -0700 Subject: [PATCH 5/8] Try removing import check from dockerfiles --- raster_api/runtime/Dockerfile | 2 -- stac_api/runtime/Dockerfile | 2 -- 2 files changed, 4 deletions(-) diff --git a/raster_api/runtime/Dockerfile b/raster_api/runtime/Dockerfile index 191b9922..81b9080b 100644 --- a/raster_api/runtime/Dockerfile +++ b/raster_api/runtime/Dockerfile @@ -68,6 +68,4 @@ python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler. mkdir -p /tmp/.cache && chmod 777 /tmp/.cache EOF -RUN python -c "from handler import handler; print('All Good')" - CMD ["handler.handler"] \ No newline at end of file diff --git a/stac_api/runtime/Dockerfile b/stac_api/runtime/Dockerfile index 3e102081..3be2782f 100644 --- a/stac_api/runtime/Dockerfile +++ b/stac_api/runtime/Dockerfile @@ -55,6 +55,4 @@ python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler. mkdir -p /tmp/.cache && chmod 777 /tmp/.cache EOF -RUN python -c "from handler import handler; print('All Good')" - CMD ["handler.handler"] From 0ad8528b34d14f72bfdd79b2cf2da5fc52879b1e Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 22 Oct 2025 14:10:48 -0700 Subject: [PATCH 6/8] try lambda_handler --- raster_api/runtime/Dockerfile | 2 +- raster_api/runtime/handler.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/raster_api/runtime/Dockerfile b/raster_api/runtime/Dockerfile index 81b9080b..ada1216d 100644 --- a/raster_api/runtime/Dockerfile +++ b/raster_api/runtime/Dockerfile @@ -68,4 +68,4 @@ python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler. mkdir -p /tmp/.cache && chmod 777 /tmp/.cache EOF -CMD ["handler.handler"] \ No newline at end of file +CMD ["handler.lambda_handler"] \ No newline at end of file diff --git a/raster_api/runtime/handler.py b/raster_api/runtime/handler.py index a357f53d..2c490b18 100644 --- a/raster_api/runtime/handler.py +++ b/raster_api/runtime/handler.py @@ -3,6 +3,7 @@ import asyncio import logging import os +from typing import Any, Dict from mangum import Mangum from src.app import app @@ -36,3 +37,7 @@ async def startup_event() -> None: handler = logger.inject_lambda_context(handler, clear_state=True) # Add metrics last to properly flush metrics. handler = metrics.log_metrics(handler, capture_cold_start_metric=True) + +def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]: + """Lambda handler with container-specific optimizations and OTEL tracing.""" + return handler(event, context) \ No newline at end of file From 35bf56fa1629a4b8e7e1fbbbf0826ba378ceeb86 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 22 Oct 2025 14:23:15 -0700 Subject: [PATCH 7/8] fix --- stac_api/runtime/Dockerfile | 2 +- stac_api/runtime/handler.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/stac_api/runtime/Dockerfile b/stac_api/runtime/Dockerfile index 3be2782f..7a027dbd 100644 --- a/stac_api/runtime/Dockerfile +++ b/stac_api/runtime/Dockerfile @@ -55,4 +55,4 @@ python -c "import py_compile; py_compile.compile('${LAMBDA_RUNTIME_DIR}/handler. mkdir -p /tmp/.cache && chmod 777 /tmp/.cache EOF -CMD ["handler.handler"] +CMD ["handler.lambda_handler"] \ No newline at end of file diff --git a/stac_api/runtime/handler.py b/stac_api/runtime/handler.py index 176be13b..b383d322 100644 --- a/stac_api/runtime/handler.py +++ b/stac_api/runtime/handler.py @@ -5,6 +5,7 @@ from mangum import Mangum from src.app import app from src.monitoring import logger, metrics, tracer +from typing import Any, Dict logging.getLogger("mangum.lifespan").setLevel(logging.ERROR) logging.getLogger("mangum.http").setLevel(logging.ERROR) @@ -18,3 +19,7 @@ handler = logger.inject_lambda_context(handler, clear_state=True) # Add metrics last to properly flush metrics. handler = metrics.log_metrics(handler, capture_cold_start_metric=True) + +def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]: + """Lambda handler with container-specific optimizations and OTEL tracing.""" + return handler(event, context) \ No newline at end of file From d80481112b9a23fde3a5f816019ce131e73af885 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 22 Oct 2025 14:38:19 -0700 Subject: [PATCH 8/8] Try fixing stac mangum import --- stac_api/runtime/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_api/runtime/Dockerfile b/stac_api/runtime/Dockerfile index 7a027dbd..26dc46e6 100644 --- a/stac_api/runtime/Dockerfile +++ b/stac_api/runtime/Dockerfile @@ -11,7 +11,7 @@ WORKDIR /tmp COPY stac_api/runtime /tmp/stac # we could have a `uv export` here, if we converted the project format -RUN uv pip install "mangum" "plpygis>=0.2.1" /tmp/stac["psycopg-binary"] --target /deps --no-binary pydantic +RUN uv pip install mangum "plpygis>=0.2.1" /tmp/stac["psycopg-binary"] --target /deps --no-binary pydantic RUN rm -rf /tmp/stac WORKDIR /deps