feat: upgrade to titiler>=0.23.1,<0.24 and Python 3.12 (#71)

hrodmn · web-flow · commit 39f7067cd71d · 2025-09-06T07:20:16.000-05:00
* deps: upgrade to titiler&gt;=0.23.1,&lt;0.24

* feat: upgrade to python 3.12 runtime for Lambda

* use uv to pip install in the Lambda dockerfile

* add more logging statements
diff --git a/.gitignore b/.gitignore
@@ -70,9 +70,6 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
-# pyenv
-.python-version
-
 # celery beat schedule file
 celerybeat-schedule
 
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.12
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+- Upgrade to titiler v0.23. and add more logging statements to help track performance ([#71](https://github.com/developmentseed/titiler-cmr/pull/71))
+
 ## [v0.1.5]
 
 ### Fixed
diff --git a/Dockerfile b/Dockerfile
@@ -9,7 +9,7 @@ COPY titiler/ titiler/
 COPY README.md README.md
 COPY LICENSE LICENSE
 
-RUN apt-get update && apt-get -y --no-install-recommends install libexpat1 && uv sync --frozen --extra uvicorn
+RUN apt-get update && apt-get -y --no-install-recommends install libexpat1 && uv sync --no-dev --frozen --extra uvicorn
 
 ARG EARTHDATA_USERNAME
 ARG EARTHDATA_PASSWORD
diff --git a/docs/examples/rasterio_backend_example.ipynb b/docs/examples/rasterio_backend_example.ipynb
@@ -41,7 +41,10 @@
    "outputs": [],
    "source": [
     "# titiler_endpoint = \"http://localhost:8081\"  # docker network endpoint\n",
-    "titiler_endpoint = \"https://staging.openveda.cloud/api/titiler-cmr\"  # deployed endpoint"
+    "titiler_endpoint = (\n",
+    "    \"https://staging.openveda.cloud/api/titiler-cmr\"  # VEDA staging endpoint\n",
+    ")\n",
+    "# titiler_endpoint = \"https://v4jec6i5c0.execute-api.us-west-2.amazonaws.com\"  # dev endpoint"
    ]
   },
   {
@@ -364,7 +367,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.12.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/examples/time_series_example.ipynb b/docs/examples/time_series_example.ipynb
@@ -37,8 +37,11 @@
     "# if running titiler-cmr in the docker network\n",
     "# titiler_endpoint = \"http://localhost:8081\"\n",
     "\n",
-    "# titiler-cmr-staging deployment\n",
-    "titiler_endpoint = \"https://staging.openveda.cloud/api/titiler-cmr\"\n",
+    "# titiler_endpoint = \"http://localhost:8081\"  # docker network endpoint\n",
+    "titiler_endpoint = (\n",
+    "    \"https://staging.openveda.cloud/api/titiler-cmr\"  # VEDA staging endpoint\n",
+    ")\n",
+    "# titiler_endpoint = \"https://v4jec6i5c0.execute-api.us-west-2.amazonaws.com\"  # dev endpoint\n",
     "\n",
     "IFrame(f\"{titiler_endpoint}/api.html#Timeseries\", 900, 500)"
    ]
@@ -519,7 +522,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.12.3"
   }
  },
  "nbformat": 4,
diff --git a/docs/examples/xarray_backend_example.ipynb b/docs/examples/xarray_backend_example.ipynb
@@ -34,7 +34,10 @@
     "from folium import Map, TileLayer\n",
     "\n",
     "# titiler_endpoint = \"http://localhost:8081\"  # docker network endpoint\n",
-    "titiler_endpoint = \"https://staging.openveda.cloud/api/titiler-cmr\"  # deployed endpoint"
+    "titiler_endpoint = (\n",
+    "    \"https://staging.openveda.cloud/api/titiler-cmr\"  # VEDA staging endpoint\n",
+    ")\n",
+    "# titiler_endpoint = \"https://v4jec6i5c0.execute-api.us-west-2.amazonaws.com\"  # dev endpoint"
    ]
   },
   {
@@ -284,7 +287,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.12.3"
   }
  },
  "nbformat": 4,
diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py
@@ -12,12 +12,14 @@
 from aws_cdk import aws_sns as sns
 from aws_cdk import aws_sns_subscriptions as subscriptions
 from aws_cdk.aws_apigatewayv2_integrations import HttpLambdaIntegration
-from config import StackSettings, AppSettings
+from config import AppSettings, StackSettings
 from constructs import Construct
 
 stack_settings, app_settings = StackSettings(), AppSettings()
 
 DEFAULT_ENV = {
+    "AWS_LAMBDA_LOG_FORMAT": "JSON",
+    "AWS_LAMBDA_LOG_LEVEL": "INFO",
     "GDAL_CACHEMAX": "200",  # 200 mb
     "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
     "GDAL_INGESTED_BYTES_AT_OPEN": "32768",  # get more bytes when opening the files.
@@ -39,7 +41,7 @@ def __init__(
         id: str,
         memory: int = 1024,
         timeout: int = 30,
-        runtime: aws_lambda.Runtime = aws_lambda.Runtime.PYTHON_3_10,
+        runtime: aws_lambda.Runtime = aws_lambda.Runtime.PYTHON_3_12,
         concurrent: Optional[int] = None,
         permissions: Optional[List[iam.PolicyStatement]] = None,
         role_arn: Optional[str] = None,
diff --git a/infrastructure/aws/lambda/Dockerfile b/infrastructure/aws/lambda/Dockerfile
@@ -1,34 +1,26 @@
-ARG PYTHON_VERSION=3.10
+ARG PYTHON_VERSION=3.12
 
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 WORKDIR /tmp
 
-COPY pyproject.toml pyproject.toml
-COPY LICENSE LICENSE
-COPY README.md README.md
-COPY titiler/ titiler/
+COPY uv.lock .python-version pyproject.toml LICENSE README.md ./
+COPY titiler/ ./titiler/
 
-# Install dependencies
+RUN dnf install -y gcc-c++  
+RUN uv export --locked --no-editable --no-dev --extra lambda --format requirements.txt -o requirements.txt && \
+  uv pip install --compile-bytecode --no-binary pydantic --target /asset -r requirements.txt 
 
-# Install system dependencies to compile (numexpr)
-RUN yum install -y gcc-c++
-# HACK: aiobotocore has a tight botocore dependency
-# https://github.com/aio-libs/aiobotocore/issues/862
-# and becaise we NEED to remove both boto3 and botocore to save space for the package
-# we have to force using old package version that seems `almost` compatible with Lambda env botocore
-# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
-RUN python -m pip install --upgrade pip
-RUN python -m pip install . "mangum>=0.10.0" "botocore==1.34.145" "aiobotocore==2.13.3" -t /asset --no-binary pydantic
+# copy libexpat.so.1 into /asset which is included in LD_LIBRARY_PATH
+RUN cp /usr/lib64/libexpat.so.1 /asset/
 
 # Reduce package size and remove useless files
 RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done;
 RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
 RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f
 RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf
 RUN rm -rdf /asset/numpy/doc/ /asset/bin /asset/geos_license /asset/Misc
-RUN rm -rdf /asset/boto3*
-RUN rm -rdf /asset/botocore*
 
 # Strip debug symbols from compiled C/C++ code (except for numpy.libs!)
 RUN cd /asset && \
diff --git a/pyproject.toml b/pyproject.toml
@@ -2,7 +2,7 @@
 name = "titiler.cmr"
 description = "Connect Common Metadata Repository (CMR) and TiTiler."
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 license = {file = "LICENSE"}
 authors = [
     {name = "Aimee Barciauskas", email = "aimee@developmentseed.com"},
@@ -16,25 +16,25 @@ classifiers = [
     "Intended Audience :: Information Technology",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering :: GIS",
 ]
 dependencies = [
-    "titiler.core>=0.22.3,<0.23",
-    "titiler.mosaic>=0.22.3,<0.23",
-    "titiler.xarray>=0.22.3,<0.23",
+    "titiler.core>=0.23.1,<0.24",
+    "titiler.mosaic>=0.23.1,<0.24",
+    "titiler.xarray>=0.23.1,<0.24",
+    "aiobotocore>=2.20",
+    "boto3>=1.34.145",
     "cftime~=1.6.4",
     "earthaccess~=0.11.0",
-    "fsspec~=2024.9.0",
     "h5netcdf~=1.1.0",
     "orjson~=3.10.7",
     "pydantic-settings~=2.0",
     "pydantic>=2.4,<3.0",
     "rio_tiler[s3]>=7.2.0,<8.0",
     "rioxarray~=0.13.4",
-    "s3fs~=2024.9.0",
+    "s3fs~=2025.9.0",
     "xarray~=2024.9.0",
     "geojson-pydantic>=2.0.0,<3.0",
     "python-dateutil>=2.9.0.post0",
@@ -50,6 +50,8 @@ uvicorn = [
     "uvicorn",
 ]
 
+lambda = ["mangum==0.19.0"]
+
 [dependency-groups]
 dev = [
     "folium>=0.17.0",
@@ -74,10 +76,11 @@ dev = [
     "ipywidgets>=8.1.5",
     "pytest-benchmark>=5.1.0",
     "seaborn>=0.13.2",
+    "uvicorn>=0.32.1",
 ]
 deployment = [
     "aws-cdk-lib~=2.177.0",
-    "boto3>=1.35.36",
+    "boto3>=1.34.145",
     "constructs>=10.4.2",
     "pydantic-settings~=2.0",
     "python-dotenv>=1.0.1",
diff --git a/titiler/cmr/backend.py b/titiler/cmr/backend.py
@@ -25,6 +25,7 @@
 from rio_tiler.mosaic import mosaic_reader
 from rio_tiler.types import BBox
 
+from titiler.cmr.logger import logger
 from titiler.cmr.settings import AuthSettings, CacheSettings, RetrySettings
 from titiler.cmr.utils import retry
 
@@ -234,6 +235,7 @@ def tile(
         **kwargs: Any,
     ) -> Tuple[ImageData, List[str]]:
         """Get Tile from multiple observation."""
+        logger.info("searching for assets")
         mosaic_assets = self.assets_for_tile(
             tile_x,
             tile_y,
@@ -242,6 +244,7 @@ def tile(
             access=s3_auth_config.access,
             bands_regex=bands_regex,
         )
+        logger.info(f"found {len(mosaic_assets)} assets")
 
         if not mosaic_assets:
             raise NoAssetFoundError(
@@ -295,6 +298,7 @@ def _reader(asset: Asset, x: int, y: int, z: int, **kwargs: Any) -> ImageData:
             ) as src_dst:
                 return src_dst.tile(x, y, z, **kwargs)
 
+        logger.info("reading assets")
         return mosaic_reader(mosaic_assets, _reader, tile_x, tile_y, tile_z, **kwargs)
 
     def point(
diff --git a/titiler/cmr/factory.py b/titiler/cmr/factory.py
@@ -33,6 +33,7 @@
     cmr_query,
 )
 from titiler.cmr.enums import MediaType
+from titiler.cmr.logger import logger
 from titiler.cmr.reader import MultiFilesBandsReader, xarray_open_dataset
 from titiler.core.algorithm import algorithms as available_algorithms
 from titiler.core.dependencies import (
@@ -48,8 +49,8 @@
 from titiler.core.models.responses import MultiBaseStatisticsGeoJSON
 from titiler.core.resources.enums import ImageType, OptionalHeader
 from titiler.core.resources.responses import GeoJSONResponse
+from titiler.xarray.dependencies import CompatXarrayParams, XarrayIOParams
 from titiler.xarray.io import Reader as XarrayReader
-from titiler.xarray.dependencies import XarrayIOParams, CompatXarrayParams
 
 jinja2_env = jinja2.Environment(
     loader=jinja2.ChoiceLoader(
@@ -526,6 +527,7 @@ def tiles_endpoint(
                 reader_params=reader_params,
             )
 
+            logger.info(f"opening data with reader: {reader}")
             with CMRBackend(
                 tms=tms,
                 reader=reader,
@@ -542,6 +544,7 @@ def tiles_endpoint(
                     reproject_method=reproject_method,
                     **read_options,
                 )
+                logger.info("successfully generated tile image")
                 dst_colormap = getattr(src_dst, "colormap", None)
 
             if post_process:
diff --git a/titiler/cmr/main.py b/titiler/cmr/main.py
@@ -1,5 +1,6 @@
 """TiTiler+cmr FastAPI application."""
 
+import json
 import logging
 import os
 from contextlib import asynccontextmanager
@@ -33,13 +34,57 @@
 auth_config = AuthSettings()
 
 
+class JSONFormatter(logging.Formatter):
+    """JSON log formatter similar to AWS Lambda."""
+
+    def format(self, record):
+        """format log record in json"""
+        log_entry = {
+            "timestamp": self.formatTime(record, "%Y-%m-%dT%H:%M:%S.%fZ"),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "filename": record.filename,
+            "lineno": record.lineno,
+        }
+
+        # Add any extra fields passed via the extra parameter
+        if hasattr(record, "__dict__"):
+            for key, value in record.__dict__.items():
+                if key not in log_entry and not key.startswith("_"):
+                    # Only add if it's not a standard logging attribute
+                    if not hasattr(logging.LogRecord("", 0, "", 0, "", (), None), key):
+                        log_entry[key] = value
+
+        if record.exc_info:
+            log_entry["exception"] = self.formatException(record.exc_info)
+
+        return json.dumps(log_entry)
+
+
 log_level = os.getenv("LOG_LEVEL", "INFO")
+
+# Configure logging format based on AWS_LAMBDA_LOG_FORMAT
+# Only use custom JSON formatter locally - AWS Lambda handles JSON formatting automatically
+formatter: JSONFormatter | logging.Formatter
+if "AWS_EXECUTION_ENV" not in os.environ:
+    formatter = JSONFormatter()
+else:
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s"
+    )
+
+handler = logging.StreamHandler()
+handler.setFormatter(formatter)
+
 logging.basicConfig(
     level=getattr(logging, log_level),
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    handlers=[logging.StreamHandler()],
+    handlers=[handler],
 )
 
+for module in ["botocore", "aiobotocore", "earthaccess"]:
+    logging.getLogger(module).setLevel("WARN")
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
@@ -152,8 +197,7 @@ async def lifespan(app: FastAPI):
 
 app.add_middleware(CacheControlMiddleware, cachecontrol=settings.cachecontrol)
 
-if settings.debug:
-    app.add_middleware(LoggerMiddleware, headers=True, querystrings=True)
+app.add_middleware(LoggerMiddleware)
 
 ###############################################################################
 # application endpoints
diff --git a/uv.lock b/uv.lock
