feat(release)!: dependency upgrades and new colormap (#461)

### Changed

- Python 3.9 to 3.11 upgrade
- Pydantic>2 and associated package upgrades

### Breaking
- Pydantic upgrades require migrating pgstac version to 0.8.5 

### Added

- New custom surface temperature colormap (`st_cmap`) added to raster
api

Author: anayeaye

.github/actions/cdk-deploy/action.yml

@@ -21,7 +21,7 @@ runs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.11'
         cache: 'pip'
         cache-dependency-path: |
           ${{ inputs.dir }}/setup.py

.github/workflows/pr.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - uses: actions/cache@v4
         with:
@@ -48,7 +48,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - uses: actions/cache@v4
         with:
@@ -102,7 +102,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - name: Setup Node
         uses: actions/setup-node@v4

common/auth/setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import find_packages, setup
 
-inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic<2"]
+inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic"]
 
 setup(
     name="veda_auth",

config.py

@@ -2,9 +2,10 @@
 from getpass import getuser
 from typing import List, Optional
 
-from pydantic import BaseSettings, Field, constr
+from pydantic import Field, constr
+from pydantic_settings import BaseSettings
 
-AwsSubnetId = constr(regex=r"^subnet-[a-z0-9]{17}$")
+AwsSubnetId = constr(pattern=r"^subnet-[a-z0-9]{17}$")
 
 
 class vedaAppSettings(BaseSettings):
@@ -146,6 +147,7 @@ class Config:
         """model config."""
 
         env_file = ".env"
+        extra = "ignore"
 
 
 veda_app_settings = vedaAppSettings()

database/infrastructure/config.py

@@ -2,7 +2,8 @@
 from typing import Optional
 
 from aws_cdk import aws_ec2, aws_rds
-from pydantic import BaseSettings, Field, validator
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings
 
 
 class vedaDBSettings(BaseSettings):
@@ -42,17 +43,17 @@ class vedaDBSettings(BaseSettings):
     max_locks_per_transaction: Optional[str] = Field(
         "1024",
         description="Number of database objects that can be locked simultaneously",
-        regex=r"^[1-9]\d*$",
+        pattern=r"^[1-9]\d*$",
     )
     work_mem: Optional[str] = Field(
         "64000",
         description="Maximum amount of memory to be used by a query operation before writing to temporary disk files",
-        regex=r"^[1-9]\d*$",
+        pattern=r"^[1-9]\d*$",
     )
     temp_buffers: Optional[str] = Field(
         "32000",
         description="maximum number of temporary buffers used by each session",
-        regex=r"^[1-9]\d*$",
+        pattern=r"^[1-9]\d*$",
     )
     use_rds_proxy: Optional[bool] = Field(
         False,
@@ -64,13 +65,15 @@ class vedaDBSettings(BaseSettings):
             "The instance class of the RDS instance "
             "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/InstanceClass.html"
         ),
+        validate_default=True,
     )
     rds_instance_size: Optional[str] = Field(
         aws_ec2.InstanceSize.SMALL.value,
         description=(
             "The size of the RDS instance "
             "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/InstanceSize.html"
         ),
+        validate_default=True,
     )
     rds_engine_full_version: Optional[str] = Field(
         aws_rds.PostgresEngineVersion.VER_14.postgres_full_version,
@@ -91,14 +94,14 @@ class vedaDBSettings(BaseSettings):
         description="Boolean if the RDS should be encrypted",
     )
 
-    @validator("rds_instance_class", pre=True, always=True)
+    @field_validator("rds_instance_class", mode="before")
     def convert_rds_class_to_uppercase(cls, value):
         """Convert to uppercase."""
         if isinstance(value, str):
             return value.upper()
         return value
 
-    @validator("rds_instance_size", pre=True, always=True)
+    @field_validator("rds_instance_size", mode="before")
     def convert_rds_size_to_uppercase(cls, value):
         """Convert to uppercase."""
         if isinstance(value, str):
@@ -110,6 +113,7 @@ class Config:
 
         env_file = ".env"
         env_prefix = "VEDA_DB_"
+        extra = "ignore"
 
 
 veda_db_settings = vedaDBSettings()

database/infrastructure/construct.py

@@ -48,13 +48,13 @@ def __init__(
             self,
             "lambda",
             handler="handler.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_9,
+            runtime=aws_lambda.Runtime.PYTHON_3_11,
             code=aws_lambda.Code.from_docker_build(
                 path=os.path.abspath("./"),
                 file="database/runtime/Dockerfile",
                 build_args={"PGSTAC_VERSION": pgstac_version},
             ),
-            timeout=Duration.minutes(2),
+            timeout=Duration.minutes(5),
             vpc=database.vpc,
             log_retention=aws_logs.RetentionDays.ONE_WEEK,
         )

database/runtime/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.9:latest
+FROM --platform=linux/amd64 public.ecr.aws/sam/build-python3.11:latest
 
 ARG PGSTAC_VERSION
 RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"

docker-compose.yml

@@ -129,6 +129,12 @@ services:
       - PGDATABASE=postgis
       - DYNAMODB_ENDPOINT=http://localhost:8085
       - VEDA_DB_PGSTAC_VERSION=0.7.10
+      - AWS_REGION=us-west-2
+      - AWS_DEFAULT_REGION=us-west-2
+      - DYNAMODB_TABLE=veda
+      - STAC_URL=http://0.0.0.0:8081
+      - USERPOOL_ID=us-west-2_123456789
+      - CLIENT_ID=123456789
     ports:
       - "8083:8083"
     command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py"

ingest_api/infrastructure/config.py

@@ -2,9 +2,11 @@
 from typing import List, Optional
 
 import aws_cdk
-from pydantic import AnyHttpUrl, BaseSettings, Field, constr
+from pydantic import AnyHttpUrl, Field, StringConstraints
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing_extensions import Annotated
 
-AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
+AwsArn = Annotated[str, StringConstraints(pattern=r"^arn:aws:iam::\d{12}:role/.+")]
 
 
 class IngestorConfig(BaseSettings):
@@ -92,11 +94,9 @@ class IngestorConfig(BaseSettings):
         False,
         description="Boolean to disable default API gateway endpoints for stac, raster, and ingest APIs. Defaults to false.",
     )
-
-    class Config:
-        case_sensitive = False
-        env_file = ".env"
-        env_prefix = "VEDA_"
+    model_config = SettingsConfigDict(
+        case_sensitive=False, env_file=".env", env_prefix="VEDA_", extra="ignore"
+    )
 
     @property
     def stack_name(self) -> str:

ingest_api/infrastructure/construct.py

@@ -52,7 +52,7 @@ def __init__(
             "RASTER_URL": config.veda_raster_api_cf_url,
             "ROOT_PATH": config.ingest_root_path,
             "STAGE": config.stage,
-            "COGNITO_DOMAIN": config.cognito_domain,
+            "COGNITO_DOMAIN": str(config.cognito_domain),
         }
 
         build_api_lambda_params = {
@@ -149,7 +149,7 @@ def build_api_lambda(
                 platform="linux/amd64",
                 build_args={"PGSTAC_VERSION": pgstac_version},
             ),
-            runtime=aws_lambda.Runtime.PYTHON_3_9,
+            runtime=aws_lambda.Runtime.PYTHON_3_11,
             timeout=Duration.seconds(30),
             handler="handler.handler",
             role=handler_role,
@@ -307,7 +307,7 @@ def build_ingestor(
                 build_args={"PGSTAC_VERSION": pgstac_version},
             ),
             handler="ingestor.handler",
-            runtime=aws_lambda.Runtime.PYTHON_3_9,
+            runtime=aws_lambda.Runtime.PYTHON_3_11,
             timeout=Duration.seconds(180),
             environment={"DB_SECRET_ARN": db_secret.secret_arn, **env},
             vpc=db_vpc,