feat: update ingestor-api runtime dependencies (#125)

---------

Co-authored-by: Vincent Sarago <[email protected]>

Author: hrodmn

lib/ingestor-api/runtime/requirements.txt

@@ -3,9 +3,8 @@ cachetools==5.3.0
 fastapi>=0.75.1
 orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
-pydantic_ssm_settings>=0.2.0,<1.0
+pydantic_ssm_settings>=1.0,<2.0
 pydantic>=1.9.0
 pypgstac==0.8.5
 requests>=2.27.1
-# Waiting for https://github.com/stac-utils/stac-pydantic/pull/116
-stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1
+stac-pydantic>=3.0,<4.0

lib/ingestor-api/runtime/src/config.py

@@ -2,10 +2,11 @@
 from getpass import getuser
 from typing import Optional
 
-from pydantic import AnyHttpUrl, BaseSettings, Field, constr
-from pydantic_ssm_settings import AwsSsmSourceConfig
+from pydantic import AnyHttpUrl, Field, constr
+from pydantic_settings import BaseSettings
+from pydantic_ssm_settings.settings import SsmBaseSettings
 
-AwsArn = constr(regex=r"^arn:aws:iam::\d{12}:role/.+")
+AwsArn = constr(pattern=r"^arn:aws:iam::\d{12}:role/.+")
 
 
 class Settings(BaseSettings):
@@ -27,8 +28,8 @@ class Settings(BaseSettings):
         description="Path from where to serve this URL.", default=False
     )
 
-    class Config(AwsSsmSourceConfig):
-        env_file = ".env"
+    class Config(SsmBaseSettings):
+        env_file: str = ".env"
 
     @classmethod
     def from_ssm(cls, stack: str):

lib/ingestor-api/runtime/src/schemas.py

@@ -14,7 +14,7 @@
     PositiveInt,
     dataclasses,
     error_wrappers,
-    validator,
+    field_validator,
 )
 from stac_pydantic import Collection, Item, shared
 
@@ -25,7 +25,7 @@
 
 
 class AccessibleAsset(shared.Asset):
-    @validator("href")
+    @field_validator("href")
     def is_accessible(cls, href):
         url = urlparse(href)
 
@@ -44,7 +44,7 @@ def is_accessible(cls, href):
 class AccessibleItem(Item):
     assets: Dict[str, AccessibleAsset]
 
-    @validator("collection")
+    @field_validator("collection")
     def exists(cls, collection):
         validators.collection_exists(collection_id=collection)
         return collection
@@ -66,15 +66,14 @@ class Status(str, enum.Enum):
 class Ingestion(BaseModel):
     id: str
     status: Status
-    message: Optional[str]
+    message: Optional[str] = None
     created_by: str
-    created_at: datetime = None
-    updated_at: datetime = None
+    created_at: datetime = datetime.now()
+    updated_at: datetime = datetime.now()
 
     item: Union[Item, Json[Item]]
 
-    @validator("created_at", pre=True, always=True, allow_reuse=True)
-    @validator("updated_at", pre=True, always=True, allow_reuse=True)
+    @field_validator("created_at", "updated_at", mode="before")
     def set_ts_now(cls, v):
         return v or datetime.now()
 
@@ -94,10 +93,10 @@ def save(self, db: "services.Database"):
     def dynamodb_dict(self):
         """DynamoDB-friendly serialization"""
         # convert to dictionary
-        output = self.dict(exclude={"item"})
+        output = self.model_dump(exclude={"item"})
 
         # add STAC item as string
-        output["item"] = self.item.json()
+        output["item"] = self.item.model_dump_json()
 
         # make JSON-friendly (will be able to do with Pydantic V2, https://github.com/pydantic/pydantic/issues/1409#issuecomment-1423995424)
         return jsonable_encoder(output)
@@ -106,7 +105,7 @@ def dynamodb_dict(self):
 @dataclasses.dataclass
 class ListIngestionRequest:
     status: Status = Status.queued
-    limit: PositiveInt = None
+    limit: Optional[PositiveInt] = None
     next: Optional[str] = None
 
     def __post_init_post_parse__(self) -> None:
@@ -133,7 +132,7 @@ class ListIngestionResponse(BaseModel):
     items: List[Ingestion]
     next: Optional[str]
 
-    @validator("next", pre=True)
+    @field_validator("next", mode="before")
     def b64_encode_next(cls, next):
         """
         Base64 encode next parameter for easier transportability
@@ -144,5 +143,5 @@ def b64_encode_next(cls, next):
 
 
 class UpdateIngestionRequest(BaseModel):
-    status: Status = None
-    message: str = None
+    status: Optional[Status] = None
+    message: Optional[str] = None

lib/ingestor-api/runtime/tests/conftest.py

@@ -21,6 +21,7 @@ def test_environ():
     os.environ["STAC_URL"] = "https://test-stac.url"
     os.environ["DATA_ACCESS_ROLE"] = "arn:aws:iam::123456789012:role/test-role"
     os.environ["DB_SECRET_ARN"] = "testing"
+    os.environ["ROOT_PATH"] = "testing"
 
 
 @pytest.fixture
@@ -259,13 +260,13 @@ def stac_collection(example_stac_collection):
     return schemas.StacCollection(**example_stac_collection)
 
 
-@pytest.fixture
+@pytest.fixture(scope="function")
 def example_ingestion(example_stac_item):
     from src import schemas
 
     return schemas.Ingestion(
         id=example_stac_item["id"],
         created_by="test-user",
         status=schemas.Status.queued,
-        item=Item.parse_obj(example_stac_item),
+        item=Item.model_validate(example_stac_item),
     )

lib/ingestor-api/runtime/tests/test_registration.py

@@ -74,7 +74,9 @@ def test_create(self, client_authenticated, collection_exists, asset_exists):
         )
 
         assert response.status_code == 201
-        assert collection_exists.called_once_with(self.example_ingestion.item.collection)
+        collection_exists.assert_called_once_with(
+            collection_id=self.example_ingestion.item.collection
+        )
 
         stored_data = self.db.fetch_many(status="queued")["items"]
         assert len(stored_data) == 1
@@ -87,7 +89,6 @@ def test_validates_missing_collection(
             ingestion_endpoint,
             json=jsonable_encoder(self.example_ingestion.item),
         )
-
         collection_missing.assert_called_once_with(
             collection_id=self.example_ingestion.item.collection
         )
@@ -140,7 +141,7 @@ def setup(
     def populate_table(self, count=100) -> List["schemas.Ingestion"]:
         example_ingestions = []
         for i in range(count):
-            ingestion = self.example_ingestion.copy()
+            ingestion = self.example_ingestion.model_copy()
             ingestion.id = str(i)
             ingestion.created_at = ingestion.created_at + timedelta(hours=i)
             self.mock_table.put_item(Item=ingestion.dynamodb_dict())

tox.ini

@@ -9,7 +9,7 @@ passenv = AWS_DEFAULT_REGION
 commands =
       pip install -r ./lib/ingestor-api/runtime/requirements.txt
       pip install -r ./lib/ingestor-api/runtime/dev_requirements.txt
-      python -m pytest -s
+      python -m pytest -s -vv
 
 [pytest]
 addopts = -ra -q