Merge branch 'main' into #213-urlencode-next-href

Author: jonhealy1

CHANGELOG.md

@@ -9,13 +9,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- use index templates for Collection and Item indices [#208](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions/208)
 - Added API `title`, `version`, and `description` parameters from environment variables `STAC_FASTAPI_TITLE`, `STAC_FASTAPI_VERSION` and `STAC_FASTAPI_DESCRIPTION`, respectively. [#207](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/207)
 
 ### Changed
 
 ### Fixed
 
 - URL encode next href: [#213](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/213)
+- Do not overwrite links in Item and Collection objects before persisting in database [#210](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/210)
 
 ## [v2.1.0]
 

README.md

@@ -116,10 +116,12 @@ make ingest
 
 ## Elasticsearch Mappings
 
-Mappings apply to search index, not source.  
+Mappings apply to search index, not source. The mappings are stored in index templates on application startup. 
+These templates will be used implicitly when creating new Collection and Item indices.
 
 
 ## Managing Elasticsearch Indices
+### Snapshots
 
 This section covers how to create a snapshot repository and then create and restore snapshots with this.
 
@@ -219,3 +221,52 @@ curl -X "POST" "http://localhost:8080/collections" \
 
 Voila! You have a copy of the collection now that has a resource URI (`/collections/my-collection-copy`) and can be
 correctly queried by collection name.
+
+### Reindexing
+This section covers how to reindex documents stored in Elasticsearch/OpenSearch. 
+A reindex operation might be useful to apply changes to documents or to correct dynamically generated mappings.
+
+The index templates will make sure that manually created indices will also have the correct mappings and settings.
+
+In this example, we will make a copy of an existing Item index `items_my-collection-000001` but change the Item identifier to be lowercase.
+
+```shell
+curl -X "POST" "http://localhost:9200/_reindex" \
+  -H 'Content-Type: application/json' \
+  -d $'{
+    "source": {
+      "index": "items_my-collection-000001"
+    }, 
+    "dest": {
+      "index": "items_my-collection-000002"
+    },
+    "script": {
+      "source": "ctx._source.id = ctx._source.id.toLowerCase()",
+      "lang": "painless"
+    }
+  }'
+```
+
+If we are happy with the data in the newly created index, we can move the alias `items_my-collection` to the new index `items_my-collection-000002`.
+```shell
+curl -X "POST" "http://localhost:9200/_aliases" \
+  -h 'Content-Type: application/json' \
+  -d $'{
+    "actions": [
+      {
+        "remove": {
+          "index": "*",
+          "alias": "items_my-collection"
+        }
+      },
+      {
+        "add": {
+          "index": "items_my-collection-000002",
+          "alias": "items_my-collection"
+        }
+      }
+    ]
+  }'
+```
+
+The modified Items with lowercase identifiers will now be visible to users accessing `my-collection` in the STAC API.

stac_fastapi/core/stac_fastapi/core/core.py

@@ -37,7 +37,6 @@
 from stac_fastapi.types.config import Settings
 from stac_fastapi.types.conformance import BASE_CONFORMANCE_CLASSES
 from stac_fastapi.types.extension import ApiExtension
-from stac_fastapi.types.links import CollectionLinks
 from stac_fastapi.types.requests import get_base_url
 from stac_fastapi.types.search import BaseSearchPostRequest
 from stac_fastapi.types.stac import Collection, Collections, Item, ItemCollection
@@ -731,10 +730,9 @@ async def create_collection(
             ConflictError: If the collection already exists.
         """
         base_url = str(kwargs["request"].base_url)
-        collection_links = CollectionLinks(
-            collection_id=collection["id"], base_url=base_url
-        ).create_links()
-        collection["links"] = collection_links
+        collection = self.database.collection_serializer.stac_to_db(
+            collection, base_url
+        )
         await self.database.create_collection(collection=collection)
 
         return CollectionSerializer.db_to_stac(collection, base_url)
@@ -767,11 +765,9 @@ async def update_collection(
             "collection_id", collection["id"]
         )
 
-        collection_links = CollectionLinks(
-            collection_id=collection["id"], base_url=base_url
-        ).create_links()
-        collection["links"] = collection_links
-
+        collection = self.database.collection_serializer.stac_to_db(
+            collection, base_url
+        )
         await self.database.update_collection(
             collection_id=collection_id, collection=collection
         )

stac_fastapi/core/stac_fastapi/core/serializers.py

@@ -60,11 +60,7 @@ def stac_to_db(cls, stac_data: stac_types.Item, base_url: str) -> stac_types.Ite
         Returns:
             stac_types.Item: The database-ready STAC item object.
         """
-        item_links = ItemLinks(
-            collection_id=stac_data["collection"],
-            item_id=stac_data["id"],
-            base_url=base_url,
-        ).create_links()
+        item_links = resolve_links(stac_data.get("links", []), base_url)
         stac_data["links"] = item_links
 
         now = now_to_rfc3339_str()
@@ -111,6 +107,24 @@ def db_to_stac(cls, item: dict, base_url: str) -> stac_types.Item:
 class CollectionSerializer(Serializer):
     """Serialization methods for STAC collections."""
 
+    @classmethod
+    def stac_to_db(
+        cls, collection: stac_types.Collection, base_url: str
+    ) -> stac_types.Collection:
+        """
+        Transform STAC Collection to database-ready STAC collection.
+
+        Args:
+            stac_data: the STAC Collection object to be transformed
+            base_url: the base URL for the STAC API
+
+        Returns:
+            stac_types.Collection: The database-ready STAC Collection object.
+        """
+        collection = deepcopy(collection)
+        collection["links"] = resolve_links(collection.get("links", []), base_url)
+        return collection
+
     @classmethod
     def db_to_stac(cls, collection: dict, base_url: str) -> stac_types.Collection:
         """Transform database model to STAC collection.

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

@@ -16,6 +16,7 @@
 from stac_fastapi.elasticsearch.database_logic import (
     DatabaseLogic,
     create_collection_index,
+    create_index_templates,
 )
 from stac_fastapi.extensions.core import (
     ContextExtension,
@@ -78,6 +79,7 @@
 
 @app.on_event("startup")
 async def _startup_event() -> None:
+    await create_index_templates()
     await create_collection_index()
 
 

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -171,9 +171,36 @@ def indices(collection_ids: Optional[List[str]]) -> str:
         return ",".join([index_by_collection_id(c) for c in collection_ids])
 
 
+async def create_index_templates() -> None:
+    """
+    Create index templates for the Collection and Item indices.
+
+    Returns:
+        None
+
+    """
+    client = AsyncElasticsearchSettings().create_client
+    await client.indices.put_template(
+        name=f"template_{COLLECTIONS_INDEX}",
+        body={
+            "index_patterns": [f"{COLLECTIONS_INDEX}*"],
+            "mappings": ES_COLLECTIONS_MAPPINGS,
+        },
+    )
+    await client.indices.put_template(
+        name=f"template_{ITEMS_INDEX_PREFIX}",
+        body={
+            "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
+            "settings": ES_ITEMS_SETTINGS,
+            "mappings": ES_ITEMS_MAPPINGS,
+        },
+    )
+    await client.close()
+
+
 async def create_collection_index() -> None:
     """
-    Create the index for a Collection.
+    Create the index for a Collection. The settings of the index template will be used implicitly.
 
     Returns:
         None
@@ -184,14 +211,13 @@ async def create_collection_index() -> None:
     await client.options(ignore_status=400).indices.create(
         index=f"{COLLECTIONS_INDEX}-000001",
         aliases={COLLECTIONS_INDEX: {}},
-        mappings=ES_COLLECTIONS_MAPPINGS,
     )
     await client.close()
 
 
 async def create_item_index(collection_id: str):
     """
-    Create the index for Items.
+    Create the index for Items. The settings of the index template will be used implicitly.
 
     Args:
         collection_id (str): Collection identifier.
@@ -206,8 +232,6 @@ async def create_item_index(collection_id: str):
     await client.options(ignore_status=400).indices.create(
         index=f"{index_by_collection_id(collection_id)}-000001",
         aliases={index_name: {}},
-        mappings=ES_ITEMS_MAPPINGS,
-        settings=ES_ITEMS_SETTINGS,
     )
     await client.close()
 

stac_fastapi/opensearch/stac_fastapi/opensearch/app.py

@@ -25,6 +25,7 @@
 from stac_fastapi.opensearch.database_logic import (
     DatabaseLogic,
     create_collection_index,
+    create_index_templates,
 )
 
 settings = OpensearchSettings()
@@ -78,6 +79,7 @@
 
 @app.on_event("startup")
 async def _startup_event() -> None:
+    await create_index_templates()
     await create_collection_index()
 
 

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -173,18 +173,44 @@ def indices(collection_ids: Optional[List[str]]) -> str:
         return ",".join([index_by_collection_id(c) for c in collection_ids])
 
 
+async def create_index_templates() -> None:
+    """
+    Create index templates for the Collection and Item indices.
+
+    Returns:
+        None
+
+    """
+    client = AsyncSearchSettings().create_client
+    await client.indices.put_template(
+        name=f"template_{COLLECTIONS_INDEX}",
+        body={
+            "index_patterns": [f"{COLLECTIONS_INDEX}*"],
+            "mappings": ES_COLLECTIONS_MAPPINGS,
+        },
+    )
+    await client.indices.put_template(
+        name=f"template_{ITEMS_INDEX_PREFIX}",
+        body={
+            "index_patterns": [f"{ITEMS_INDEX_PREFIX}*"],
+            "settings": ES_ITEMS_SETTINGS,
+            "mappings": ES_ITEMS_MAPPINGS,
+        },
+    )
+    await client.close()
+
+
 async def create_collection_index() -> None:
     """
-    Create the index for a Collection.
+    Create the index for a Collection. The settings of the index template will be used implicitly.
 
     Returns:
         None
 
     """
     client = AsyncSearchSettings().create_client
 
-    search_body = {
-        "mappings": ES_COLLECTIONS_MAPPINGS,
+    search_body: Dict[str, Any] = {
         "aliases": {COLLECTIONS_INDEX: {}},
     }
 
@@ -203,7 +229,7 @@ async def create_collection_index() -> None:
 
 async def create_item_index(collection_id: str):
     """
-    Create the index for Items.
+    Create the index for Items. The settings of the index template will be used implicitly.
 
     Args:
         collection_id (str): Collection identifier.
@@ -214,10 +240,8 @@ async def create_item_index(collection_id: str):
     """
     client = AsyncSearchSettings().create_client
     index_name = index_by_collection_id(collection_id)
-    search_body = {
+    search_body: Dict[str, Any] = {
         "aliases": {index_name: {}},
-        "mappings": ES_ITEMS_MAPPINGS,
-        "settings": ES_ITEMS_SETTINGS,
     }
 
     try:
@@ -335,7 +359,7 @@ async def get_all_collections(
             search_body["search_after"] = search_after
 
         response = await self.client.search(
-            index="collections",
+            index=COLLECTIONS_INDEX,
             body=search_body,
         )
 

stac_fastapi/tests/conftest.py

@@ -23,6 +23,7 @@
     from stac_fastapi.opensearch.database_logic import (
         DatabaseLogic,
         create_collection_index,
+        create_index_templates,
     )
 else:
     from stac_fastapi.elasticsearch.config import (
@@ -32,6 +33,7 @@
     from stac_fastapi.elasticsearch.database_logic import (
         DatabaseLogic,
         create_collection_index,
+        create_index_templates,
     )
 
 from stac_fastapi.extensions.core import (  # FieldsExtension,
@@ -215,6 +217,7 @@ async def app():
 
 @pytest_asyncio.fixture(scope="session")
 async def app_client(app):
+    await create_index_templates()
     await create_collection_index()
 
     async with AsyncClient(app=app, base_url="http://test-server") as c: