better queryables and illegal_argument_exception fix for OS items load (stac-utils#427)

added missing copy(), updated DEFAULT_QUERYABLES so it contains only
generic attributes
**Description:**
- Removed bug (missing copy()) causing default queryables to be enriched
by queryables from previous queryables endpoint query.
- Removed attributes cloud_cover, cloud_shadow_percentage,
nodata_pixel_percentage since these are not generic for all collections
ex. SAR data.
- updated async_prep_create_item for OS item load so it works with
multiple indices

**PR Checklist:**

- [x] Code is formatted and linted (run `pre-commit run --all-files`)
- [ ] Tests pass (run `make test`)
- [ ] Documentation has been updated to reflect changes, if applicable
- [ ] Changes are added to the changelog

---------

Co-authored-by: Jonathan Healy <[email protected]>

Author: MathewNWSH

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 
+- Fixed a bug where missing `copy()` caused default queryables to be incorrectly enriched by results from previous queries. [#427](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/427)
+- Removed non-generic attributes (`cloud_cover`, `cloud_shadow_percentage`, `nodata_pixel_percentage`) not applicable to all collections (e.g., SAR data).[#427](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/427)
+- Updated `async_prep_create_item` to support OS item loading with multiple indices.[#427](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/427)
 - unified the type of queryables endpoint to `application/schema+json`. [#445](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/445)
 - updated `numReturned` & `numMatched` fields in itemCollection return to `numberReturned` & `numberMatched`. [#446](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/446)
 

stac_fastapi/core/stac_fastapi/core/extensions/filter.py

@@ -41,24 +41,6 @@
         "description": "Creation Timestamp",
         "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/updated",
     },
-    "cloud_cover": {
-        "description": "Cloud Cover",
-        "$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fields/properties/eo:cloud_cover",
-    },
-    "cloud_shadow_percentage": {
-        "title": "Cloud Shadow Percentage",
-        "description": "Cloud Shadow Percentage",
-        "type": "number",
-        "minimum": 0,
-        "maximum": 100,
-    },
-    "nodata_pixel_percentage": {
-        "title": "No Data Pixel Percentage",
-        "description": "No Data Pixel Percentage",
-        "type": "number",
-        "minimum": 0,
-        "maximum": 100,
-    },
 }
 """Queryables that are present in all collections."""
 

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -3,8 +3,9 @@
 import asyncio
 import logging
 from base64 import urlsafe_b64decode, urlsafe_b64encode
+from collections.abc import Iterable
 from copy import deepcopy
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Any, Dict, List, Optional, Tuple, Type
 
 import attr
 import orjson
@@ -679,14 +680,21 @@ async def async_prep_create_item(
 
         """
         await self.check_collection_exists(collection_id=item["collection"])
+        alias = index_alias_by_collection_id(item["collection"])
+        doc_id = mk_item_id(item["id"], item["collection"])
 
-        if not exist_ok and await self.client.exists(
-            index=index_alias_by_collection_id(item["collection"]),
-            id=mk_item_id(item["id"], item["collection"]),
-        ):
-            raise ConflictError(
-                f"Item {item['id']} in collection {item['collection']} already exists"
-            )
+        if not exist_ok:
+            alias_exists = await self.client.indices.exists_alias(name=alias)
+
+            if alias_exists:
+                alias_info = await self.client.indices.get_alias(name=alias)
+                indices = list(alias_info.keys())
+
+                for index in indices:
+                    if await self.client.exists(index=index, id=doc_id):
+                        raise ConflictError(
+                            f"Item {item['id']} in collection {item['collection']} already exists"
+                        )
 
         return self.item_serializer.stac_to_db(item, base_url)
 
@@ -903,7 +911,6 @@ async def json_patch_item(
                 "add",
                 "replace",
             ]:
-
                 if operation.path == "collection" and collection_id != operation.value:
                     await self.check_collection_exists(collection_id=operation.value)
                     new_collection_id = operation.value
@@ -957,8 +964,8 @@ async def json_patch_item(
                     "script": {
                         "lang": "painless",
                         "source": (
-                            f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""  # noqa: E702
-                            f"""ctx._source.collection = '{new_collection_id}';"""  # noqa: E702
+                            f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""
+                            f"""ctx._source.collection = '{new_collection_id}';"""
                         ),
                     },
                 },
@@ -1180,7 +1187,7 @@ async def update_collection(
                     "source": {"index": f"{ITEMS_INDEX_PREFIX}{collection_id}"},
                     "script": {
                         "lang": "painless",
-                        "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""",  # noqa: E702
+                        "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""",
                     },
                 },
                 wait_for_completion=True,

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py

@@ -4,6 +4,7 @@
 from typing import Any, Dict, Optional, Tuple
 
 import attr
+from fastapi import Request
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
 from stac_fastapi.core.extensions.filter import ALL_QUERYABLES, DEFAULT_QUERYABLES
@@ -37,9 +38,11 @@ async def get_queryables(
         Returns:
             Dict[str, Any]: A dictionary containing the queryables for the given collection.
         """
+        request: Optional[Request] = kwargs.get("request")
+        url_str: str = str(request.url) if request else ""
         queryables: Dict[str, Any] = {
-            "$schema": "https://json-schema.org/draft/2019-09/schema",
-            "$id": "https://stac-api.example.com/queryables",
+            "$schema": "https://json-schema.org/draft-07/schema",
+            "$id": f"{url_str}",
             "type": "object",
             "title": "Queryables for STAC API",
             "description": "Queryable names for the STAC API Item Search filter.",
@@ -49,7 +52,7 @@ async def get_queryables(
         if not collection_id:
             return queryables
 
-        properties: Dict[str, Any] = queryables["properties"]
+        properties: Dict[str, Any] = queryables["properties"].copy()
         queryables.update(
             {
                 "properties": properties,