wip

jeafreezy · jeafreezy · commit 395ad06085fe · 2025-07-12T11:27:40.000+02:00
diff --git a/README.md b/README.md
@@ -29,4 +29,14 @@ catalogs:
 
 # Todo - auth configuration documentation.
 
-See [.superstac.yml](./superstac/.superstac.yml) for an example configuration file.
+See [.superstac.yml](./superstac/.superstac.yml) for an example configuration file.
+
+
+# todo
+
+- retries - https://pystac-client.readthedocs.io/en/stable/usage.html#configuring-retry-behavior
+- modifier - https://pystac-client.readthedocs.io/en/stable/usage.html#automatically-modifying-results
+- refresh
+- auth
+- ues cases e.g when a catalog is offline - store latency ?
+- when a catalog is specified and changed it still works - band matching
diff --git a/main.py b/main.py
@@ -1,5 +1,32 @@
+import time
+import asyncio
+
+
 def main():
-    print("Hello from superstac!")
+    from superstac import get_catalog_registry, federated_search_async
+
+    cr = get_catalog_registry()
+    cr.load_catalogs_from_config()
+
+    print("\nRunning asynchronous federated_search_async...")
+    start_async = time.perf_counter()
+    results_async = asyncio.run(
+        federated_search_async(
+            registry=cr,
+            collections=["sentinel-2-l2a"],
+            bbox=[6.0, 49.0, 7.0, 50.0],
+            datetime="2024-01-01/2024-01-31",
+            query={"eo:cloud_cover": {"lt": 20}},
+            sortby=[{"field": "properties.datetime", "direction": "desc"}],
+        )
+    )
+    end_async = time.perf_counter()
+    print(
+        f"Asynchronous search found {len(results_async)} items in {end_async - start_async:.2f} seconds."
+    )
+
+    for x in results_async:
+        print(x.self_href)
 
 
 if __name__ == "__main__":
diff --git a/superstac/.superstac.yml b/superstac/.superstac.yml
@@ -1,9 +1,7 @@
 catalogs:
   Earth Search:
     url: https://earth-search.aws.element84.com/v1
-    summary: 'Element 84 STAC catalog'
     is_private: False 
   Planetary Computer:
     url: https://planetarycomputer.microsoft.com/api/stac/v1
-    summary: 'Microsoft Planetary Computer STAC catalog'
     is_private: False 
diff --git a/superstac/__init__.py b/superstac/__init__.py
@@ -1,7 +1,6 @@
 from superstac.catalog_registry import (
-    register_catalog,
     get_catalog_registry,
-    load_catalogs_from_config,
 )
+from superstac.search import federated_search, federated_search_async
 
-__all__ = ["register_catalog", "get_catalog_registry", "load_catalogs_from_config"]
+__all__ = ["get_catalog_registry", "federated_search", "federated_search_async"]
diff --git a/superstac/assets_mapper.py b/superstac/assets_mapper.py
@@ -0,0 +1,17 @@
+planetary_mapping = {
+    "B01": "coastal",
+    "B02": "blue",
+    "B03": "green",
+    "B04": "red",
+    "B05": "rededge1",
+    "B06": "rededge2",
+    "B07": "rededge3",
+    "B08": "nir",
+    "B8A": "nir08",
+    "B09": "nir09",
+    "B11": "swir16",
+    "B12": "swir22",
+    "AOT": "aot",
+    "SCL": "scl",
+    "WVP": "wvp",
+}
diff --git a/superstac/catalog.py b/superstac/catalog.py
@@ -2,7 +2,9 @@
 
 from pathlib import Path
 import attr
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, List, Optional, Union
+
+from pystac_client import Client
 
 from superstac.enums import CatalogOutputFormat
 from superstac.exceptions import (
@@ -29,7 +31,6 @@ def register_catalog(
         name: str,
         url: str,
         is_private: Optional[bool] = False,
-        summary: Optional[str] = None,
         auth: Optional[AuthInfo] = None,
     ) -> CatalogEntry:
         """Register a single STAC catalog in state.
@@ -48,54 +49,120 @@ def register_catalog(
             CatalogEntry: The registered STAC catalog.
         """
         logger.info(f"Registering catalog: {name}")
-        logger.debug(
-            f"Params - url: {url}, is_private: {is_private}, summary: {summary}, auth: {auth}"
-        )
+        logger.debug(f"Params - url: {url}, is_private: {is_private},  auth: {auth}")
         if is_private and auth is None:
             logger.error(
                 f"Private catalog '{name}' requires authentication but none was provided."
             )
             raise InvalidCatalogSchemaError(
                 f"Authentication parameters is required for private catalogs. If this is a mistake, you can set 'is_private' to False or provide the {AuthInfo.__annotations__} parameters."
             )
-
-        logger.info(f"Catalog '{name}' registered successfully.")
-        self.catalogs[name] = CatalogEntry(
+        client = None
+        try:
+            # todo - add auth parameters and config...
+            client = Client.open(url)
+            metadata = {
+                "client": client,
+                "catalog": client.get_root(),
+                "is_available": True,
+            }
+            logger.info(f"Catalog '{name}' is reachable and valid.")
+        except Exception as e:
+            logger.warning(f"Catalog '{name}' could not be reached or parsed: {e}")
+            metadata = {
+                "client": None,
+                "catalog": None,
+                "is_available": False,
+            }
+        entry = CatalogEntry(
             name=name,
             url=url,
-            summary=summary,
             is_private=is_private,
             auth=AuthInfo(**auth.__dict__) if auth and not is_private else None,
+            **metadata,
         )
-        return self.catalogs[name]
+        self.catalogs[name] = entry
+        logger.info(f"Catalog '{name}' registered successfully.")
+        return entry
 
-    def get_available_catalogs(
-        self, format: Union[str, CatalogOutputFormat] = CatalogOutputFormat.DICT
+    def get_catalogs(
+        self,
+        format: Union[str, CatalogOutputFormat] = CatalogOutputFormat.DICT,
+        available: bool = False,
     ) -> list[Union[dict[str, Any], str]]:
-        """Get the available STAC catalogs.
+        """Get the STAC catalogs.
+
+        Args:
+            format (Union[str, CatalogOutputFormat]): Output format, dict or json string.
+            available (bool): If True, return only available catalogs; else return all.
 
         Raises:
             ValueError: When an invalid format is provided.
 
         Returns:
-            list[CatalogEntry]: The list of all available STAC catalogs.
+            list[CatalogEntry]: List of catalogs in the requested format.
         """
-        logger.info("Retrieving available catalogs.")
+        logger.info(f"Retrieving {'available' if available else 'all'} catalogs.")
         if isinstance(format, str):
             try:
                 format = CatalogOutputFormat(format.lower())
             except ValueError:
                 logger.error(f"Invalid output format: {format}")
                 raise ValueError(f"Invalid format: {format}")
 
-        available = [
+        catalogs = [
             c.as_dict() if format == CatalogOutputFormat.DICT else c.as_json()
             for c in self.catalogs.values()
-            if c.is_available
+            if (c.is_available if available else True)
         ]
 
-        logger.info(f"{len(available)} catalogs available in format '{format.value}'.")
-        return available
+        logger.info(f"{len(catalogs)} catalogs retrieved in format '{format.value}'.")
+        return catalogs
+
+    def get_all_collections(self, available: bool = False) -> Dict[str, List[str]]:
+        """
+        Returns a dictionary mapping catalog names to a list of collection IDs
+        available in each catalog.
+
+        Args:
+            available (bool): If True, only include catalogs that are available.
+
+        Returns:
+            Dict[str, List[str]]: Catalog name -> list of collection IDs
+        """
+        logger.info(f"Getting collections with available={available}")
+        collections = {}
+        for name, entry in self.catalogs.items():
+            if (entry.is_available if available else True) and entry.catalog:
+                logger.debug(
+                    f"Processing catalog '{name}' (available={entry.is_available})"
+                )
+                try:
+                    all_collections = entry.catalog.get_all_collections()
+                    if all_collections:
+                        collection_ids = []
+                        for c in all_collections:
+                            if hasattr(c, "id"):
+                                collection_ids.append(c.id)
+                            elif isinstance(c, dict) and "id" in c:
+                                collection_ids.append(c["id"])
+                            else:
+                                collection_ids.append(str(c))
+                        collections[name] = collection_ids
+                        logger.info(
+                            f"Found {len(collection_ids)} collections in catalog '{name}'"
+                        )
+                    else:
+                        logger.info(f"No collections found in catalog '{name}'")
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to get collections for catalog '{name}': {e}"
+                    )
+                    continue
+            else:
+                logger.debug(f"Skipping catalog '{name}' due to availability filter")
+        logger.info(f"Total catalogs with collections returned: {len(collections)}")
+        return collections
 
     def load_catalogs_from_config(
         self, config: Union[str, Path, None] = None
@@ -155,7 +222,6 @@ def load_catalogs_from_config(
                     name=name,
                     url=spec.get("url"),
                     is_private=spec.get("is_private", False),
-                    summary=spec.get("summary"),
                     auth=AuthInfo(**spec["auth"]) if "auth" in spec else None,
                 )
             except Exception as e:
diff --git a/superstac/catalog_registry.py b/superstac/catalog_registry.py
@@ -1,35 +1,25 @@
 """SuperSTAC Catalog Registry"""
 
-from typing import Union
 from superstac.catalog import CatalogManager
 
 
-_catalog_registry = CatalogManager()
+_catalog_registry = None
 
 
 def get_catalog_registry() -> CatalogManager:
     """
     Returns the singleton CatalogManager instance.
     """
+    global _catalog_registry
+    if _catalog_registry is None:
+        _catalog_registry = CatalogManager()
     return _catalog_registry
 
 
-def register_catalog(*args, **kwargs):
+def clear_registry() -> None:
     """
-    Shortcut to register a catalog globally.
+    Reset the registry, mainly for testing.
     """
-    return _catalog_registry.register_catalog(*args, **kwargs)
-
-
-def load_catalogs_from_config(file: Union[str, None] = None):
-    """
-    Loads catalogs from YAML into the global registry.
-    """
-    return _catalog_registry.load_catalogs_from_config(file)
-
-
-def clear_registry():
-    """
-    Optional: Reset the registry, mainly for testing.
-    """
-    _catalog_registry.catalogs.clear()
+    if _catalog_registry:
+        _catalog_registry.catalogs.clear()
+    return None
diff --git a/superstac/models.py b/superstac/models.py
@@ -2,9 +2,13 @@
 
 from dataclasses import asdict, dataclass
 import json
-from typing import Optional
+from typing import Optional, Union
 from urllib.parse import urlparse
 
+
+from pystac import Catalog
+from pystac_client import Client
+
 from superstac.enums import AuthType
 from superstac.utils import compute_catalog_id
 
@@ -32,18 +36,27 @@ class CatalogEntry:
     """CatalogEntry"""
 
     name: str
+
     url: str
+    """URL to the STAC catalog."""
+
     id: Optional[str] = None
     """Internal unique ID for the catalog. Will be autogenerated post init."""
-    """URL to the STAC catalog."""
+
+    client: Union[Client, None] = None
+    """PySTAC Client"""
+
+    catalog: Union[Catalog, None] = None
+    """Root Catalog"""
+
     is_private: Optional[bool] = False
     """Indicates whether the catalog is a private catalog or not."""
-    summary: Optional[str] = None
-    """Short description of the catalog."""
+
     auth: Optional[AuthInfo] = None
     """Authentication parameters."""
+
     is_available: Optional[bool] = True
-    """Defaults to True on instantiation. It will be updated based on the status code of the catalog. If 200 True and False if otherwise."""
+    """Defaults to True on instantiation. It will be updated based on the status code of the catalog from pystac client."""
 
     def __post_init__(self):
         # Validate the URL
diff --git a/superstac/query.py b/superstac/query.py
diff --git a/superstac/search.py b/superstac/search.py

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,6 @@`
`1`	`1`	`from superstac.catalog_registry import (`
`2`		`- register_catalog,`
`3`	`2`	`get_catalog_registry,`
`4`		`- load_catalogs_from_config,`
`5`	`3`	`)`
	`4`	`+from superstac.search import federated_search, federated_search_async`
`6`	`5`
`7`		`-__all__ = ["register_catalog", "get_catalog_registry", "load_catalogs_from_config"]`
	`6`	`+__all__ = ["get_catalog_registry", "federated_search", "federated_search_async"]`