Modified some fields to make them more STAC-compliant

Author: s-boomi

README.md

@@ -147,9 +147,12 @@ settings:
   # Folder containing anything that can be used as a side material for processing
   extra_data_path: "./data/extra"
   # Contains the scrapped results from the WKT projections
-  wkt_file_name: "wkt_solar_system.csv"
+  wkt_file_path: "wkt_solar_system.csv"
   # Contains all the scraped results by using `psup-scraper get-data-ref -O <psup-inventory-file> -f csv --clean`
   psup_inventory_file: "./data/raw/psup_refs.csv"
+  # For debug purposes, you can pass the number of OMEGA items to generate (optional)
+  n_omega_items: 10
+
 ```
 You can find an example from [`converter-params.example.yml`](./converter-params.example.yml)
 

converter-params.example.yml

@@ -16,3 +16,5 @@ settings:
   wkt_file_path: "wkt_solar_system.csv"
   # Contains all the scraped results by using `psup-scraper get-data-ref -O <psup-inventory-file> -f csv --clean`
   psup_inventory_file: "./data/raw/psup_refs.csv"
+  # For debug purposes, you can pass the number of OMEGA items to generate (optional)
+  n_omega_items: 10

notebooks/catalog_reading.py

@@ -9,13 +9,16 @@ def _():
     import marimo as mo
     import pystac
     from pathlib import Path
+    from pystac.extensions.datacube import DatacubeExtension
 
     return Path, mo, pystac
 
 
 @app.cell
 def _(Path, pystac):
-    catalog = pystac.Catalog.from_file(Path.home() / "data-folder/catalog/catalog.json")
+    catalog = pystac.Catalog.from_file(
+        Path.home() / "PDSSP/psup-stac-repo/catalog/catalog.json"
+    )
     catalog.describe()
     return (catalog,)
 
@@ -55,12 +58,27 @@ def _(catalog):
 
 
 @app.cell
-def _(catalog):
-    items = list(catalog.get_all_items())
+def _(catalog, pystac):
+    for _item in catalog.get_all_items():
+        try:
+            _item.validate()
+        except pystac.errors.STACValidationError as e:
+            print(f"{_item.id} is invalid")
+            print(e)
+            # print("==========================")
+            # if DatacubeExtension.has_extension(_item):
+            #     cube_item = DatacubeExtension.ext(_item)
+            #     print(cube_item.dimensions)
+            print("========================== =============================")
+        else:
+            pass
+            # print("Valid!")
+    return
 
-    print(f"Number of items: {len(items)}")
-    for item in items:
-        print(f"- {item.id}")
+
+@app.cell
+def _(catalog):
+    catalog.validate_all()
     return
 
 

notebooks/omega_cubes.py

@@ -81,6 +81,9 @@ def divide_by_channel(df: pd.DataFrame, name: str):
 
     data_cubes_refs = divide_by_channel(psup_refs, "cubes_L2")
     c_channel_refs = divide_by_channel(psup_refs, "cubes_L3")
+
+    print("N° items in C channel:", c_channel_refs.index.nunique())
+    print("N° items in data cubes:", data_cubes_refs.index.nunique())
     return c_channel_refs, data_cubes_refs
 
 
@@ -887,10 +890,16 @@ def _(dt, ex_nc_ds_l2, re):
     return
 
 
+@app.cell
+def _(ex_nc_ds_l2):
+    ex_nc_ds_l2.isel(wavelength=100)
+    return
+
+
 @app.cell
 def _(ex_nc_ds_l2, plt):
-    ex_nc_ds_l2.Reflectance.mean("wavelength").plot(cmap=plt.cm.get_cmap("viridis"))
-    plt.title("Reflectance map based on wavelengh mean")
+    ex_nc_ds_l2.Reflectance[100].plot(cmap=plt.cm.get_cmap("viridis"))
+    plt.title("Reflectance map based on 100th wavelength")
     return
 
 
@@ -924,16 +933,16 @@ def convert_arr_to_thumbnail(
 
         result = (result - result_min) / (result_max - result_min + 1e-8)
         # occult the NaNs and infs
-        result = np.nan_to_num(result, nan=0.0, posinf=255.0, neginf=0.0)
+        # result = np.nan_to_num(result, nan=0., posinf=255., neginf=0.)
 
         if cmap is not None:
             cm = plt.get_cmap(cmap)
-            result = cm(data)[..., :4]  # includes alpha
+            result = cm(result)[..., :4]  # includes alpha
             result = (result * 255).astype(np.uint8)
             if mode == "RGB":
                 result = result[..., :3]
         else:
-            result = (data * 255).astype(np.uint8)
+            result = (result * 255).astype(np.uint8)
             if mode in ["RGB", "RGBA"]:
                 result = np.stack([result] * (3 if mode == "RGB" else 4), axis=-1)
 
@@ -944,9 +953,9 @@ def convert_arr_to_thumbnail(
 
     tempdir = TemporaryDirectory()
     resized_img = convert_arr_to_thumbnail(
-        ex_nc_ds_l2.Reflectance.mean("wavelength").values,
+        getattr(ex_nc_ds_l2.Reflectance, "mean")("wavelength").values,
         (256, 256),
-        mode="RGB",
+        mode="RGBA",
         cmap="viridis",
     )
 

src/psup_stac_converter/_main.py

@@ -62,6 +62,7 @@ def create_catalog(
     psup_data_inventory_file: Path = None,
     wkt_file_path: Path = None,
     clean_prev_output: bool = False,
+    n_omega_items: int | None = None,
     **kwargs,
 ):
     catalog_creator = CatalogCreator(
@@ -70,5 +71,6 @@ def create_catalog(
         psup_data_inventory_file=psup_data_inventory_file,
         wkt_file=wkt_file_path,
         log=kwargs.get("logger"),
+        n_omega_files=n_omega_items,
     )
     return catalog_creator.create_catalog(clean_previous_output=clean_prev_output)

src/psup_stac_converter/cli.py

@@ -115,6 +115,12 @@ def create_stac_catalog(
             resolve_path=True,
         ),
     ] = None,
+    n_omega_items: Annotated[
+        int,
+        typer.Option(
+            "--n-omega", help="Specifies the limit of OMEGA items to generate"
+        ),
+    ] = None,
     clean_previous_output: Annotated[
         bool,
         typer.Option("--clean/--no-clean", "-c/-nc", help="Cleans the output folder"),
@@ -131,6 +137,7 @@ def create_stac_catalog(
         psup_data_inventory_file=psup_inventory_file or settings.psup_inventory_file,
         wkt_file_path=wkt_file_path or settings.wkt_file_path,
         clean_prev_output=clean_previous_output,
+        n_omega_items=n_omega_items or settings.n_omega_items,
         **{k: v for k, v in ctx.obj.items() if k not in ["settings"]},
     )
 

src/psup_stac_converter/extensions.py

@@ -13,9 +13,20 @@
 type StacInstance = pystac.Catalog | pystac.Collection | pystac.Item
 
 
-def apply_ssys(
-    stac_instance: StacInstance, mars_local_time: str | None = None
-) -> StacInstance:
+def apply_ssys(stac_instance: StacInstance, mars_local_time: str = "") -> StacInstance:
+    """Applies Solary Stsem extension over a Stac instance object
+
+    Note:
+        ssys v1.1.0 disallows Nonetypes. If the date has to be fixed somehow,
+        pass an empty string.
+
+    Args:
+        stac_instance (StacInstance): _description_
+        mars_local_time (str, optional): _description_. Defaults to "".
+
+    Returns:
+        StacInstance: _description_
+    """
     if isinstance(stac_instance, pystac.Item):
         ssys = SolSysExtension.ext(stac_instance, add_if_missing=True)
         ssys.apply(

src/psup_stac_converter/informations/publications.py

@@ -3,72 +3,72 @@
 omega_map_publications = [
     Publication(
         citation="A. Ody, F. Poulet, Y. Langevin, J.-P. Bibring, G. Bellucci, F. Altieri, B. Gondet, M. Vincendon, J.Carter and N. Manaud (2012), Global maps of anhydrous minerals at the surface of Mars from OMEGA/MEx, J. Geophys. Res., 117, E00J14",
-        doi="doi:10.1029/2012JE004117",
+        doi="10.1029/2012JE004117",
     ),
     Publication(
         citation="M. Vincendon, J. Audouard, F. Altieri, A. Ody (2015), Mars Express measurements of surface albedo changes over 2004–2010, In Icarus, Volume 251, 2015, Pages 145-163, ISSN 0019-1035",
-        doi="doi:10.1016/j.icarus.2014.10.029",
+        doi="10.1016/j.icarus.2014.10.029",
     ),
     Publication(
         citation="J. Audouard, F. Poulet, M. Vincendon, J.-P. Bibring, F. Forget, Y. Langevin, B. Gondet (2014), Mars surface thermal inertia and heterogeneities from OMEGA/MEX, In Icarus Volume 233, 2014, Pages 194-213",
-        doi="doi:10.1016/j.icarus.2014.01.045",
+        doi="10.1016/j.icarus.2014.01.045",
     ),
     Publication(
         citation="F. Poulet, C. Quantin-Nataf, H. Ballans, K. Dassas, J. Audouard, J. Carter, B. Gondet, L. Lozac’h, J.-C. Malapert, C. Marmo, L. Riu, A. Séjourné (2018), PSUP: a Planetary SUrface Portal, Planetary and Space Science, Volume 150, 2018, Pages 2-8",
-        doi="doi:10.1016/j.pss.2017.01.016",
+        doi="10.1016/j.pss.2017.01.016",
     ),
 ]
 omega_c_channel = [
     Publication(
         citation="""
 Lucie Riu, John Carter, François Poulet, Alejandro Cardesín-Moinelo, Patrick Martin, Global surficial water content stored in hydrated silicates at Mars from OMEGA/MEx, Icarus, Volume 398, 2023, 115537, ISSN 0019-1035,
 """,
-        doi="https://doi.org/10.1016/j.icarus.2023.115537",
+        doi="10.1016/j.icarus.2023.115537",
     ),
     Publication(
         citation="""
         M. Vincendon, J. Audouard, F. Altieri, A. Ody, Mars Express measurements of surface albedo changes over 2004–2010, Icarus, Volume 251, 2015, Pages 145-163, ISSN 0019-1035
         """,
-        doi="https://doi.org/10.1016/j.icarus.2014.10.029",
+        doi="10.1016/j.icarus.2014.10.029",
     ),
     Publication(
         citation="""
 Audouard, J., F. Poulet, M. Vincendon, R. E. Milliken, D. Jouglet, J.-P. Bibring, B. Gondet, and Y. Langevin (2014), Water in the Martian regolith from OMEGA/Mars Express, J. Geophys. Res. Planets, 119, 1969–1989
 """,
-        doi="doi:10.1002/2014JE004649",
+        doi="10.1002/2014JE004649",
     ),
     Publication(
         citation="""
 Langevin, Y., J.-P. Bibring, F. Montmessin, F. Forget, M. Vincendon, S. Douté, F. Poulet, and B. Gondet (2007), Observations of the south seasonal cap of Mars during recession in 2004–2006 by the OMEGA visible/near-infrared imaging spectrometer on board Mars Express, J. Geophys. Res., 112, E08S12
 """,
-        doi="doi:10.1029/2006JE002841",
+        doi="10.1029/2006JE002841",
     ),
 ]
 omega_data_cubes = [
     Publication(
         citation="M. Vincendon, J. Audouard, F. Altieri, A. Ody (2015), Mars Express measurements of surface albedo changes over 2004–2010, In Icarus, Volume 251, 2015, Pages 145-163, ISSN 0019-1035",
-        doi="doi:10.1016/j.icarus.2014.10.029",
+        doi="10.1016/j.icarus.2014.10.029",
     ),
     Publication(
         citation="""
 Audouard, J., F. Poulet, M. Vincendon, R. E. Milliken, D. Jouglet, J.-P. Bibring, B. Gondet, and Y. Langevin (2014), Water in the Martian regolith from OMEGA/Mars Express, J. Geophys. Res. Planets, 119, 1969–1989
 """,
-        doi="doi:10.1002/2014JE004649",
+        doi="10.1002/2014JE004649",
     ),
     Publication(
         citation="""
 Langevin, Y., J.-P. Bibring, F. Montmessin, F. Forget, M. Vincendon, S. Douté, F. Poulet, and B. Gondet (2007), Observations of the south seasonal cap of Mars during recession in 2004–2006 by the OMEGA visible/near-infrared imaging spectrometer on board Mars Express, J. Geophys. Res., 112, E08S12
 """,
-        doi="doi:10.1029/2006JE002841",
+        doi="10.1029/2006JE002841",
     ),
     Publication(
         citation="A. Ody, F. Poulet, Y. Langevin, J.-P. Bibring, G. Bellucci, F. Altieri, B. Gondet, M. Vincendon, J.Carter and N. Manaud (2012), Global maps of anhydrous minerals at the surface of Mars from OMEGA/MEx, J. Geophys. Res., 117, E00J14",
-        doi="doi:10.1029/2012JE004117",
+        doi="10.1029/2012JE004117",
     ),
     Publication(
         citation="""
 Denis Jouglet, François Poulet, Yves Langevin, Jean-Pierre Bibring, Brigitte Gondet, Mathieu Vincendon, Michel Berthe, OMEGA long wavelength channel: Data reduction during non-nominal stages, Planetary and Space Science, Volume 57, Issues 8–9, 2009, Pages 1032-1042, ISSN 0032-0633
 """,
-        doi="https://doi.org/10.1016/j.pss.2008.07.025.",
+        doi="10.1016/j.pss.2008.07.025.",
     ),
 ]

src/psup_stac_converter/omega/_base.py

@@ -42,6 +42,15 @@ def default(self, o: Any) -> Any:
 
 
 def reformat_nc_info(nc_info: dict[str, Any]) -> dict[str, Any]:
+    """Opens previously extracted NetCDF4 datacube info for
+    catalog input
+
+    Args:
+        nc_info (dict[str, Any]): _description_
+
+    Returns:
+        dict[str, Any]: _description_
+    """
     for k, v in nc_info["variables"].items():
         nc_info["variables"][k] = Variable(properties={**v})
     for k, v in nc_info["dimensions"].items():
@@ -171,6 +180,11 @@ def omega_data_ids(self) -> pd.Index:
     def n_elements(self) -> int:
         return self.omega_data_ids.size
 
+    def get_omega_data_ids(self, n_limit: int | None = None) -> pd.Index:
+        if n_limit is None:
+            return self.omega_data_ids
+        return self.omega_data_ids[:n_limit]
+
     def find_info_by_orbit_cube(
         self,
         orbit_cube_idx: str,
@@ -361,7 +375,7 @@ def find_temporal_extent(self) -> pystac.TemporalExtent:
             ]
         )
 
-    def create_collection(self) -> pystac.Collection:
+    def create_collection(self, n_limit: int | None = None) -> pystac.Collection:
         """Creates a STAC collection based over the OMEGA data series.
 
         Returns:
@@ -388,7 +402,10 @@ def create_collection(self) -> pystac.Collection:
         # TODO: make a pystac extension for processing
         # collection.extra_fields["processing:level"] = self.processing_level
 
-        for omega_data_idx in tqdm(self.omega_data_ids, total=self.n_elements):
+        for omega_data_idx in tqdm(
+            self.get_omega_data_ids(n_limit=n_limit),
+            total=n_limit if n_limit else self.n_elements,
+        ):
             try:
                 omega_data_item = self.create_stac_item(omega_data_idx)
                 collection.add_item(omega_data_item)
@@ -497,6 +514,7 @@ def create_stac_item(self, orbit_cube_idx: str, **kwargs) -> pystac.Item:
                     description="PNG thumbnail preview for visualizations",
                 )
                 pystac_item.add_asset("thumbnail", thumbn_asset)
+                self.log.debug(f"Added {thumbn_asset} to item.")
             except OSError as ose:
                 self.log.error(f"[{ose.__class__.__name__}] {ose}")
                 self.log.error(
@@ -510,6 +528,16 @@ def create_stac_item(self, orbit_cube_idx: str, **kwargs) -> pystac.Item:
                 self.log.error(f"[{e.__class__.__name__}] {e}")
             finally:
                 nc_data.close()
+        else:
+            # Thumbnail
+            thumbn_asset = pystac.Asset(
+                href=thumbnail_location.as_posix(),
+                media_type=pystac.MediaType.PNG,
+                roles=["thumbnail"],
+                description="PNG thumbnail preview for visualizations",
+            )
+            pystac_item.add_asset("thumbnail", thumbn_asset)
+            self.log.debug(f"Added {thumbn_asset} to item.")
 
         # extensions
         pystac_item = cast(pystac.Item, apply_ssys(pystac_item))
@@ -692,6 +720,8 @@ def retrieve_nc_info_from_saved_state(self, orbit_cube_idx: str) -> dict[str, An
 
         return nc_info
 
+    # TODO: revise startegy when this function is called
+    # You need to pick specific reflectance data for that
     def make_thumbnail(
         self,
         orbit_cube_idx: str,

src/psup_stac_converter/omega/c_channel_proj.py

@@ -31,18 +31,9 @@ def __init__(
             publications=omega_c_channel,
             log=log,
         )
-        self.sav_metadata_folder = psup_io_handler.output_folder / "l3_sav"
-        self.nc_metadata_folder = psup_io_handler.output_folder / "l3_nc"
-        self.log.debug(f".sav metadata folder: {self.sav_metadata_folder}")
-        self.log.debug(f".nc metadata folder: {self.nc_metadata_folder}")
-        if not self.sav_metadata_folder.exists():
-            self.sav_metadata_folder.mkdir()
 
-        if not self.nc_metadata_folder.exists():
-            self.nc_metadata_folder.mkdir()
-
-    def create_collection(self) -> pystac.Collection:
-        collection = super().create_collection()
+    def create_collection(self, n_limit: int | None = None) -> pystac.Collection:
+        collection = super().create_collection(n_limit=n_limit)
 
         # Only the C band is needed
         collection = cast(
@@ -105,7 +96,7 @@ def extract_sav_metadata(self, orbit_cube_idx: str, **kwargs) -> dict[str, Any]:
         return {}
 
     def create_stac_item(self, orbit_cube_idx) -> pystac.Item:
-        text_data: OmegaDataTextItem = cast(
+        text_data = cast(
             OmegaDataTextItem, self.open_file(orbit_cube_idx, "txt", on_disk=True)
         )
         footprint = json.loads(to_geojson(text_data.bbox))