fixing download issues

Author: s-boomi

src/psup_stac_converter/omega/_base.py

@@ -559,27 +559,29 @@ def create_stac_item(self, orbit_cube_idx: str, **kwargs) -> pystac.Item:
         self.log.debug("Applying DatacubeExtension")
         cubedata = self.retrieve_nc_info_from_saved_state(orbit_cube_idx=orbit_cube_idx)
         self.log.debug(f"Loading: {cubedata}")
-        dc_ext = DatacubeExtension.ext(pystac_item, add_if_missing=True)
-
-        # This operation prevents the key from finding itself attached to "Variables" and "Dimensions"
-        dc_dimensions: dict[str, Dimension] = {
-            k: Dimension.from_dict(v.to_dict()[k])
-            for k, v in cubedata["dimensions"].items()
-        }
-        dc_variables: dict[str, Variable] = {
-            k: Variable.from_dict(v.to_dict()[k])
-            for k, v in cubedata["variables"].items()
-        }
-
-        dc_ext.apply(dimensions=dc_dimensions, variables=dc_variables)
-
-        for extra_name, extra_value in cubedata["extras"].items():
-            pystac_item.assets["nc"].extra_fields[extra_name] = extra_value
+        if cubedata:
+            dc_ext = DatacubeExtension.ext(pystac_item, add_if_missing=True)
+
+            # This operation prevents the key from finding itself attached to "Variables" and "Dimensions"
+            dc_dimensions: dict[str, Dimension] = {
+                k: Dimension.from_dict(v.to_dict()[k])
+                for k, v in cubedata["dimensions"].items()
+            }
+            dc_variables: dict[str, Variable] = {
+                k: Variable.from_dict(v.to_dict()[k])
+                for k, v in cubedata["variables"].items()
+            }
+
+            dc_ext.apply(dimensions=dc_dimensions, variables=dc_variables)
+
+            for extra_name, extra_value in cubedata["extras"].items():
+                pystac_item.assets["nc"].extra_fields[extra_name] = extra_value
+        else:
+            self.log.warning(f"Cubedata for {orbit_cube_idx} appears to be empty.")
 
         # common metadata
         pystac_item.common_metadata.mission = "mex"
         pystac_item.common_metadata.instruments = ["omega"]
-
         self.log.debug(f"Created item from base method {pystac_item}")
 
         return pystac_item
@@ -725,6 +727,12 @@ def retrieve_nc_info_from_saved_state(self, orbit_cube_idx: str) -> dict[str, An
                 nc_info = json.load(nc_md)
                 nc_info = reformat_nc_info(nc_info)
                 self.log.debug(f"nc_info loaded with {nc_info}")
+            if not nc_info:
+                nc_md_state.unlink()
+                return self.retrieve_nc_info_from_saved_state(
+                    orbit_cube_idx=orbit_cube_idx
+                )
+
         else:
             self.log.debug(
                 f"{nc_md_state} not found. Creating it from # {orbit_cube_idx}"
@@ -736,14 +744,12 @@ def retrieve_nc_info_from_saved_state(self, orbit_cube_idx: str) -> dict[str, An
                 self.log.debug(f"{nc_md_state} with {nc_info} created!")
             except Exception as e:
                 self.log.warning(
-                    f"Couldn't save .sav information for # {orbit_cube_idx} because of the following: {e}"
+                    f"Couldn't save .nc information for # {orbit_cube_idx} because of the following: {e}"
                 )
                 nc_info = {}
 
         return nc_info
 
-    # TODO: revise startegy when this function is called
-    # You need to pick specific reflectance data for that
     def make_thumbnail(
         self,
         orbit_cube_idx: str,

src/psup_stac_converter/omega/c_channel_proj.py

@@ -97,30 +97,9 @@ def extract_sav_metadata(self, orbit_cube_idx: str, **kwargs) -> dict[str, Any]:
 
         return {}
 
-    def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
-        text_data = cast(
-            OmegaDataTextItem, self.open_file(orbit_cube_idx, "txt", on_disk=True)
-        )
-        # TODO: need some contour function for footprint
-        footprint = json.loads(to_geojson(self.get_contour_data(orbit_cube_idx)))
-        bbox = bounds(text_data.bbox).tolist()
-
-        pystac_item = super().create_stac_item(
-            orbit_cube_idx,
-            timestamp=text_data.start_time,
-            start_datetime=text_data.start_time,
-            end_datetime=text_data.stop_time,
-            footprint=footprint,
-            bbox=bbox,
-            item_properties={
-                "solar_longitude": text_data.solar_longitude,
-                "orbit_number": text_data.orbit_number,
-                "cube_number": text_data.cube_number,
-                # Would fit more in processing extension
-                "data_quality_id": text_data.data_quality_id,
-            },
-        )
-
+    def retrieve_sav_info_from_saved_state(
+        self, orbit_cube_idx: str, **kwargs
+    ) -> dict[str, Any]:
         sav_md_state = self.sav_metadata_folder / f"sav_{orbit_cube_idx}.json"
         if sav_md_state.exists():
             self.log.debug(f"{sav_md_state} found! Opening...")
@@ -131,15 +110,18 @@ def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
             ) as sav_md:
                 sav_info = json.load(sav_md)
                 self.log.debug(f"sav_info loaded with {sav_info}")
+            if not sav_info:
+                self.log.warning(
+                    f"Cube {orbit_cube_idx} happens to not have info. Redownloading..."
+                )
+                sav_md_state.unlink()
+                return self.retrieve_sav_info_from_saved_state(orbit_cube_idx, **kwargs)
         else:
             self.log.debug(
                 f"{sav_md_state} Not found. Creating from # {orbit_cube_idx}"
             )
             try:
-                sav_info = self.extract_sav_metadata(
-                    orbit_cube_idx,
-                    sav_size=pystac_item.assets["sav"].extra_fields["size"],
-                )
+                sav_info = self.extract_sav_metadata(orbit_cube_idx, **kwargs)
                 with open(sav_md_state, "w", encoding="utf-8") as sav_md:
                     json.dump(sav_info, sav_md)
                     self.log.debug(f"{sav_md_state} with {sav_info} created!")
@@ -148,6 +130,35 @@ def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
                     f"Couldn't save .sav information for # {orbit_cube_idx} because of the following: {e}"
                 )
                 sav_info = {}
+        return sav_info
+
+    def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
+        text_data = cast(
+            OmegaDataTextItem, self.open_file(orbit_cube_idx, "txt", on_disk=True)
+        )
+
+        footprint = json.loads(to_geojson(self.get_contour_data(orbit_cube_idx)))
+        bbox = bounds(text_data.bbox).tolist()
+
+        pystac_item = super().create_stac_item(
+            orbit_cube_idx,
+            timestamp=text_data.start_time,
+            start_datetime=text_data.start_time,
+            end_datetime=text_data.stop_time,
+            footprint=footprint,
+            bbox=bbox,
+            item_properties={
+                "solar_longitude": text_data.solar_longitude,
+                "orbit_number": text_data.orbit_number,
+                "cube_number": text_data.cube_number,
+                # Would fit more in processing extension
+                "data_quality_id": text_data.data_quality_id,
+            },
+        )
+
+        sav_info = self.retrieve_sav_info_from_saved_state(
+            orbit_cube_idx, sav_size=pystac_item.assets["sav"].extra_fields["size"]
+        )
 
         pystac_item.assets["sav"].extra_fields["map_dimensions"] = sav_info.get("dims")
         self.log.debug(f"Item created: {pystac_item.to_dict()}")

src/psup_stac_converter/omega/data_cubes.py

@@ -187,13 +187,21 @@ def find_extra_nc_data(self, nc_data: xr.Dataset) -> dict[str, Any]:
 
         return extras
 
-    def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
+    def retrieve_sav_info_from_saved_state(
+        self, orbit_cube_idx: str, **kwargs
+    ) -> dict[str, Any]:
         sav_md_state = self.sav_metadata_folder / f"sav_{orbit_cube_idx}.json"
         self.log.debug(f"Opening {sav_md_state}")
         if sav_md_state.exists():
             with open(sav_md_state, "r", encoding="utf-8") as sav_md:
                 sav_info = json.load(sav_md)
                 self.log.debug(f"sav_info loaded with {sav_info}")
+            if not sav_info:
+                self.log.warning(
+                    f"Cube {orbit_cube_idx} happens to not have info. Redownloading..."
+                )
+                sav_md_state.unlink()
+                return self.retrieve_sav_info_from_saved_state(orbit_cube_idx, **kwargs)
         else:
             self.log.debug(
                 f"{sav_md_state} not found. Creating it from # {orbit_cube_idx}"
@@ -202,12 +210,18 @@ def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
                 sav_info = self.extract_sav_info(orbit_cube_idx)
                 with open(sav_md_state, "w", encoding="utf-8") as sav_md:
                     json.dump(sav_info, sav_md)
-                self.log.debug(f"{sav_md_state} with {sav_info} created!")
+                    self.log.debug(f"{sav_md_state} with {sav_info} created!")
             except Exception as e:
                 self.log.warning(
                     f"Couldn't save .sav information for # {orbit_cube_idx} because of the following: {e}"
                 )
                 sav_info = {}
+        return sav_info
+
+    def create_stac_item(self, orbit_cube_idx: str) -> pystac.Item:
+        # TODO: regroup that in a single function
+
+        sav_info = self.retrieve_sav_info_from_saved_state(orbit_cube_idx)
 
         # This one is given by the data description
         default_end_datetime = dt.datetime(2016, 4, 11, 0, 0)

src/psup_stac_converter/utils/downloader.py

@@ -287,7 +287,7 @@ def open_resource(self, file_href: str):
                 log.debug(f"A total of {total_bytes} bytes will be downloaded")
 
                 with (
-                    tempfile.NamedTemporaryFile() as tmp_f,
+                    tempfile.NamedTemporaryFile(delete_on_close=False) as tmp_f,
                     tqdm(
                         total=total_bytes,
                         unit="B",
@@ -296,17 +296,21 @@ def open_resource(self, file_href: str):
                         unit_divisor=1024,
                     ) as pbar,
                 ):
-                    num_bytes_downloaded = response.num_bytes_downloaded
-                    for chunk in response.iter_bytes():
-                        tmp_f.write(chunk)
-                        pbar.update(
-                            response.num_bytes_downloaded - num_bytes_downloaded
-                        )
+                    try:
                         num_bytes_downloaded = response.num_bytes_downloaded
-                    log.info(f"Saved {file_href} temporarily")
-                    yield tmp_f
-                    log.debug(f"{tmp_f.name} ready to use")
-                log.debug(f"{file_href} disposed")
+                        for chunk in response.iter_bytes():
+                            tmp_f.write(chunk)
+                            pbar.update(
+                                response.num_bytes_downloaded - num_bytes_downloaded
+                            )
+                            num_bytes_downloaded = response.num_bytes_downloaded
+                        tmp_f.close()
+                        log.info(f"Saved {file_href} temporarily on {tmp_f.name}")
+                        yield tmp_f
+                        log.debug(f"{tmp_f.name} ready to use")
+                    finally:
+                        Path(tmp_f.name).unlink()
+                        log.debug(f"{file_href} disposed ({tmp_f.name})")
 
     def save_slice_on_disk(
         self,