Merge pull request #150 from c-hydro/dev

Dev

Author: ltrotter

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "door"
-version = "2.3.6"
+version = "2.3.9"
 description = "A package for operational retrieval of raster data from different sources"
 authors = [
     { name = "Andrea Libertino", email = "andrea.libertino@cimafoundation.org" },

src/door/base_downloaders.py

@@ -380,7 +380,7 @@ def download(self, destination: str, min_size: float = None, missing_action: str
 
         url = self.format_url(**kwargs)
         try:
-            r = requests.get(url, kwargs["auth"])
+            r = requests.get(url, auth = kwargs["auth"])
             if r.status_code != 200:
                 raise FileNotFoundError(r.text)
 

src/door/data_sources/hsaf/hsaf_downloader.py

@@ -11,7 +11,6 @@
 from ...utils.io import decompress_bz2
 
 import datetime as dt
-import requests
 
 from d3tools.spatial import BoundingBox, crop_to_bb
 from d3tools import timestepping as ts
@@ -33,15 +32,16 @@ class HSAFDownloader(FTPDownloader):
     }
 
     available_products: dict = {
-        "HSAF-h141": {
+        "hsaf-h141": {
             "ts_per_year": 365,
             "url" : "/products/h141/h141/netCDF4/{timestep.start:%Y}/h141_{timestep.start:%Y%m%d}00_R01.nc",
             "nodata" : -9999,
             "format" : 'nc'
         },
-        "HSAF-h14": {
+        "hsaf-h14": {
             "ts_per_year": 365,
-            "url" : "/hsaf_archive/h14/{timestep.start:%Y/%m/%d}/h14_{timestep.start:%Y%m%d}_0000.grib.bz2",
+            "url_alt" : "/hsaf_archive/h14/{timestep.start:%Y/%m/%d}/h14_{timestep.start:%Y%m%d}_0000.grib.bz2",
+            "url" : "/products/h14/h14_cur_mon_grib/h14_{timestep.start:%Y%m%d}_0000.grib.bz2",
             "nodata" : -9999,
             "format" : 'bz2'
         },
@@ -89,15 +89,6 @@ def __init__(self, product: str) -> None:
         self.credentials = get_credentials(env_variables=self.credential_env_vars, url = 'ftp://' + url_host, encode = False)
         username, password = self.credentials.split(':')
         super().__init__(url_host, protocol = 'ftp', user=username, password=password)
-        
-    def set_product(self, product: str) -> None:
-        self.product = product
-        if product not in self.available_products:
-            raise ValueError(f'Product {product} not available. Choose one of {self.available_products.keys()}')
-        self.ts_per_year = self.available_products[product]["ts_per_year"]
-        self.url_blank = self.available_products[product]["url"]
-        self.nodata = self.available_products[product]["nodata"]
-        self.format = self.available_products[product]["format"]
 
     def set_variables(self, variables: list) -> None:
         if self.custom_variables:
@@ -178,8 +169,19 @@ def _get_data_ts(self,
 
         # Download the data
         retries = self.retries
+
+        urls = [self.url.format(timestep=timestep)]
+        if hasattr(self, 'url_alt'): urls.append(self.url_alt.format(timestep=timestep))
+        # find the correct url
+        for url in urls:
+            if self.check_data(url, timestep=timestep):
+                break
+        else:
+            print(f"No data available for {timestep}")
+            return
+
         while True:
-            success = self.download(self.url_blank, tmp_file, timestep = timestep, auth = self.credentials, missing_action = 'ignore', min_size = 50000)
+            success = self.download(url, tmp_file, timestep = timestep, auth = self.credentials, missing_action = 'ignore', min_size = 50000)
             if success:
                 break
             elif retries <= 0:

src/door/data_sources/lsa_saf/lsasaf_downloader.py

@@ -18,6 +18,9 @@ class LSASAFDownloader(URLDownloader):
 
     single_temp_folder = False
 
+    retries = 3
+    retry_delay = 10  # seconds
+
     default_options = {
         "ts_per_year": 365,
         "variables" : None,  # all variables
@@ -115,24 +118,41 @@ def _get_data_ts(self,
                      tmp_path: str) -> Generator[tuple[xr.DataArray, dict], None, None]:
 
 
+        credentials = self.get_credentials()
         tmp_file_nc = f'temp_{self.product}{timestep.start:%Y%m%d}.nc'
 
         # check if the file is not already downloaded in the tmp_path
         tmp_destination = os.path.join(tmp_path, tmp_file_nc)
         this_filename = self.filename.format(time = timestep.start)
-        self.download(tmp_destination, min_size = 2000, missing_action = 'warning',
-                      time = timestep.start, product_name = self.product_name, satellite = self.satellite, filename = this_filename)
-
-        # open the file
-        raw_data = xr.open_dataset(tmp_destination, engine = 'netcdf4')
-        for var, varopts in self.variables.items():
-            vardata = raw_data[var].isel(time = 0, drop = True)  # remove the time dimension if present
-
-            # crop to the bounding box
-            vardata = crop_to_bb(vardata, space_bounds)
-
-            # set the metadata
-            vardata = vardata.rio.write_crs('EPSG:4326')
-            vardata = vardata.rio.set_spatial_dims(x_dim = 'lon', y_dim = 'lat')
 
-            yield vardata, {'variable' : var}
+        success = False
+        while not success and self.retries > 0:
+
+            success = self.download(tmp_destination, min_size = 2000, missing_action = 'warning', auth = tuple(credentials.split(':')),
+                        time = timestep.start, product_name = self.product_name, satellite = self.satellite, filename = this_filename)
+
+            if not success:
+                self.retries -= 1
+                if self.retries > 0:
+                    print(f'Retrying download in {self.retry_delay} seconds... ({self.retries} retries left)')
+                    import time
+                    time.sleep(self.retry_delay)
+                    continue
+                else:
+                    print('Max retries reached. Giving up.')
+                    break
+
+            # open the file
+            raw_data = xr.open_dataset(tmp_destination, engine = 'netcdf4')
+            raw_data.close()
+            for var, varopts in self.variables.items():
+                vardata = raw_data[var].isel(time = 0, drop = True)  # remove the time dimension if present
+
+                # crop to the bounding box
+                vardata = crop_to_bb(vardata, space_bounds)
+
+                # set the metadata
+                vardata = vardata.rio.write_crs('EPSG:4326')
+                vardata = vardata.rio.set_spatial_dims(x_dim = 'lon', y_dim = 'lat')
+
+                yield vardata, {'variable' : var}