Load IBTrACs file from archive in case (#956)

* hazard.test: fall back for missing IBTraCs file

* typo

Author: emanuel-schmid

climada/hazard/test/__init__.py

@@ -18,3 +18,29 @@
 
 init test
 """
+
+import shutil
+
+from climada.util.constants import SYSTEM_DIR
+from climada.util.api_client import Client
+from climada.util.files_handler import download_ftp
+from climada.hazard.tc_tracks import IBTRACS_FILE, IBTRACS_URL
+
+
+def download_ibtracs():
+    """This makes sure a IBTrACS.ALL.v04r00.nc file is present in SYSTEM_DIR
+    First, downloading from the original sources is attempted. If that fails an old version
+    is downloaded from the CLIMADA Data API
+    """
+    if SYSTEM_DIR.joinpath(IBTRACS_FILE).is_file():
+        return  # Nothing to do
+
+    try:
+        download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE)
+        shutil.move(IBTRACS_FILE, SYSTEM_DIR)
+
+    except ValueError:  # plan b: download an old version of that file from the climada api
+        client = Client()
+        dsinfo = client.get_dataset_info(name="IBTrACS", version="v04r00", status="external")
+        [fileinfo] = [fi for fi in dsinfo.files if fi.file_name == 'IBTrACS.ALL.v04r00.nc']
+        client._download_file(local_path=SYSTEM_DIR, fileinfo=fileinfo)

climada/hazard/test/test_tc_tracks.py

@@ -19,7 +19,9 @@
 Test tc_tracks module.
 """
 
+from datetime import datetime as dt
 import unittest
+
 import xarray as xr
 import numpy as np
 import pandas as pd
@@ -32,7 +34,8 @@
 from climada.util.constants import TC_ANDREW_FL
 import climada.util.coordinates as u_coord
 from climada.entity import Exposures
-from datetime import datetime as dt
+from climada.hazard.test import download_ibtracs
+
 
 DATA_DIR = CONFIG.hazard.test_data.dir()
 TEST_TRACK = DATA_DIR.joinpath("trac_brb_test.csv")
@@ -50,6 +53,10 @@
 class TestIbtracs(unittest.TestCase):
     """Test reading and model of TC from IBTrACS files"""
 
+    @classmethod
+    def setUpClass(cls):
+        download_ibtracs()
+
     def test_raw_ibtracs_empty_pass(self):
         """Test reading empty TC from IBTrACS files"""
         tc_track = tc.TCTracks.from_ibtracs_netcdf(
@@ -290,6 +297,11 @@ def test_ibtracs_additional_variables(self):
 
 class TestIO(unittest.TestCase):
     """Test reading of tracks from files of different formats"""
+
+    @classmethod
+    def setUpClass(cls):
+        download_ibtracs()
+
     def test_netcdf_io(self):
         """Test writting and reading netcdf4 TCTracks instances"""
         path = DATA_DIR.joinpath("tc_tracks_nc")
@@ -556,6 +568,10 @@ def test_to_geodataframe_line(self):
 class TestFuncs(unittest.TestCase):
     """Test functions over TC tracks"""
 
+    @classmethod
+    def setUpClass(cls):
+        download_ibtracs()
+
     def test_get_track_pass(self):
         """Test get_track."""
         tc_track = tc.TCTracks.from_processed_ibtracs_csv(TEST_TRACK_SHORT)

climada/hazard/test/test_tc_tracks_synth.py

@@ -30,6 +30,8 @@
 import climada.hazard.tc_tracks_synth as tc_synth
 import climada.util.coordinates
 from climada.util.constants import TC_ANDREW_FL
+from climada.hazard.test import download_ibtracs
+
 
 DATA_DIR = Path(__file__).parent.joinpath('data')
 TEST_TRACK = DATA_DIR.joinpath("trac_brb_test.csv")
@@ -39,7 +41,13 @@
 TEST_TRACK_DECAY_PENV_GT_PCEN = DATA_DIR.joinpath('1988021S12080_gen2.nc')
 TEST_TRACK_DECAY_PENV_GT_PCEN_HIST = DATA_DIR.joinpath('1988021S12080.nc')
 
+
 class TestDecay(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        download_ibtracs()
+
     def test_apply_decay_no_landfall_pass(self):
         """Test _apply_land_decay with no historical tracks with landfall"""
         tc_track = tc.TCTracks.from_processed_ibtracs_csv(TEST_TRACK_SHORT)
@@ -469,6 +477,11 @@ def test_decay_penv_gt_pcen(self):
         self.assertTrue(np.all(np.diff(p_env_lf - p_synth_lf) <= 0))
 
 class TestSynth(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        download_ibtracs()
+
     def test_angle_funs_pass(self):
         """Test functions used by random walk code"""
         self.assertAlmostEqual(tc_synth._get_bearing_angle(np.array([15, 20]),

climada/hazard/test/test_trop_cyclone.py

@@ -34,18 +34,24 @@
 from climada.hazard.trop_cyclone.trop_cyclone import (
     TropCyclone, )
 from climada.hazard.centroids.centr import Centroids
+from climada.hazard.test import download_ibtracs
+
 
 DATA_DIR = Path(hazard_test.__file__).parent.joinpath('data')
 
 TEST_TRACK = DATA_DIR.joinpath("trac_brb_test.csv")
 TEST_TRACK_SHORT = DATA_DIR.joinpath("trac_short_test.csv")
 
-
 CENTR_TEST_BRB = Centroids.from_hdf5(get_test_file('centr_test_brb', file_format='hdf5'))
 
 
 class TestReader(unittest.TestCase):
     """Test loading funcions from the TropCyclone class"""
+
+    @classmethod
+    def setUpClass(cls):
+        download_ibtracs()
+
     def test_memory_limit(self):
         """Test from_tracks when memory is (very) limited"""
         tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK)