All specific IO tests are done (#519)

* All specific IO tests are done

* Fixed last IO test?

Author: sidekock

environment_dev.yml

@@ -21,7 +21,6 @@ dependencies:
   - dask
   - pyfftw
   - h5py
-  - PyWavelets
   - pygrib
   - black
   - pytest-cov
@@ -32,3 +31,5 @@ dependencies:
   - pandas
   - rasterio
   - xarray
+  - geotiff
+  - cookiecutter

pysteps/io/importers.py

@@ -2023,7 +2023,6 @@ def _read_hdf5_cont(f, d):
                 }
 
         else:
-
             # Save h5py.Dataset by group name
             d[key] = np.array(value)
 

pysteps/pystepsrc

@@ -51,7 +51,7 @@
             "path_fmt": "%Y%m%d",
             "fn_pattern": "%Y%m%d%H%M_FINUTM",
             "fn_ext": "tif",
-            "importer": "geotiff",
+            "importer": "fmi_geotiff",
             "timestep": 5,
             "importer_kwargs": {}
         },

pysteps/tests/test_io_bom_rf3.py

@@ -5,83 +5,54 @@
 import pytest
 
 import pysteps
-from pysteps.tests.helpers import smart_assert
+from pysteps.tests.helpers import smart_assert, get_precipitation_fields
+
+precip_dataset = get_precipitation_fields(
+    num_prev_files=0,
+    num_next_files=0,
+    return_raw=True,
+    metadata=True,
+    source="bom",
+    log_transform=False,
+)
+
+precip_var = precip_dataset.attrs["precip_var"]
+precip_dataarray = precip_dataset[precip_var]
+
+
+def test_io_import_bom_shape():
+    """Test the shape of the read file."""
+    assert precip_dataarray.shape == (1, 512, 512)
 
-netCDF4 = pytest.importorskip("netCDF4")
 
 # Test import_bom_rf3 function
-expected_proj1 = (
+expected_proj = (
     "+proj=aea  +lon_0=144.752 +lat_0=-37.852 " "+lat_1=-18.000 +lat_2=-36.000"
 )
 
 test_metadata_bom = [
-    ("transform", None, None),
-    ("zerovalue", 0.0, 0.1),
-    ("projection", expected_proj1, None),
-    ("unit", "mm", None),
-    ("accutime", 6, 0.1),
-    ("x1", -128000.0, 0.1),
-    ("x2", 127500.0, 0.1),
-    ("y1", -127500.0, 0.1),
-    ("y2", 128000.0, 0.1),
-    ("xpixelsize", 500.0, 0.1),
-    ("ypixelsize", 500.0, 0.1),
-    ("cartesian_unit", "m", None),
-    ("yorigin", "upper", None),
-    ("institution", "Commonwealth of Australia, Bureau of Meteorology", None),
+    (precip_dataset.attrs["projection"], expected_proj, None),
+    (
+        precip_dataset.attrs["institution"],
+        "Commonwealth of Australia, Bureau of Meteorology",
+        None,
+    ),
+    (precip_dataset.x.isel(x=0).values, -127750.0, 1e-5),
+    (precip_dataset.y.isel(y=0).values, -127250.0, 1e-5),
+    (precip_dataset.x.isel(x=-1).values, 127250.0, 1e-5),
+    (precip_dataset.y.isel(y=-1).values, 127750.0, 1e-5),
+    (precip_dataset.x.attrs["stepsize"], 500.0, 1e-4),
+    (precip_dataset.y.attrs["stepsize"], 500.0, 1e-4),
+    (precip_dataset.x.attrs["units"], "m", None),
+    (precip_dataset.y.attrs["units"], "m", None),
+    (precip_dataarray.attrs["accutime"], 6, 1e-4),
+    (precip_dataarray.attrs["transform"], None, None),
+    (precip_dataarray.attrs["zerovalue"], 0.0, 1e-4),
+    (precip_dataarray.attrs["units"], "mm", None),
 ]
 
 
 @pytest.mark.parametrize("variable, expected, tolerance", test_metadata_bom)
 def test_io_import_bom_rf3_metadata(variable, expected, tolerance):
     """Test the importer Bom RF3."""
-    root_path = pysteps.rcparams.data_sources["bom"]["root_path"]
-    rel_path = os.path.join("prcp-cscn", "2", "2018", "06", "16")
-    filename = os.path.join(root_path, rel_path, "2_20180616_100000.prcp-cscn.nc")
-    precip, _, metadata = pysteps.io.import_bom_rf3(filename)
-    smart_assert(metadata[variable], expected, tolerance)
-    assert precip.shape == (512, 512)
-
-
-# Test _import_bom_rf3_data function
-def test_io_import_bom_rf3_shape():
-    """Test the importer Bom RF3."""
-    root_path = pysteps.rcparams.data_sources["bom"]["root_path"]
-    rel_path = os.path.join("prcp-cscn", "2", "2018", "06", "16")
-    filename = os.path.join(root_path, rel_path, "2_20180616_100000.prcp-cscn.nc")
-    precip, _ = pysteps.io.importers._import_bom_rf3_data(filename)
-    assert precip.shape == (512, 512)
-
-
-# Test _import_bom_rf3_geodata function
-expected_proj2 = (
-    "+proj=aea  +lon_0=144.752 +lat_0=-37.852 " "+lat_1=-18.000 +lat_2=-36.000"
-)
-# test_geodata: list of (variable,expected,tolerance) tuples
-test_geodata_bom = [
-    ("projection", expected_proj2, None),
-    ("unit", "mm", None),
-    ("accutime", 6, 0.1),
-    ("x1", -128000.0, 0.1),
-    ("x2", 127500.0, 0.1),
-    ("y1", -127500.0, 0.1),
-    ("y2", 128000.0, 0.1),
-    ("xpixelsize", 500.0, 0.1),
-    ("ypixelsize", 500.0, 0.1),
-    ("cartesian_unit", "m", None),
-    ("yorigin", "upper", None),
-    ("institution", "Commonwealth of Australia, Bureau of Meteorology", None),
-]
-
-
-@pytest.mark.parametrize("variable, expected, tolerance", test_geodata_bom)
-def test_io_import_bom_rf3_geodata(variable, expected, tolerance):
-    """Test the importer Bom RF3."""
-    root_path = pysteps.rcparams.data_sources["bom"]["root_path"]
-    rel_path = os.path.join("prcp-cscn", "2", "2018", "06", "16")
-    filename = os.path.join(root_path, rel_path, "2_20180616_100000.prcp-cscn.nc")
-    ds_rainfall = netCDF4.Dataset(filename)
-    geodata = pysteps.io.importers._import_bom_rf3_geodata(ds_rainfall)
-    smart_assert(geodata[variable], expected, tolerance)
-
-    ds_rainfall.close()
+    smart_assert(variable, expected, tolerance)

pysteps/tests/test_io_dwd_hdf5.py

@@ -12,7 +12,6 @@
     metadata=True,
     source="dwd",
     log_transform=False,
-    importer_kwargs=dict(qty="RATE"),
 )
 
 precip_var = precip_dataset.attrs["precip_var"]
@@ -54,7 +53,7 @@ def test_io_import_dwd_hdf5_ry_shape():
         None,
     ),
     (precip_dataarray.attrs["accutime"], 5.0, 1e-10),
-    (precip_dataset.time.attrs["stepsize"], 5.0, 1e-10),
+    (precip_dataset.time.attrs["stepsize"], 300, 1e-10),
     (precip_dataarray.attrs["units"], "mm/h", None),
     (precip_dataarray.attrs["transform"], None, None),
     (precip_dataarray.attrs["zerovalue"], 0.0, 1e-6),

pysteps/tests/test_io_knmi_hdf5.py

@@ -39,8 +39,8 @@ def test_io_import_knmi_hdf5_shape():
     (precip_dataset.x.attrs["stepsize"], 1000.0, 1e-10),
     (precip_dataset.y.attrs["stepsize"], 1000.0, 1e-10),
     (precip_dataarray.attrs["accutime"], 5.0, 1e-10),
-    (precip_dataset.time.attrs["stepsize"], 5.0, 1e-10),
-    (precip_dataarray.attrs["units"], "mm/h", None),
+    (precip_dataset.time.attrs["stepsize"], 300, 1e-10),
+    (precip_dataarray.attrs["units"], "mm", None),
     (precip_dataset.x.attrs["units"], "m", None),
     (precip_dataset.y.attrs["units"], "m", None),
     (

pysteps/tests/test_io_mch_gif.py

@@ -36,7 +36,7 @@ def test_io_import_mch_gif_shape():
     (precip_dataset.attrs["projection"], expected_proj, None),
     (precip_dataset.attrs["institution"], "MeteoSwiss", None),
     (precip_dataarray.attrs["accutime"], 5.0, 1e-10),
-    (precip_dataset.time.attrs["stepsize"], 5.0, 1e-10),
+    (precip_dataset.time.attrs["stepsize"], 300, 1e-10),
     (precip_dataarray.attrs["units"], "mm", None),
     (precip_dataarray.attrs["transform"], None, None),
     (precip_dataarray.attrs["zerovalue"], 0.0, 1e-6),

pysteps/tests/test_io_mrms_grib.py

@@ -1,83 +1,129 @@
 # -*- coding: utf-8 -*-
 
-import os
-
-import numpy as np
 import pytest
 from numpy.testing import assert_array_almost_equal
+from pysteps.tests.helpers import smart_assert, get_precipitation_fields
 
-import pysteps
+precip_dataset = get_precipitation_fields(
+    num_prev_files=0,
+    num_next_files=0,
+    return_raw=True,
+    metadata=True,
+    source="mrms",
+    log_transform=False,
+    window_size=1,
+)
 
-pytest.importorskip("pygrib")
+print(precip_dataset)
+precip_var = precip_dataset.attrs["precip_var"]
+precip_dataarray = precip_dataset[precip_var]
 
 
 def test_io_import_mrms_grib():
     """Test the importer for NSSL data."""
-
-    root_path = pysteps.rcparams.data_sources["mrms"]["root_path"]
-    filename = os.path.join(
-        root_path, "2019/06/10/", "PrecipRate_00.00_20190610-000000.grib2"
+    assert precip_dataarray.shape == (1, 3500, 7000)
+    assert precip_dataarray.dtype == "single"
+
+
+expected_proj = "+proj=longlat  +ellps=IAU76"
+
+# list of (variable,expected,tolerance) tuples
+test_attrs = [
+    (precip_dataset.attrs["projection"], expected_proj, None),
+    (
+        precip_dataset.attrs["institution"],
+        "NOAA National Severe Storms Laboratory",
+        None,
+    ),
+    (precip_dataarray.attrs["units"], "mm/h", None),
+    (precip_dataarray.attrs["transform"], None, None),
+    (precip_dataarray.attrs["zerovalue"], 0.0, 1e-6),
+    (precip_dataarray.attrs["threshold"], 0.1, 1e-10),
+    (precip_dataset.x.isel(x=0).values, -129.995, 1e-10),
+    (precip_dataset.y.isel(y=0).values, 20.005001, 1e-10),
+    (precip_dataset.x.isel(x=-1).values, -60.005002, 1e-10),
+    (precip_dataset.y.isel(y=-1).values, 54.995, 1e-10),
+    (precip_dataset.x.attrs["stepsize"], 0.01, 1e-4),
+    (precip_dataset.y.attrs["stepsize"], 0.01, 1e-4),
+    (precip_dataset.x.attrs["units"], "degrees", None),
+    (precip_dataset.y.attrs["units"], "degrees", None),
+]
+
+
+@pytest.mark.parametrize("variable, expected, tolerance", test_attrs)
+def test_io_import_mrms_grib_dataset_attrs(variable, expected, tolerance):
+    """Test the importer MRMS_GRIB."""
+    smart_assert(variable, expected, tolerance)
+
+
+def test_io_import_mrms_grib_dataset_extent():
+    """Test the importer MRMS_GRIB."""
+
+    precip_dataset_smaller = get_precipitation_fields(
+        num_prev_files=0,
+        num_next_files=0,
+        return_raw=True,
+        metadata=True,
+        source="mrms",
+        log_transform=False,
+        extent=(230, 300, 20, 55),
+        window_size=1,
     )
-    precip, _, metadata = pysteps.io.import_mrms_grib(filename, fillna=0, window_size=1)
-
-    assert precip.shape == (3500, 7000)
-    assert precip.dtype == "single"
-
-    expected_metadata = {
-        "institution": "NOAA National Severe Storms Laboratory",
-        "xpixelsize": 0.01,
-        "ypixelsize": 0.01,
-        "unit": "mm/h",
-        "transform": None,
-        "zerovalue": 0,
-        "projection": "+proj=longlat  +ellps=IAU76",
-        "yorigin": "upper",
-        "threshold": 0.1,
-        "x1": -129.99999999999997,
-        "x2": -60.00000199999991,
-        "y1": 20.000001,
-        "y2": 55.00000000000001,
-        "cartesian_unit": "degrees",
-    }
-
-    for key, value in expected_metadata.items():
-        if isinstance(value, float):
-            assert_array_almost_equal(metadata[key], expected_metadata[key])
-        else:
-            assert metadata[key] == expected_metadata[key]
-
-    x = np.arange(metadata["x1"], metadata["x2"], metadata["xpixelsize"])
-    y = np.arange(metadata["y1"], metadata["y2"], metadata["ypixelsize"])
-
-    assert y.size == precip.shape[0]
-    assert x.size == precip.shape[1]
-
-    # The full latitude range is (20.005, 54.995)
-    # The full longitude range is (230.005, 299.995)
-
-    # Test that if the bounding box is larger than the domain, all the points are returned.
-    precip2, _, _ = pysteps.io.import_mrms_grib(
-        filename, fillna=0, extent=(220, 300, 20, 55), window_size=1
-    )
-    assert precip2.shape == (3500, 7000)
-
-    assert_array_almost_equal(precip, precip2)
 
-    del precip2
+    precip_var_smaller = precip_dataset_smaller.attrs["precip_var"]
+    precip_dataarray_smaller = precip_dataset_smaller[precip_var_smaller]
+    smart_assert(precip_dataarray_smaller.shape, (1, 3500, 7000), None)
+    assert_array_almost_equal(precip_dataarray.values, precip_dataarray_smaller.values)
+
+    precip_dataset_even_smaller = get_precipitation_fields(
+        num_prev_files=0,
+        num_next_files=0,
+        return_raw=True,
+        metadata=True,
+        source="mrms",
+        log_transform=False,
+        extent=(250, 260, 30, 35),
+        window_size=1,
+    )
 
-    # Test that a portion of the domain is returned correctly
-    precip3, _, _ = pysteps.io.import_mrms_grib(
-        filename, fillna=0, extent=(250, 260, 30, 35), window_size=1
+    precip_var_even_smaller = precip_dataset_even_smaller.attrs["precip_var"]
+    precip_dataarray_even_smaller = precip_dataset_even_smaller[precip_var_even_smaller]
+    smart_assert(precip_dataarray_even_smaller.shape, (1, 500, 1000), None)
+    # XR: we had to change the selection of the original field since these is a flip happening in the way the data is read in.
+    # XR: We had two ways to solve this: precip_dataarray[:,::-1, :][:, 2000:2500, 2000:3000][:,::-1, :] or switch the 2000:2500 to
+    assert_array_almost_equal(
+        precip_dataarray.values[:, 1000:1500, 2000:3000],
+        precip_dataarray_even_smaller.values,
     )
 
-    assert precip3.shape == (500, 1000)
-    assert_array_almost_equal(precip3, precip[2000:2500, 2000:3000])
-    del precip3
+    precip_dataset_double = get_precipitation_fields(
+        num_prev_files=0,
+        num_next_files=0,
+        return_raw=True,
+        metadata=True,
+        source="mrms",
+        log_transform=False,
+        extent=(250, 260, 30, 35),
+        window_size=1,
+        dtype="double",
+    )
 
-    precip4, _, _ = pysteps.io.import_mrms_grib(filename, dtype="double", fillna=0)
-    assert precip4.dtype == "double"
-    del precip4
+    precip_var_double = precip_dataset_double.attrs["precip_var"]
+    precip_dataarray_double = precip_dataset_double[precip_var_double]
+    smart_assert(precip_dataarray_double.dtype, "double", None)
+
+    precip_dataset_single = get_precipitation_fields(
+        num_prev_files=0,
+        num_next_files=0,
+        return_raw=True,
+        metadata=True,
+        source="mrms",
+        log_transform=False,
+        extent=(250, 260, 30, 35),
+        window_size=1,
+        dtype="single",
+    )
 
-    precip5, _, _ = pysteps.io.import_mrms_grib(filename, dtype="single", fillna=0)
-    assert precip5.dtype == "single"
-    del precip5
+    precip_var_single = precip_dataset_single.attrs["precip_var"]
+    precip_dataarray_single = precip_dataset_single[precip_var_single]
+    smart_assert(precip_dataarray_single.dtype, "single", None)