Improve legacy gridspec (#916)

Author: sandorkertesz

pyproject.toml

@@ -28,8 +28,9 @@ dependencies = [
   "cfgrib>=0.9.10.1",
   "dask",
   "deprecation",
-  "earthkit-utils>=0.2",
+  "earthkit-utils>0.2.1",
   "eccodes>=1.7",
+  "eckit==2.0.1.13",
   "entrypoints",
   "filelock",
   "jinja2",
@@ -65,7 +66,6 @@ optional-dependencies.docs = [
 ]
 optional-dependencies.ecmwf-opendata = [ "ecmwf-opendata>=0.3.3" ]
 optional-dependencies.fdb = [ "pyfdb>=0.1" ]
-optional-dependencies.geo = [ "earthkit-geo>=0.2" ]
 optional-dependencies.geopandas = [ "geopandas" ]
 optional-dependencies.geotiff = [ "pyproj", "rasterio", "rioxarray" ]
 optional-dependencies.gribjump = [ "pyfdb>=0.1", "pygribjump" ]

src/earthkit/data/conf/gridspec.yaml

@@ -83,13 +83,6 @@ types:
       - dx
       - dy
     - *area
-  reduced_ll:
-    grid_type: reduced_ll
-    spec:
-    - *shared_keys
-    - grid:
-      - dy
-    - *area
   regular_gg:
     grid_type: regular_gg
     rotated_type: rotated_gg
@@ -111,56 +104,6 @@ types:
       octahedral:
         0: N
         1: O
-  # mercator:
-  #   grid_type: mercator
-  #   spec:
-  #   - *shared_keys
-  #   - grid:
-  #     - dx_in_metres
-  #     - dy_in_metres
-  #   - *area
-  #   - nx
-  #   - ny
-  #   - lad
-  #   - orientation
-  # polar_stereographic:
-  #   grid_type: polar_stereographic
-  #   spec:
-  #   - *shared_keys
-  #   - grid:
-  #     - dx_in_metres
-  #     - dy_in_metres
-  #   - *first_point
-  #   - nx
-  #   - ny
-  #   - lad
-  #   - orientation
-  # lambert:
-  #   grid_type: lambert
-  #   spec:
-  #   - *shared_keys
-  #   - grid:
-  #     - dx_in_metres
-  #     - dy_in_metres
-  #   - *first_point
-  #   - nx
-  #   - ny
-  #   - lad
-  #   - lov
-  #   - latin_1
-  #   - latin_2
-  # lambert_azimuthal_equal_area:
-  #   grid_type: lambert_azimuthal_equal_area
-  #   spec:
-  #   - *shared_keys
-  #   - grid:
-  #     - dx_in_metres
-  #     - dy_in_metres
-  #   - *first_point
-  #   - nx
-  #   - ny
-  #   - standard_parallel
-  #   - central_longitude
   healpix:
     grid_type: healpix
     spec:

src/earthkit/data/conf/gridspec_schema.json

@@ -216,28 +216,6 @@
                 ]
             }
         },
-        {
-            "if": {
-                "properties": {
-                    "type": {
-                        "const": "reduced_ll"
-                    }
-                },
-                "required": [
-                    "type"
-                ]
-            },
-            "then": {
-                "required": [
-                    "grid"
-                ],
-                "properties": {
-                    "grid": {
-                        "type": "number"
-                    }
-                }
-            }
-        },
         {
             "if": {
                 "properties": {
@@ -287,113 +265,6 @@
                 }
             }
         },
-        {
-            "if": {
-                "properties": {
-                    "type": {
-                        "const": "mercator"
-                    }
-                },
-                "required": [
-                    "type"
-                ]
-            },
-            "then": {
-                "required": [
-                    "grid",
-                    "nx",
-                    "ny",
-                    "lad"
-                ],
-                "properties": {
-                    "grid": {
-                        "type": "array"
-                    }
-                }
-            }
-        },
-        {
-            "if": {
-                "properties": {
-                    "type": {
-                        "const": "polar_stereographic"
-                    }
-                },
-                "required": [
-                    "type"
-                ]
-            },
-            "then": {
-                "required": [
-                    "grid",
-                    "first_point",
-                    "nx",
-                    "ny",
-                    "lad"
-                ],
-                "properties": {
-                    "grid": {
-                        "type": "array"
-                    }
-                }
-            }
-        },
-        {
-            "if": {
-                "properties": {
-                    "type": {
-                        "const": "lambert"
-                    }
-                },
-                "required": [
-                    "type"
-                ]
-            },
-            "then": {
-                "required": [
-                    "grid",
-                    "first_point",
-                    "nx",
-                    "ny",
-                    "lad",
-                    "lov",
-                    "latin_1",
-                    "latin_2"
-                ],
-                "properties": {
-                    "grid": {
-                        "type": "array"
-                    }
-                }
-            }
-        },
-        {
-            "if": {
-                "properties": {
-                    "type": {
-                        "const": "lambert_azimuthal_equal_area"
-                    }
-                },
-                "required": [
-                    "type"
-                ]
-            },
-            "then": {
-                "required": [
-                    "grid",
-                    "first_point",
-                    "nx",
-                    "ny",
-                    "standard_parallel",
-                    "central_longitude"
-                ],
-                "properties": {
-                    "grid": {
-                        "type": "array"
-                    }
-                }
-            }
-        },
         {
             "if": {
                 "properties": {

src/earthkit/data/core/config.py

@@ -189,7 +189,7 @@ def validate(self, name, value):
         none_ok=True,
     ),
     "maximum-cache-disk-usage": _(
-        "95%",
+        "98%",
         """Disk usage threshold after which earthkit-data expires older cached
         entries (% of the full disk capacity). Can be set to None.
         See :ref:`caching` for more information.""",

src/earthkit/data/field/grib/legacy_grid_spec.py

@@ -146,6 +146,27 @@ def make(self):
             for k in self.conf["rotation_keys"]:
                 d.pop(k, None)
 
+            # only canonical scanning mode is supported
+        for k in list(d.keys()):
+            if k == "j_points_consecutive":
+                if d[k] != 0:
+                    return dict()
+                else:
+                    d.pop(k)
+            if k == "i_scans_negatively":
+                if d[k] != 0:
+                    return dict()
+                else:
+                    d.pop(k)
+            if k == "j_scans_positively":
+                if d[k] != 0:
+                    return dict()
+                else:
+                    d.pop(k)
+
+            if k == "type":
+                d.pop(k, None)
+
         # CONF.validate(d)
         return d
 

tests/grib/test_grib_gridspec.py

@@ -25,12 +25,17 @@
 SUPPORTED_GRID_TYPES = [
     "sh",
     "regular_ll",
-    "reduced_ll",
     "regular_gg",
     "reduced_gg",
     "healpix",
 ]
-UNSUPPORTED_GRID_TYPES = ["rotated_ll", "rotated_gg", "reduced_rotated_gg"]
+
+UNSUPPORTED_GRID_TYPES = [
+    "rotated_ll",
+    "rotated_gg",
+    "reduced_rotated_gg",
+    "reduced_ll",
+]
 
 
 def _make_gridspec_list(grid_types):
@@ -82,6 +87,10 @@ def test_grib_gridspec_from_metadata_valid(metadata, ref, name):
     ]:
         pytest.skip()
 
+    ref = ref.copy()
+    for k in ["type", "i_scans_negatively", "j_points_consecutive", "j_scans_positively"]:
+        ref.pop(k, None)
+
     gridspec = make_legacy_gridspec(metadata)
     assert dict(gridspec) == ref, name
 
@@ -116,12 +125,12 @@ def test_grib_gridspec_from_file():
     )
 
     ref = {
-        "type": "regular_ll",
+        # "type": "regular_ll",
         "grid": [5.0, 5.0],
         "area": [75.0, -60.0, 10.0, 40.0],
-        "j_points_consecutive": 0,
-        "i_scans_negatively": 0,
-        "j_scans_positively": 0,
+        # "j_points_consecutive": 0,
+        # "i_scans_negatively": 0,
+        # "j_scans_positively": 0,
     }
     gs = ds[0].geography.grid_spec()
     assert isinstance(gs, dict), type(gs)

tests/xr_engine/test_xr_engine_grid.py

@@ -101,13 +101,13 @@ def test_xr_engine_gridspec(allow_holes, lazy_load):
     r = ds.to_xarray(profile="mars", allow_holes=allow_holes, lazy_load=lazy_load)
 
     gs = r["r"].earthkit.grid_spec
-    assert gs["type"] == "reduced_gg"
+    # assert gs["type"] == "reduced_gg"
     assert gs["grid"] == "O32"
 
     gs = r["t"].earthkit.grid_spec
-    assert gs["type"] == "reduced_gg"
+    # assert gs["type"] == "reduced_gg"
     assert gs["grid"] == "O32"
 
     gs = r.earthkit.grid_spec
-    assert gs["type"] == "reduced_gg"
+    #  assert gs["type"] == "reduced_gg"
     assert gs["grid"] == "O32"