Merge pull request #289 from NREL/bnb/lr_features_ordering

- invert_uv=True as default for forward pass strategy.
- fix ordering of lr_features in dual sampler.

Author: bnb32

pixi.lock

@@ -58,7 +58,6 @@ tensorflow = {version = "~=2.15.0", channel = "conda-forge"}
 
 [project.optional-dependencies]
 dev = [
-  "build>=0.5",
   "flake8",
   "pre-commit",
   "pylint",
@@ -74,9 +73,9 @@ test = [
   "pytest-env"
 ]
 build = [
-  "build>=1.2.2,<2",
+  "build>=0.6",
   "pkginfo>=1.10.0,<2",
-  "twine>=6.1.0,<7",
+  "twine>=5.0",
 ]
 
 [project.scripts]
@@ -298,7 +297,6 @@ numpy = "~=1.7"
 pandas = ">=2.0"
 scipy = ">=1.0.0"
 xarray = ">=2023.0"
-ipython = ">=9.7.0,<10"
 
 [tool.pixi.pypi-dependencies]
 NREL-sup3r = { path = ".", editable = true }
@@ -309,14 +307,21 @@ NREL-farms = { version = ">=1.0.4" }
 
 [tool.pixi.environments]
 default = { solve-group = "default" }
-dev = { features = ["dev", "doc", "test"], solve-group = "default" }
+dev = { features = ["dev", "doc", "test", "viz", "build"], solve-group = "default" }
 doc = { features = ["doc"], solve-group = "default" }
 test = { features = ["test"], solve-group = "default" }
 viz = { features = ["viz"], solve-group = "default" }
+build = { features = ["build"], solve-group = "default" }
 
-[tool.pixi.tasks]
+[tool.pixi.feature.test.tasks]
 test = "pytest --pdb --durations=10 tests"
 
+[tool.pixi.feature.build.tasks]
+clean-readme = { cmd = "python sup3r/utilities/_clean_readme.py README.rst" }
+build-wheels = { cmd = "uv build --sdist --wheel --out-dir dist/ .", depends-on = ["clean-readme"] }
+check-wheels = { cmd = ["twine", "check", "dist/*"], depends-on = ["build-wheels"] }
+upload-wheels = { cmd = ["twine", "upload", "dist/*"], depends-on = ["check-wheels"] }
+
 [tool.pixi.feature.doc.dependencies]
 sphinx = ">=8.1.3,<9"
 sphinx-book-theme = ">=1.1.3,<2"
@@ -327,8 +332,6 @@ pytest = ">=5.2"
 pytest-cov = ">=5.0.0"
 
 [tool.pixi.feature.dev.dependencies]
-build = ">=0.6"
-twine = ">=5.0"
 ruff = ">=0.4"
 ipython = ">=8.0"
 pytest-xdist = ">=3.0"
@@ -337,6 +340,10 @@ pytest-xdist = ">=3.0"
 jupyter = ">=1.0"
 hvplot = ">=0.10"
 
+[tool.pixi.feature.build.dependencies]
+build = ">=0.6"
+twine = ">=5.0"
+
 [tool.pytest_env]
 CUDA_VISIBLE_DEVICES = "-1"
 TF_ENABLE_ONEDNN_OPTS = "0"

pyproject.toml

@@ -535,6 +535,7 @@ def _run_parallel(cls, strategy, node_index):
                         model_class=strategy.model_class,
                         allowed_const=strategy.allowed_const,
                         output_workers=strategy.output_workers,
+                        invert_uv=strategy.invert_uv,
                         meta=fwp.meta,
                     )
                     futures[fut] = {
@@ -585,7 +586,7 @@ def run_chunk(
         model_kwargs,
         model_class,
         allowed_const,
-        invert_uv=None,
+        invert_uv=False,
         meta=None,
         nn_fill=True,
         output_workers=None,
@@ -613,8 +614,8 @@ def run_chunk(
             information on this argument.
         invert_uv : bool
             Whether to convert uv to windspeed and winddirection for writing
-            output. This defaults to True for H5 output and False for NETCDF
-            output.
+            output. When this method is called during a pipeline forward pass
+            the value is taken from the strategy.invert_uv attribute.
         nn_fill : bool
             Whether to fill data outside of limits with nearest neighbour or
             cap to limits.
@@ -637,7 +638,7 @@ def run_chunk(
         model = get_model(model_class, model_kwargs)
 
         mask = np.isnan(chunk.input_data).any(axis=(0, 1, 2))
-        feats = np.array(model.lr_features[:len(mask)])[mask]
+        feats = np.array(model.lr_features[: len(mask)])[mask]
         if np.any(mask):
             msg = f'Input data for {feats} contains NaN values!'
             logger.error(msg)

sup3r/pipeline/forward_pass.py

@@ -171,10 +171,9 @@ class ForwardPassStrategy:
         chunks and overwrite any pre-existing outputs (False).
     output_workers : int | None
         Max number of workers to use for writing forward pass output.
-    invert_uv : bool | None
+    invert_uv : bool
         Whether to convert u and v wind components to windspeed and direction
-        for writing to output. This defaults to True for H5 output and False
-        for NETCDF output.
+        for writing to output. This defaults to True.
     nn_fill : bool
         Whether to fill data outside of accepted limits (e.g. relative
         humidity 0-100) with nearest neighbour or cap to limits.
@@ -221,7 +220,7 @@ class ForwardPassStrategy:
     allowed_const: Optional[Union[list, bool]] = None
     incremental: bool = True
     output_workers: int = 1
-    invert_uv: Optional[bool] = None
+    invert_uv: Optional[bool] = True
     nn_fill: bool = True
     pass_workers: int = 1
     max_nodes: int = 1
@@ -305,7 +304,7 @@ def meta(self):
         return meta_data
 
     def get_time_slices(self):
-        """Get the time slice for initializaing the input handler and the
+        """Get the time slice for initializing the input handler and the
         time slice applied to the data given by the input handler to get the
         actual requested time period. These are different because we want the
         data stored by the input handler to have extra time steps at the start

sup3r/pipeline/strategy.py

@@ -305,7 +305,7 @@ def _transform_output(
         data,
         features,
         lat_lon,
-        invert_uv=None,
+        invert_uv=False,
         nn_fill=False,
         max_workers=None,
     ):
@@ -322,7 +322,7 @@ def _transform_output(
             Array of high res lat/lon for output data.
             (spatial_1, spatial_2, 2)
             Last dimension has ordering (lat, lon)
-        invert_uv : bool | None
+        invert_uv : bool
             Whether to convert u and v wind components to windspeed and
             direction
         nn_fill : bool
@@ -554,7 +554,7 @@ def _write_output(
         times,
         out_file,
         meta_data,
-        invert_uv=True,
+        invert_uv=False,
         nn_fill=False,
         max_workers=None,
         gids=None,
@@ -570,7 +570,7 @@ def write_output(
         low_res_times,
         out_file,
         meta_data=None,
-        invert_uv=None,
+        invert_uv=False,
         nn_fill=False,
         max_workers=None,
         gids=None,
@@ -593,7 +593,7 @@ def write_output(
             Output file path
         meta_data : dict | None
             Dictionary of meta data from model
-        invert_uv : bool | None
+        invert_uv : bool
             Whether to convert u and v wind components to windspeed and
             direction
         nn_fill : bool

sup3r/postprocessing/writers/base.py

@@ -25,7 +25,7 @@ def _write_output(
         times,
         out_file,
         meta_data=None,
-        invert_uv=None,
+        invert_uv=False,
         nn_fill=False,
         max_workers=None,
         gids=None,
@@ -49,7 +49,7 @@ def _write_output(
             Output file path
         meta_data : dict | None
             Dictionary of meta data from model
-        invert_uv : bool | None
+        invert_uv : bool
             Whether to convert u and v wind components to windspeed and
             direction
         nn_fill : bool
@@ -71,7 +71,6 @@ def _write_output(
             f'({len(times)}) conflict.'
         )
         assert data.shape[-2] == len(times), msg
-        invert_uv = True if invert_uv is None else invert_uv
         data, features = cls._transform_output(
             data.copy(),
             features,
@@ -85,13 +84,11 @@ def _write_output(
             if gids is not None
             else np.arange(np.prod(lat_lon.shape[:-1]))
         )
-        meta = pd.DataFrame(
-            {
-                'gid': gids.flatten(),
-                'latitude': lat_lon[..., 0].flatten(),
-                'longitude': lat_lon[..., 1].flatten(),
-            }
-        )
+        meta = pd.DataFrame({
+            'gid': gids.flatten(),
+            'latitude': lat_lon[..., 0].flatten(),
+            'longitude': lat_lon[..., 1].flatten(),
+        })
         data_list = []
         for i, _ in enumerate(features):
             flat_data = data[..., i].reshape((-1, len(times)))

sup3r/postprocessing/writers/h5.py

@@ -1,5 +1,6 @@
 """Output handling"""
 
+import datetime
 import logging
 from datetime import datetime as dt
 
@@ -27,7 +28,7 @@ def _write_output(
         out_file,
         meta_data=None,
         max_workers=None,
-        invert_uv=None,
+        invert_uv=False,
         nn_fill=False,
         gids=None,
     ):
@@ -52,7 +53,7 @@ def _write_output(
             Dictionary of meta data from model
         max_workers : int | None
             Max workers to use for inverse transform.
-        invert_uv : bool | None
+        invert_uv : bool
             Whether to convert u and v wind components to windspeed and
             direction
         nn_fill : bool
@@ -62,8 +63,6 @@ def _write_output(
             List of coordinate indices used to label each lat lon pair and to
             help with spatial chunk data collection
         """
-
-        invert_uv = False if invert_uv is None else invert_uv
         data, features = cls._transform_output(
             data=data,
             features=features,
@@ -88,7 +87,7 @@ def _write_output(
             )
 
         attrs = meta_data or {}
-        now = dt.utcnow().isoformat()
+        now = dt.now(datetime.timezone.utc).isoformat()
         attrs['date_modified'] = now
         attrs['date_created'] = attrs.get('date_created', now)
 

sup3r/postprocessing/writers/nc.py

@@ -83,8 +83,9 @@ def __init__(
         self._lr_only_features = feature_sets.get('lr_only_features', [])
         self._hr_exo_features = feature_sets.get('hr_exo_features', [])
         self.features = self.get_features(feature_sets)
-        lr_feats = self.lr_data.features
-        self.lr_features = [f for f in lr_feats if f in self.features]
+        self.lr_features = [
+            f for f in self.features if f in self.lr_data.features
+        ]
         self.s_enhance = s_enhance
         self.t_enhance = t_enhance
         self.check_for_consistent_shapes()

sup3r/preprocessing/samplers/dual.py

@@ -73,6 +73,7 @@ def test_fwp_nc_cc():
             out_pattern=out_files,
             input_handler_name='DataHandlerNCforCC',
             pass_workers=None,
+            invert_uv=False
         )
         forward_pass = ForwardPass(strat)
         forward_pass.run(strat, node_index=0)
@@ -172,6 +173,7 @@ def test_fwp_spatial_only(input_files):
             out_pattern=out_files,
             pass_workers=1,
             output_workers=1,
+            invert_uv=False
         )
         forward_pass = ForwardPass(strat)
         assert strat.output_workers == 1
@@ -225,6 +227,7 @@ def test_fwp_nc(input_files):
             },
             out_pattern=out_files,
             pass_workers=1,
+            invert_uv=False
         )
         forward_pass = ForwardPass(strat)
         assert forward_pass.strategy.pass_workers == 1
@@ -614,6 +617,7 @@ def test_fwp_multi_step_model(input_files):
             model_class=strat.model_class,
             allowed_const=strat.allowed_const,
             output_workers=strat.output_workers,
+            invert_uv=strat.invert_uv,
             meta=fwp.meta,
         )
 

tests/forward_pass/test_forward_pass.py

@@ -79,6 +79,7 @@ def test_qa(input_files, ext):
             input_handler_kwargs=input_handler_kwargs.copy(),
             out_pattern=out_files,
             max_nodes=1,
+            invert_uv=False
         )
 
         forward_pass = ForwardPass(strategy)