Moving tests from flat to hierarchical structure (#1349)

* o Reorg test files, use paths.py

* Fix remaining path references after merge

* Clean up remaining redundant variables

* o .. Consolidate test file paths: remove redundant gridfile_* variables, use direct imports from paths module

* Fix additional redundant variables and indentation in test files

* o Missed a few more

* o Fix files in test/ folder; add new mesh files to paths; fix more leftover from last time

* fix whitespace

* o Add pytest fixtures for test data management

- Add conftest.py with gridpath and datasetpath fixtures
- Migrate test_accessors.py to use fixtures instead of sys.path manipulation
- Remove 'from paths import *' anti-pattern
- Enable running tests with 'python -m pytest' without path hacks

* o Enhance fixtures to support multiple files and migrate test_dataarray.py

- Update gridpath and datasetpath fixtures to support returning lists of paths
- When last argument is a list, returns list of Path objects
- Migrate test_dataarray.py to use fixtures instead of sys.path manipulation
- Remove 'from paths import *' anti-pattern from test_dataarray.py

* o Migrate to conftest.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update test_plot.py

* o Remove unnecessary imports

* o Remove stale comments

---------

Co-authored-by: Orhan Eroglu <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

test/__init__.py

@@ -0,0 +1,108 @@
+"""Pytest configuration and fixtures for uxarray tests."""
+
+import pytest
+import numpy as np
+from pathlib import Path
+
+
+@pytest.fixture
+def gridpath():
+    """Get the path to test grid/mesh file(s).
+
+    Parameters
+    ----------
+    *args : str
+        Path components relative to test/meshfiles/
+        The last argument can be a list of filenames to return multiple paths.
+
+    Returns
+    -------
+    Path or list of Path
+        Single path or list of paths to test grid files
+    """
+    base_path = Path(__file__).parent / "meshfiles"
+
+    def _get_path(*args):
+        # If the last argument is a list, handle multiple files
+        if args and isinstance(args[-1], list):
+            base_parts = args[:-1]
+            filenames = args[-1]
+            paths = []
+            for filename in filenames:
+                path = base_path.joinpath(*base_parts, filename)
+                if not path.exists():
+                    pytest.skip(f"Test grid file not found: {path}")
+                paths.append(path)
+            return paths
+        else:
+            # Single file case
+            path = base_path.joinpath(*args)
+            if not path.exists():
+                raise FileNotFoundError(f"Test grid file not found: {path}")
+            return path
+
+    return _get_path
+
+
+@pytest.fixture
+def datasetpath():
+    """Get the path to test dataset file(s).
+
+    Parameters
+    ----------
+    *args : str
+        Path components relative to test/meshfiles/
+        The last argument can be a list of filenames to return multiple paths.
+
+    Returns
+    -------
+    Path or list of Path
+        Single path or list of paths to test dataset files
+    """
+    base_path = Path(__file__).parent / "meshfiles"
+
+    def _get_path(*args):
+        # If the last argument is a list, handle multiple files
+        if args and isinstance(args[-1], list):
+            base_parts = args[:-1]
+            filenames = args[-1]
+            paths = []
+            for filename in filenames:
+                path = base_path.joinpath(*base_parts, filename)
+                if not path.exists():
+                    pytest.skip(f"Test dataset file not found: {path}")
+                paths.append(path)
+            return paths
+        else:
+            # Single file case
+            path = base_path.joinpath(*args)
+            if not path.exists():
+                raise FileNotFoundError(f"Test dataset file not found: {path}")
+            return path
+
+    return _get_path
+
+
+@pytest.fixture
+def test_data_dir():
+    """Return the path to the test data directory."""
+    return Path(__file__).parent / "meshfiles"
+
+
+@pytest.fixture
+def mesh_constants():
+    """Test constants for mesh validation."""
+    return {
+        'NNODES_ov_RLL10deg_CSne4': 683,
+        'NNODES_outCSne8': 386,
+        'NNODES_outCSne30': 5402,
+        'NNODES_outRLL1deg': 64442,
+        'DATAVARS_outCSne30': 4,
+        'TRI_AREA': 0.02216612469199045,
+        'CORRECTED_TRI_AREA': 0.02244844510268421,
+        'MESH30_AREA': 12.566,
+        'PSI_INTG': 12.566,
+        'VAR2_INTG': 12.566,
+        'UNIT_SPHERE_AREA': 4 * np.pi,
+        'FACE_VERTS_AREA': 0.515838,
+    }

test/conftest.py

@@ -5,8 +5,6 @@
 and return UxDataArray/UxDataset objects.
 """
 
-import os
-from pathlib import Path
 import numpy as np
 import pandas as pd
 import xarray as xr
@@ -15,19 +13,11 @@
 import pytest
 
 
-current_path = Path(os.path.dirname(os.path.realpath(__file__)))
-
-gridfile_ne30 = current_path / "meshfiles" / "ugrid" / "outCSne30" / "outCSne30.ug"
-dsfile_var2_ne30 = current_path / "meshfiles" / "ugrid" / "outCSne30" / "outCSne30_var2.nc"
-gridfile_geoflow = current_path / "meshfiles" / "ugrid" / "geoflow-small" / "grid.nc"
-dsfile_v1_geoflow = current_path / "meshfiles" / "ugrid" / "geoflow-small" / "v1.nc"
-mpas_ds_path = current_path / 'meshfiles' / "mpas" / "QU" / 'mesh.QU.1920km.151026.nc'
-
-
-def test_groupby_preserves_uxgrid():
+def test_groupby_preserves_uxgrid(gridpath):
     """Test that groupby operations preserve the uxgrid attribute."""
     # Create a dataset from a file
-    uxds = ux.open_dataset(mpas_ds_path, mpas_ds_path)
+    mesh_path = gridpath("mpas", "QU", "mesh.QU.1920km.151026.nc")
+    uxds = ux.open_dataset(mesh_path, mesh_path)
     original_grid = uxds.uxgrid
 
     # Create bins from latitude values (extract data explicitly)
@@ -47,10 +37,12 @@ def test_groupby_preserves_uxgrid():
     assert ds_result.uxgrid is not None
     assert ds_result.uxgrid == original_grid
 
-def test_groupby_bins_preserves_uxgrid():
+
+def test_groupby_bins_preserves_uxgrid(gridpath):
     """Test that groupby_bins operations preserve the uxgrid attribute."""
     # Create a dataset from a file
-    uxds = ux.open_dataset(mpas_ds_path, mpas_ds_path)
+    mesh_path = gridpath("mpas", "QU", "mesh.QU.1920km.151026.nc")
+    uxds = ux.open_dataset(mesh_path, mesh_path)
     original_grid = uxds.uxgrid
 
     # Create bins from latitude values (extract data explicitly)
@@ -68,8 +60,7 @@ def test_groupby_bins_preserves_uxgrid():
     assert ds_result.uxgrid == original_grid
 
 
-
-def test_resample_preserves_uxgrid_and_reduces_time():
+def test_resample_preserves_uxgrid_and_reduces_time(gridpath):
     """Test that resample operations preserve uxgrid and reduce time dimension."""
 
     # Create a simple test with only time dimension
@@ -84,7 +75,7 @@ def test_resample_preserves_uxgrid_and_reduces_time():
 
     # Open the minimal dataset with a real grid
     # Use existing test file we know works
-    uxgrid = ux.open_grid(gridfile_ne30)
+    uxgrid = ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug"))
 
     # Create a UxDataset with this grid
     uxds = ux.UxDataset(xr_ds, uxgrid=uxgrid)
@@ -97,7 +88,8 @@ def test_resample_preserves_uxgrid_and_reduces_time():
     assert result.uxgrid == uxds.uxgrid, "uxgrid not equal after resample"
     assert len(result.time) < len(uxds.time), "time dimension not reduced"
 
-def test_resample_preserves_uxgrid():
+
+def test_resample_preserves_uxgrid(gridpath):
     """Test that resample preserves the uxgrid attribute."""
 
     # Create a simple dataset with a time dimension
@@ -111,7 +103,8 @@ def test_resample_preserves_uxgrid():
     )
 
     # Create a UxDataset with a real grid
-    uxds = ux.open_dataset(gridfile_ne30, gridfile_ne30)
+    grid_path = gridpath("ugrid", "outCSne30", "outCSne30.ug")
+    uxds = ux.open_dataset(grid_path, grid_path)
     original_uxgrid = uxds.uxgrid
 
     # Create a new UxDataset with our time data and the real grid
@@ -128,7 +121,7 @@ def test_resample_preserves_uxgrid():
     assert ds_result.uxgrid is original_uxgrid, "uxgrid not identical after Dataset resample"
 
 
-def test_resample_reduces_time_dimension():
+def test_resample_reduces_time_dimension(gridpath):
     """Test that resample properly reduces the time dimension."""
 
     # Create dataset with daily data for a year
@@ -142,7 +135,7 @@ def test_resample_reduces_time_dimension():
     )
 
     # Create a UxDataset
-    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridfile_ne30))
+    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug")))
 
     # Test monthly resampling reduces from 365 days to 12 months
     monthly = uxds.resample(time="1M").mean()
@@ -151,7 +144,7 @@ def test_resample_reduces_time_dimension():
     assert monthly.dims["time"] <= 12, "monthly resampling should give 12 or fewer time points"
 
 
-def test_resample_with_cftime():
+def test_resample_with_cftime(gridpath):
     """Test that resample works with cftime objects."""
 
     try:
@@ -170,7 +163,7 @@ def test_resample_with_cftime():
     )
 
     # Create a UxDataset
-    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridfile_ne30))
+    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug")))
 
     # Test that quarterly resampling works with cftime
     quarterly = uxds.resample(time="Q").mean()
@@ -179,7 +172,7 @@ def test_resample_with_cftime():
     assert quarterly.dims["time"] < uxds.dims["time"], "time dimension not reduced with cftime"
 
 
-def test_rolling_preserves_uxgrid():
+def test_rolling_preserves_uxgrid(gridpath):
     """Test that rolling operations preserve the uxgrid attribute."""
 
     # Create a dataset with time dimension
@@ -193,7 +186,7 @@ def test_rolling_preserves_uxgrid():
     )
 
     # Create a UxDataset with a real grid
-    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridfile_ne30))
+    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug")))
     original_uxgrid = uxds.uxgrid
 
     # Test DataArray rolling preserves uxgrid
@@ -213,7 +206,7 @@ def test_rolling_preserves_uxgrid():
     assert not np.isnan(da_result.values[6:]).any(), "rolling mean should have valid values after window size"
 
 
-def test_coarsen_preserves_uxgrid():
+def test_coarsen_preserves_uxgrid(gridpath):
     """Test that coarsen operations preserve the uxgrid attribute."""
 
     # Create a dataset with time dimension (multiple of coarsen factor)
@@ -227,7 +220,7 @@ def test_coarsen_preserves_uxgrid():
     )
 
     # Create a UxDataset with a real grid
-    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridfile_ne30))
+    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug")))
     original_uxgrid = uxds.uxgrid
 
     # Test DataArray coarsen preserves uxgrid
@@ -247,14 +240,17 @@ def test_coarsen_preserves_uxgrid():
     assert ds_result.dims["time"] == 8, "coarsen should reduce time dimension"
 
 
-def test_weighted_preserves_uxgrid():
+def test_weighted_preserves_uxgrid(gridpath, datasetpath):
     """Test that weighted operations preserve the uxgrid attribute."""
 
     # Create a dataset with time and face dimensions
     times = pd.date_range("2000-01-01", periods=10, freq="D")
 
     # Open a real dataset to get face dimension
-    uxds_base = ux.open_dataset(gridfile_ne30, dsfile_var2_ne30)
+    uxds_base = ux.open_dataset(
+        gridpath("ugrid", "outCSne30", "outCSne30.ug"),
+        datasetpath("ugrid", "outCSne30", "outCSne30_var2.nc")
+    )
     n_face = uxds_base.dims["n_face"]
 
     # Create data with time and face dimensions
@@ -292,7 +288,7 @@ def test_weighted_preserves_uxgrid():
     assert da_result.shape == (n_face,), "result should only have face dimension"
 
 
-def test_cumulative_preserves_uxgrid():
+def test_cumulative_preserves_uxgrid(gridpath):
     """Test that cumulative operations preserve the uxgrid attribute."""
 
     # Create a dataset with time dimension
@@ -306,7 +302,7 @@ def test_cumulative_preserves_uxgrid():
     )
 
     # Create a UxDataset with a real grid
-    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridfile_ne30))
+    uxds = ux.UxDataset(ds, uxgrid=ux.open_grid(gridpath("ugrid", "outCSne30", "outCSne30.ug")))
     original_uxgrid = uxds.uxgrid
 
     # Test DataArray cumulative preserves uxgrid