Testing improvements.

1. Record duration
2. Speed up some tests
3. Silence logging

Author: dcherian

.github/workflows/ci.yaml

@@ -62,7 +62,7 @@ jobs:
       - name: Run Tests
         id: status
         run: |
-          pytest -n auto --cov=./ --cov-report=xml --hypothesis-profile ci
+          pytest --durations=20 --durations-min=0.5 -n auto --cov=./ --cov-report=xml --hypothesis-profile ci
       - name: Upload code coverage to Codecov
         uses: codecov/[email protected]
         with:

flox/aggregations.py

@@ -76,7 +76,7 @@ def generic_aggregate(
         try:
             method = getattr(aggregate_flox, func)
         except AttributeError:
-            logger.debug(f"Couldn't find {func} for engine='flox'. Falling back to numpy")
+            # logger.debug(f"Couldn't find {func} for engine='flox'. Falling back to numpy")
             method = get_npg_aggregation(func, engine="numpy")
 
     elif engine == "numbagg":
@@ -94,7 +94,7 @@ def generic_aggregate(
                 method = getattr(aggregate_numbagg, func)
 
         except AttributeError:
-            logger.debug(f"Couldn't find {func} for engine='numbagg'. Falling back to numpy")
+            # logger.debug(f"Couldn't find {func} for engine='numbagg'. Falling back to numpy")
             method = get_npg_aggregation(func, engine="numpy")
 
     elif engine in ["numpy", "numba"]:

flox/core.py

@@ -2011,7 +2011,7 @@ def _validate_reindex(
                 reindex = True
 
     assert isinstance(reindex, bool)
-    logger.debug("Leaving _validate_reindex: reindex is {}".format(reindex))  # noqa
+    # logger.debug("Leaving _validate_reindex: reindex is {}".format(reindex))  # noqa
 
     return reindex
 

tests/conftest.py

@@ -28,3 +28,15 @@
 )
 def engine(request):
     return request.param
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        "flox",
+        "numpy",
+        pytest.param("numbagg", marks=requires_numbagg),
+    ],
+)
+def engine_no_numba(request):
+    return request.param

tests/test_core.py

@@ -187,7 +187,7 @@ def test_groupby_reduce(
     assert_equal(expected_result, result)
 
 
-def gen_array_by(size, func: str):
+def gen_array_by(size, func):
     by = np.ones(size[-1])
     rng = np.random.default_rng(12345)
     array = rng.random(tuple(6 if s == 1 else s for s in size))

tests/test_properties.py

@@ -1,3 +1,4 @@
+import warnings
 from typing import Any, Callable
 
 import pandas as pd
@@ -51,7 +52,9 @@ def not_overflowing_array(array: np.ndarray[Any, Any]) -> bool:
 
     array = array.ravel()
     array = array[notnull(array)]
-    result = bool(np.all((array < info.max / array.size) & (array > info.min / array.size)))
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", RuntimeWarning)
+        result = bool(np.all((array < info.max / array.size) & (array > info.min / array.size)))
     # note(f"returning {result}, {array.min()} vs {info.min}, {array.max()} vs {info.max}")
     return result
 

tests/test_xarray.py

@@ -22,13 +22,9 @@
 
     dask.config.set(scheduler="sync")
 
-try:
-    # test against legacy xarray implementation
-    xr.set_options(use_flox=False)
-except ValueError:
-    pass
-
-
+# test against legacy xarray implementation
+# avoid some compilation overhead
+xr.set_options(use_flox=False, use_numbagg=False)
 tolerance64 = {"rtol": 1e-15, "atol": 1e-18}
 np.random.seed(123)
 
@@ -37,7 +33,8 @@
 @pytest.mark.parametrize("min_count", [None, 1, 3])
 @pytest.mark.parametrize("add_nan", [True, False])
 @pytest.mark.parametrize("skipna", [True, False])
-def test_xarray_reduce(skipna, add_nan, min_count, engine, reindex):
+def test_xarray_reduce(skipna, add_nan, min_count, engine_no_numba, reindex):
+    engine = engine_no_numba
     if skipna is False and min_count is not None:
         pytest.skip()
 
@@ -57,7 +54,13 @@ def test_xarray_reduce(skipna, add_nan, min_count, engine, reindex):
 
     expected = da.groupby("labels").sum(skipna=skipna, min_count=min_count)
     actual = xarray_reduce(
-        da, "labels", func="sum", skipna=skipna, min_count=min_count, engine=engine, reindex=reindex
+        da,
+        "labels",
+        func="sum",
+        skipna=skipna,
+        min_count=min_count,
+        engine=engine,
+        reindex=reindex,
     )
     assert_equal(expected, actual)
 
@@ -85,9 +88,10 @@ def test_xarray_reduce(skipna, add_nan, min_count, engine, reindex):
 # TODO: sort
 @pytest.mark.parametrize("pass_expected_groups", [True, False])
 @pytest.mark.parametrize("chunk", (pytest.param(True, marks=requires_dask), False))
-def test_xarray_reduce_multiple_groupers(pass_expected_groups, chunk, engine):
+def test_xarray_reduce_multiple_groupers(pass_expected_groups, chunk, engine_no_numba):
     if chunk and pass_expected_groups is False:
         pytest.skip()
+    engine = engine_no_numba
 
     arr = np.ones((4, 12))
     labels = np.array(["a", "a", "c", "c", "c", "b", "b", "c", "c", "b", "b", "f"])
@@ -131,9 +135,10 @@ def test_xarray_reduce_multiple_groupers(pass_expected_groups, chunk, engine):
 
 @pytest.mark.parametrize("pass_expected_groups", [True, False])
 @pytest.mark.parametrize("chunk", (pytest.param(True, marks=requires_dask), False))
-def test_xarray_reduce_multiple_groupers_2(pass_expected_groups, chunk, engine):
+def test_xarray_reduce_multiple_groupers_2(pass_expected_groups, chunk, engine_no_numba):
     if chunk and pass_expected_groups is False:
         pytest.skip()
+    engine = engine_no_numba
 
     arr = np.ones((2, 12))
     labels = np.array(["a", "a", "c", "c", "c", "b", "b", "c", "c", "b", "b", "f"])
@@ -187,7 +192,8 @@ def test_validate_expected_groups(expected_groups):
 
 @requires_cftime
 @requires_dask
-def test_xarray_reduce_single_grouper(engine):
+def test_xarray_reduce_single_grouper(engine_no_numba):
+    engine = engine_no_numba
     # DataArray
     ds = xr.Dataset(
         {"Tair": (("time", "x", "y"), dask.array.ones((36, 205, 275), chunks=(9, -1, -1)))},
@@ -293,15 +299,17 @@ def test_rechunk_for_blockwise(inchunks, expected):
 # TODO: dim=None, dim=Ellipsis, groupby unindexed dim
 
 
-def test_groupby_duplicate_coordinate_labels(engine):
+def test_groupby_duplicate_coordinate_labels(engine_no_numba):
+    engine = engine_no_numba
     # fix for http://stackoverflow.com/questions/38065129
     array = xr.DataArray([1, 2, 3], [("x", [1, 1, 2])])
     expected = xr.DataArray([3, 3], [("x", [1, 2])])
     actual = xarray_reduce(array, array.x, func="sum", engine=engine)
     assert_equal(expected, actual)
 
 
-def test_multi_index_groupby_sum(engine):
+def test_multi_index_groupby_sum(engine_no_numba):
+    engine = engine_no_numba
     # regression test for xarray GH873
     ds = xr.Dataset(
         {"foo": (("x", "y", "z"), np.ones((3, 4, 2)))},
@@ -327,7 +335,8 @@ def test_multi_index_groupby_sum(engine):
 
 
 @pytest.mark.parametrize("chunks", (None, pytest.param(2, marks=requires_dask)))
-def test_xarray_groupby_bins(chunks, engine):
+def test_xarray_groupby_bins(chunks, engine_no_numba):
+    engine = engine_no_numba
     array = xr.DataArray([1, 1, 1, 1, 1], dims="x")
     labels = xr.DataArray([1, 1.5, 1.9, 2, 3], dims="x", name="labels")
 
@@ -495,11 +504,11 @@ def test_alignment_error():
 @pytest.mark.parametrize("dtype_out", [np.float64, "float64", np.dtype("float64")])
 @pytest.mark.parametrize("dtype", [np.float32, np.float64])
 @pytest.mark.parametrize("chunk", (pytest.param(True, marks=requires_dask), False))
-def test_dtype(add_nan, chunk, dtype, dtype_out, engine):
-    if engine == "numbagg":
+def test_dtype(add_nan, chunk, dtype, dtype_out, engine_no_numba):
+    if engine_no_numba == "numbagg":
         # https://github.com/numbagg/numbagg/issues/121
         pytest.skip()
-
+    engine = engine_no_numba
     xp = dask.array if chunk else np
     data = xp.linspace(0, 1, 48, dtype=dtype).reshape((4, 12))
 
@@ -707,7 +716,7 @@ def test_multiple_quantiles(q, chunk, by_ndim, skipna):
     da = xr.DataArray(array, dims=("x", *dims))
     by = xr.DataArray(labels, dims=dims, name="by")
 
-    actual = xarray_reduce(da, by, func="quantile", skipna=skipna, q=q)
+    actual = xarray_reduce(da, by, func="quantile", skipna=skipna, q=q, engine="flox")
     with xr.set_options(use_flox=False):
         expected = da.groupby(by).quantile(q, skipna=skipna)
     xr.testing.assert_allclose(expected, actual)