Fix: Dependency pinning and upgrade issues (#312)

* fix pytest v9 typing issue

* update for python 3.14 support now that pygam is being updated again

* adding fill_value=False to shift function in filters.unresponsive_flag to get expected behavior at end of time series with NumPy v2

* change np.Inf to np.inf in timeseries tests for compatibility with NumPy v2

* fix scikit-learn max version for unsupported functionality in pygam

* udpate python in ci-tests

* convert python versions to strings

* attempt sklearn 1.5.x

* drop 3.14 support due to windows scikit-learn build issues

* update CI runner

* update changelog

* bump pytest minimum to support subtests for finnicky tests

* add xfail to finnicky gbm test and subtests for multiple checks

* add reason to xfail

* update target versions for linting

* autoupdate pre-commit

* add subtests to yaw and xfail for UQ

* update changelog

* move xfail to the correct test

* limit pandas to <3

* update changelog with correct date

---------

Co-authored-by: Eric Simley <[email protected]>

Author: RHammond2

.github/workflows/ci-tests.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: [3.9, 3.11]
+        python-version: ["3.10", "3.13"]
 
     steps:
     - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 7.0.0
     hooks:
       - id: isort
         additional_dependencies: [toml]
@@ -15,21 +15,21 @@ repos:
         types: [pyi]
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.21.2
     hooks:
     - id: pyupgrade
       args: [--py38-plus]
 
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 26.1.0
     hooks:
         - id: black
           name: black
           stages: [pre-commit]
           language_version: python3
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v6.0.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -42,6 +42,6 @@ repos:
         args: [--autofix]
 
   -   repo: https://github.com/pycqa/flake8
-      rev: '6.0.0'
+      rev: '7.3.0'
       hooks:
       -   id: flake8

CHANGELOG.md

@@ -1,6 +1,19 @@
 # Changelog
 All notable changes to this project will be documented in this file. If you make a notable change to the project, please add a line describing the change to the "unreleased" section. The maintainers will make an effort to keep the [Github Releases](https://github.com/NREL/OpenOA/releases) page up to date with this changelog. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## v3.1.4 - 2026-01-29
+
+- During the custom test collection, convert the `Path` objects to `str` to avoid issues with the
+  type enforcement of `list[str]` for `args` in Pytest v9.
+- Update PyGAM minimum version for its latest update that includes Python 3.10-3.13 support.
+- Remove maximum version pins for scipy and statsmodels with the support of the latest Python versions.
+- Adds a maximum version for scikit-learn for a change in their `__sklearn_tags__` support.
+- Deprecate support for Python 3.8 and 3.9, with additional support for Python 3.12 and 3.13.
+- Update the min and max versions to test in the testing CI workflow.
+- Utilizes pytest xfail and subtests to manange intermittent and finnicky test failures until a
+  long-term solution can be implemented.
+- Pin Pandas maximum version to the 2.x release cycle.
+
 ## v3.1.3 - 2025-01-31
 
 - Pin SciPy to >= 1.7 and <1.14 to avoid an incompatibility error with PyGAM.

openoa/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.1.3"
+__version__ = "3.1.4"
 
 """
 When bumping version, please be sure to also update parameters in sphinx/conf.py

openoa/utils/filters.py

@@ -103,7 +103,9 @@ def unresponsive_flag(
     flag = flag == 0
 
     # Need to flag preceding `threshold` values as well
-    flag = flag | np.any([flag.shift(-1 - i, axis=0) for i in range(threshold - 1)], axis=0)
+    flag = flag | np.any(
+        [flag.shift(-1 - i, axis=0, fill_value=False) for i in range(threshold - 1)], axis=0
+    )
 
     # Return back a pd.Series if one was provided, else a pd.DataFrame
     return flag[col[0]] if to_series else flag

pyproject.toml

@@ -8,20 +8,20 @@ dynamic = ["version"]
 authors = [{name = "NREL PRUF OA Team", email = "[email protected]"}]
 readme = {file = "README.md", content-type = "text/markdown"}
 description = "A package for collecting and assigning wind turbine metrics"
-requires-python = ">=3.8, <3.12"
+requires-python = ">=3.10,<3.14"
 license = {file = "LICENSE.txt"}
 dependencies = [
-    "scikit-learn>=1.0",
+    "scikit-learn>=1.0,<1.7",
     "requests>=2.21.0",
     "eia-python>=1.22",
     "pyproj>=3.5",
     "shapely>=1.8",
     "numpy>=1.24",
-    "pandas>=2.2",
-    "pygam>=0.9.0",
-    "scipy>=1.7,<1.14",
+    "pandas>=2.2,<3",
+    "pygam>=0.11.0",
+    "scipy>=1.7",
     "statsmodels>=0.11; python_version<'3.11'",
-    "statsmodels>=0.13.3; python_version=='3.11'",
+    "statsmodels>=0.13.3; python_version>='3.11'",
     "tqdm>=4.28.1",
     "matplotlib>=3.6",
     "bokeh>=3.3",
@@ -50,10 +50,11 @@ classifiers = [  # https://pypi.org/classifiers/
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Scientific/Engineering",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Typing :: Typed",
@@ -72,7 +73,7 @@ develop = [
   "isort",
   "flake8",
   "flake8-docstrings",
-  "pytest>=5.4.2",
+  "pytest>=9",
   "pytest-cov>=2.8.1",
 ]
 docs = [
@@ -140,7 +141,7 @@ filterwarnings = [
 [tool.black]
 # https://github.com/psf/black
 line-length = 100
-target-version = ["py38", "py39", "py310"]
+target-version = ["py310", "py311", "py312", "py313"]
 include = '\.pyi?$'
 exclude = '''
 # A regex preceded with ^/ will apply only to files and directories
@@ -166,7 +167,7 @@ exclude = '''
 [tool.isort]
 # https://github.com/PyCQA/isort
 profile = "black"
-py_version = 39
+py_version = 310
 src_paths = ["isort", "test"]
 line_length = "100"
 length_sort = "True"

test/conftest.py

@@ -25,8 +25,8 @@ def pytest_configure(config):
     regression = config.getoption("--regression")
 
     # Provide the appropriate directories
-    unit_tests = [el for el in (ROOT / "unit").iterdir() if el.suffix == ".py"]
-    regression_tests = [el for el in (ROOT / "regression").iterdir() if el.suffix == ".py"]
+    unit_tests = [str(el) for el in (ROOT / "unit").iterdir() if el.suffix == ".py"]
+    regression_tests = [str(el) for el in (ROOT / "regression").iterdir() if el.suffix == ".py"]
 
     # If both, run them all; if neither skip any modifications; otherwise run just the appropriate subset
     if regression and unit:

test/regression/long_term_monte_carlo_aep.py

@@ -329,6 +329,7 @@ def test_daily_gam(self):
         sim_results = self.analysis.results
         self.check_simulation_results_gam_daily(sim_results)
 
+    @pytest.mark.xfail(reason="Fails intermittently depending, mostly on MacOS.")
     def test_daily_gbm(self):
         reset_prng()
         # ____________________________________________________________________
@@ -484,7 +485,8 @@ def check_process_reanalysis_data_daily(self, df):
         print(computed)
 
         for key in expected.keys():
-            nptest.assert_array_almost_equal(expected[key], computed[key])
+            with self.subTest(f"checking {key}"):
+                nptest.assert_array_almost_equal(expected[key], computed[key])
 
     def check_simulation_results_lin_monthly(self, s):
         # Make sure AEP results are consistent to six decimal places

test/regression/yaw_misalignment.py

@@ -50,6 +50,7 @@ def test_yaw_misaliginment_without_UQ(self):
         )
         self.check_simulation_results_yaw_misalignment_without_UQ()
 
+    @pytest.mark.xfail(reason="System-dependendent intermittent failures")
     def test_yaw_misaliginment_with_UQ(self):
         reset_prng()
         # ____________________________________________________________________
@@ -171,21 +172,24 @@ def check_simulation_results_yaw_misalignment_without_UQ(self):
 
         calculated_yaw_mis_results_overall = self.analysis.yaw_misalignment
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_overall, calculated_yaw_mis_results_overall, decimal=5
-        )
+        with self.subTest("Checking overall results"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_overall, calculated_yaw_mis_results_overall, decimal=5
+            )
 
         calculated_yaw_mis_results_ws = self.analysis.yaw_misalignment_ws
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_ws, calculated_yaw_mis_results_ws, decimal=5
-        )
+        with self.subTest("Checking wind speeds"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_ws, calculated_yaw_mis_results_ws, decimal=5
+            )
 
         calculated_mean_vane_results_ws = self.analysis.mean_vane_angle_ws
 
-        nptest.assert_array_almost_equal(
-            expected_mean_vane_results_ws, calculated_mean_vane_results_ws, decimal=5
-        )
+        with self.subTest("Checking wind vane"):
+            nptest.assert_array_almost_equal(
+                expected_mean_vane_results_ws, calculated_mean_vane_results_ws, decimal=5
+            )
 
     def check_simulation_results_yaw_misalignment_with_UQ(self):
         # Make sure yaw misalignment results are consistent to six decimal places with UQ.
@@ -254,15 +258,21 @@ def check_simulation_results_yaw_misalignment_with_UQ(self):
 
         calculated_yaw_mis_results_avg_overall = self.analysis.yaw_misalignment_avg
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_avg_overall, calculated_yaw_mis_results_avg_overall, decimal=5
-        )
+        with self.subTest("Checking average of overall results"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_avg_overall,
+                calculated_yaw_mis_results_avg_overall,
+                decimal=5,
+            )
 
         calculated_yaw_mis_results_std_overall = self.analysis.yaw_misalignment_std
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_std_overall, calculated_yaw_mis_results_std_overall, decimal=5
-        )
+        with self.subTest("Checking standard deviation wind speeds"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_std_overall,
+                calculated_yaw_mis_results_std_overall,
+                decimal=5,
+            )
 
         # calculated_yaw_mis_results_95ci_overall = self.analysis.yaw_misalignment_95ci
 
@@ -274,15 +284,17 @@ def check_simulation_results_yaw_misalignment_with_UQ(self):
 
         calculated_yaw_mis_results_avg_ws = self.analysis.yaw_misalignment_avg_ws
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_avg_ws, calculated_yaw_mis_results_avg_ws, decimal=5
-        )
+        with self.subTest("Checking average wind speeds"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_avg_ws, calculated_yaw_mis_results_avg_ws, decimal=5
+            )
 
         calculated_yaw_mis_results_std_ws = self.analysis.yaw_misalignment_std_ws
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_std_ws, calculated_yaw_mis_results_std_ws, decimal=5
-        )
+        with self.subTest("Checking standard deviation wind speeds"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_std_ws, calculated_yaw_mis_results_std_ws, decimal=5
+            )
 
         # calculated_yaw_mis_results_95ci_ws = self.analysis.yaw_misalignment_95ci_ws
 
@@ -335,15 +347,21 @@ def check_simulation_results_yaw_misalignment_with_UQ_new_params(self):
 
         calculated_yaw_mis_results_avg_overall = self.analysis.yaw_misalignment_avg
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_avg_overall, calculated_yaw_mis_results_avg_overall, decimal=5
-        )
+        with self.subTest("Checking average of overall results"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_avg_overall,
+                calculated_yaw_mis_results_avg_overall,
+                decimal=5,
+            )
 
         calculated_yaw_mis_results_std_overall = self.analysis.yaw_misalignment_std
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_std_overall, calculated_yaw_mis_results_std_overall, decimal=5
-        )
+        with self.subTest("Checking standard deviation of overall results"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_std_overall,
+                calculated_yaw_mis_results_std_overall,
+                decimal=5,
+            )
 
         # calculated_yaw_mis_results_95ci_overall = self.analysis.yaw_misalignment_95ci
 
@@ -355,15 +373,17 @@ def check_simulation_results_yaw_misalignment_with_UQ_new_params(self):
 
         calculated_yaw_mis_results_avg_ws = self.analysis.yaw_misalignment_avg_ws
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_avg_ws, calculated_yaw_mis_results_avg_ws, decimal=5
-        )
+        with self.subTest("Checking average wind speeds"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_avg_ws, calculated_yaw_mis_results_avg_ws, decimal=5
+            )
 
         calculated_yaw_mis_results_std_ws = self.analysis.yaw_misalignment_std_ws
 
-        nptest.assert_array_almost_equal(
-            expected_yaw_mis_results_std_ws, calculated_yaw_mis_results_std_ws, decimal=5
-        )
+        with self.subTest("Checking standard deviation of wind speeds"):
+            nptest.assert_array_almost_equal(
+                expected_yaw_mis_results_std_ws, calculated_yaw_mis_results_std_ws, decimal=5
+            )
 
         # calculated_yaw_mis_results_95ci_ws = self.analysis.yaw_misalignment_95ci_ws
 

test/unit/test_timeseries_toolkit.py

@@ -145,8 +145,8 @@ def test_percent_nan(self):
         test_dict = {}
 
         # All should be float Series given PlantData requirements
-        test_dict["a"] = pd.Series([True, 1, 2, 1e5, np.Inf]).astype(float)
-        test_dict["b"] = pd.Series([False, np.nan, 2, 1e5, np.Inf]).astype(float)
+        test_dict["a"] = pd.Series([True, 1, 2, 1e5, np.inf]).astype(float)
+        test_dict["b"] = pd.Series([False, np.nan, 2, 1e5, np.inf]).astype(float)
         test_dict["c"] = pd.Series([np.nan, 1, 2, 1e5, np.nan]).astype(float)
 
         nan_values = {"a": 0.0, "b": 0.2, "c": 0.4}