update for pvlib 0.9.0a4 (#124)

* update for pvlib 0.9.0a4

a4 because pvlib/pvlib-python#1162 is required
along with a tbd issue for computing from effective irradiance

closes #119 closes #109

also adds real tests of the pvlib modelchain to catch errors there

* mypy

* test more modelchain configurations

* fix when missing poa global or solar position for from_effective_irr

* make sure poa_global is NaN not None

* use pvlib 0.9.0a4

* comment update

Author: alorenzo175

api/requirements.txt

@@ -26,7 +26,7 @@ prometheus-fastapi-instrumentator==5.5.1
 pandas==1.1.4
 pymysql==0.10.1
 sqlalchemy==1.3.20
-pvlib==0.9.0a1
+pvlib==0.9.0a4
 numpy==1.19.4
 scipy==1.5.4
 requests==2.24.0

api/solarperformanceinsight_api/compute.py

@@ -222,8 +222,8 @@ def process_single_modelchain(
     ).to_frame()  # type: ignore
     weather_sum = pd.DataFrame(
         {
-            "poa_global": 0,  # type: ignore
             "effective_irradiance": 0,  # type: ignore
+            "poa_global": 0,  # type: ignore
             "cell_temperature": 0,  # type: ignore
         },
         index=performance.index,
@@ -233,13 +233,17 @@ def process_single_modelchain(
     num_arrays = len(mc.system.arrays)
     out = []
     for i in range(num_arrays):
-        array_weather: pd.DataFrame = _get_index(results, "total_irrad", i)[
-            ["poa_global"]
-        ].copy()  # type: ignore
-        # copy avoids setting values on a copy of slice later
-        array_weather.loc[:, "effective_irradiance"] = _get_index(
+        array_weather: pd.DataFrame = _get_index(
             results, "effective_irradiance", i
+        ).to_frame(
+            "effective_irradiance"
         )  # type: ignore
+        # total irrad empty if effective irradiance supplied initially
+        array_weather.loc[:, "poa_global"] = _get_index(
+            results, "total_irrad", i
+        ).get(  # type: ignore
+            "poa_global", float("NaN")
+        )
         array_weather.loc[:, "cell_temperature"] = _get_index(
             results, "cell_temperature", i
         )  # type: ignore
@@ -254,9 +258,19 @@ def process_single_modelchain(
         )
     # mean
     weather_avg: pd.DataFrame = weather_sum / num_arrays  # type: ignore
-    adjusted_zenith: pd.DataFrame = adjust(
-        results.solar_position[["zenith"]]
-    )  # type: ignore
+    adjusted_zenith: pd.DataFrame
+    # not calculated if effective irradiance is provided
+    if results.solar_position is not None:
+        adjusted_zenith = adjust(results.solar_position[["zenith"]])  # type: ignore
+    else:
+        # calculate solar position making sure to shift times and shift back
+        # modelchain passes through air temperature and pressure, but that only
+        # affects apparent_zenith
+        adjusted_zenith = adjust(
+            mc.location.get_solarposition(
+                weather_avg.index.shift(freq=tshift)  # type: ignore
+            )[["zenith"]]
+        )  # type: ignore
     summary_frame = pd.concat(
         [
             performance,

api/solarperformanceinsight_api/conftest.py

@@ -1,12 +1,15 @@
 from base64 import b64decode
 from contextlib import contextmanager
+from copy import deepcopy
 import datetime as dt
 from uuid import UUID, uuid1
 
 
 from fakeredis import FakeRedis  # type: ignore
 from fastapi.testclient import TestClient
 import httpx
+from pvlib.pvsystem import PVSystem  # type: ignore
+from pvlib.tracking import SingleAxisTracker  # type: ignore
 import pymysql
 import pytest
 from rq import Queue  # type: ignore
@@ -314,3 +317,32 @@ def async_queue(mock_redis, mocker):
     q = Queue("jobs", connection=mock_redis)
     mocker.patch.object(queuing, "_get_queue", return_value=q)
     return q
+
+
+@pytest.fixture()
+def fixed_tracking():
+    return models.FixedTracking(tilt=32, azimuth=180.9)
+
+
+@pytest.fixture()
+def single_axis_tracking():
+    return models.SingleAxisTracking(
+        axis_tilt=0, axis_azimuth=179.8, backtracking=False, gcr=1.8
+    )
+
+
+@pytest.fixture(params=["fixed_axis", "single_axis", "multi_array_fixed"])
+def either_tracker(request, system_def, fixed_tracking, single_axis_tracking):
+    inv = system_def.inverters[0]
+    if request.param == "fixed_axis":
+        inv.arrays[0].tracking = fixed_tracking
+        return inv, PVSystem, False
+    elif request.param == "multi_array_fixed":
+        inv.arrays[0].tracking = fixed_tracking
+        arr1 = deepcopy(inv.arrays[0])
+        arr1.name = "Array 2"
+        inv.arrays.append(arr1)
+        return inv, PVSystem, True
+    else:
+        inv.arrays[0].tracking = single_axis_tracking
+        return inv, SingleAxisTracker, False

api/solarperformanceinsight_api/models.py

@@ -339,7 +339,7 @@ class PVWattsInverterParameters(BaseModel):
     eta_inv_ref: float = Field(
         0.9637, description="Reference inverter efficiency, unitless"
     )
-    _modelchain_ac_model: str = PrivateAttr("pvwatts_multi")
+    _modelchain_ac_model: str = PrivateAttr("pvwatts")
 
 
 class SandiaInverterParameters(BaseModel):
@@ -402,7 +402,7 @@ class SandiaInverterParameters(BaseModel):
             " (i.e., night tare), W"
         ),
     )
-    _modelchain_ac_model: str = PrivateAttr("sandia_multi")
+    _modelchain_ac_model: str = PrivateAttr("sandia")
 
 
 class AOIModelEnum(str, Enum):

api/solarperformanceinsight_api/tests/test_compute.py

@@ -5,10 +5,12 @@
 
 
 import pandas as pd
+from pvlib.location import Location
+from pvlib.modelchain import ModelChain
 import pytest
 
 
-from solarperformanceinsight_api import compute, storage, models
+from solarperformanceinsight_api import compute, storage, models, pvmodeling
 
 
 pytestmark = pytest.mark.usefixtures("add_example_db_data")
@@ -348,64 +350,125 @@ class Res:
     assert pd.isna(nans).all()
 
 
-def test_process_single_modelchain(mocker):
+# pytest param ids are helpful finding combos that fail
+@pytest.mark.parametrize(
+    "tempcols",
+    (
+        # pytest parm ids are
+        pytest.param(["temp_air", "wind_speed"], id="standard_temp"),
+        pytest.param(["temp_air"], id="air_temp_only"),
+        pytest.param(["module_temperature"], id="module_temp"),
+        pytest.param(["cell_temperature"], id="cell_temp"),
+        pytest.param(["module_temperature", "wind_speed"], id="module_temp+ws"),
+        pytest.param(["cell_temperature", "wind_speed"], id="cell_temp+ws"),
+    ),
+)
+@pytest.mark.parametrize(
+    "method,colmap",
+    (
+        pytest.param("run_model", {}, id="run_model"),
+        pytest.param(
+            "run_model_from_poa",
+            {"ghi": "poa_global", "dni": "poa_direct", "dhi": "poa_diffuse"},
+            id="run_model_poa",
+        ),
+        pytest.param(
+            "run_model_from_effective_irradiance",
+            {"ghi": "effective_irradiance", "dni": "noped", "dhi": "nah"},
+            id="run_model_eff",
+        ),
+    ),
+)
+def test_process_single_modelchain(
+    system_def, either_tracker, method, colmap, tempcols
+):
+    # full run through a modelchain with a fixed tilt single array,
+    # fixed tilt two array, and single axis tracker single array
     tshift = dt.timedelta(minutes=5)
-    df = pd.DataFrame({"poa_global": [1.0]}, index=[pd.Timestamp("2020-01-01T12:00")])
-    df.index.name = "time"
-    shifted = df.shift(freq=-tshift)
-
-    class Res:
-        ac = df["poa_global"]
-        total_irrad = (df, df)
-        effective_irradiance = (df, df)
-        cell_temperature = (df, df)
-        solar_position = pd.DataFrame({"zenith": 91.0}, index=df.index)
-
-    class Sys:
-        arrays = [0, 1]
-
-    class MC:
-        results = Res()
-        system = Sys()
-
-        def run_model(self, data):
-            pd.testing.assert_frame_equal(df, data[0])
-            return self
-
-    with pytest.raises(AttributeError):
-        compute.process_single_modelchain(MC(), [df], "run_from_poa", tshift, 0)
+    index = pd.DatetimeIndex([pd.Timestamp("2020-01-01T12:00:00-07:00")], name="time")
+    tempdf = pd.DataFrame(
+        {
+            "temp_air": [25.0],
+            "wind_speed": [10.0],
+            "module_temperature": [30.0],
+            "cell_temperature": [32.0],
+            "poa_global": [1100.0],
+        },
+        index=index,
+    )[tempcols]
+    irrad = pd.DataFrame(
+        {
+            "ghi": [1100.0],
+            "dni": [1000.0],
+            "dhi": [100.0],
+        },
+        index=index,
+    ).rename(columns=colmap)
+    df = pd.concat([irrad, tempdf], axis=1)
+    inv, _, multi = either_tracker
+    location = Location(latitude=32.1, longitude=-110.8, altitude=2000, name="test")
+    pvsys = pvmodeling.construct_pvsystem(inv)
+    mc = ModelChain(system=pvsys, location=location, **dict(inv._modelchain_models))
+    weather = [df]
+    if multi:
+        weather.append(df)
 
     # shifted (df - 5min) goes in, and shifted right (df) goes to be processed
-    dblist, summary = compute.process_single_modelchain(
-        MC(), [shifted], "run_model", tshift, 0
-    )
-    pd.testing.assert_frame_equal(
-        summary,
-        pd.DataFrame(
-            {
-                "performance": [1.0],
-                "poa_global": [1.0],
-                "effective_irradiance": [1.0],
-                "cell_temperature": [1.0],
-                "zenith": [91.0],
-            },
-            index=shifted.index,
-        ),
-    )
+    dblist, summary = compute.process_single_modelchain(mc, weather, method, tshift, 0)
+    assert summary.performance.iloc[0] == 250.0
+    assert set(summary.columns) == {
+        "performance",
+        "poa_global",
+        "effective_irradiance",
+        "cell_temperature",
+        "zenith",
+    }
+    pd.testing.assert_index_equal(summary.index, df.index)
 
     # performance for the inverter, and weather for each array
-    assert {d.schema_path for d in dblist} == {
-        "/inverters/0",
-        "/inverters/0/arrays/0",
-        "/inverters/0/arrays/1",
-    }
-    inv0arr0_weather = pd.read_feather(BytesIO(dblist[1].data))
-    exp_weather = shifted.copy()
-    exp_weather.loc[:, "effective_irradiance"] = 1.0
-    exp_weather.loc[:, "cell_temperature"] = 1.0
+    if multi:
+        assert {d.schema_path for d in dblist} == {
+            "/inverters/0",
+            "/inverters/0/arrays/0",
+            "/inverters/0/arrays/1",
+        }
+    else:
+        assert {d.schema_path for d in dblist} == {
+            "/inverters/0",
+            "/inverters/0/arrays/0",
+        }
+
+    inv_perf = list(
+        filter(
+            lambda x: x.type == "performance data" and x.schema_path == "/inverters/0",
+            dblist,
+        )
+    )[0]
     pd.testing.assert_frame_equal(
-        inv0arr0_weather, exp_weather.astype("float32").reset_index()
+        pd.read_feather(BytesIO(inv_perf.data)),
+        pd.DataFrame(
+            {"performance": [250.0]}, dtype="float32", index=df.index
+        ).reset_index(),
+    )
+    arr0_weather_df = pd.read_feather(
+        BytesIO(
+            list(
+                filter(
+                    lambda x: x.type == "weather data"
+                    and x.schema_path == "/inverters/0/arrays/0",
+                    dblist,
+                )
+            )[0].data
+        )
     )
+    assert set(arr0_weather_df.columns) == {
+        "poa_global",
+        "effective_irradiance",
+        "cell_temperature",
+        "time",
+    }
+    # pvlib>0.9.0a2
+    assert not pd.isna(arr0_weather_df.cell_temperature).any()
 
 
 def test_run_performance_job(stored_job, auth0_id, nocommit_transaction, mocker):

api/solarperformanceinsight_api/tests/test_pvmodeling.py

@@ -1,15 +1,13 @@
-from copy import deepcopy
 from inspect import signature
 
 
 from pvlib.location import Location
 from pvlib.modelchain import ModelChain
 from pvlib.pvsystem import PVSystem
 from pvlib.tracking import SingleAxisTracker
-import pytest
 
 
-from solarperformanceinsight_api import models, pvmodeling
+from solarperformanceinsight_api import pvmodeling
 
 
 def test_construct_location(system_def):
@@ -18,35 +16,6 @@ def test_construct_location(system_def):
     assert isinstance(pvmodeling.construct_location(system_def), Location)
 
 
-@pytest.fixture()
-def fixed_tracking():
-    return models.FixedTracking(tilt=32, azimuth=180.9)
-
-
-@pytest.fixture()
-def single_axis_tracking():
-    return models.SingleAxisTracking(
-        axis_tilt=0, axis_azimuth=179.8, backtracking=False, gcr=1.8
-    )
-
-
-@pytest.fixture(params=["fixed", "single", "multi_fixed"])
-def either_tracker(request, system_def, fixed_tracking, single_axis_tracking):
-    inv = system_def.inverters[0]
-    if request.param == "fixed":
-        inv.arrays[0].tracking = fixed_tracking
-        return inv, PVSystem, False
-    elif request.param == "multi_fixed":
-        inv.arrays[0].tracking = fixed_tracking
-        arr1 = deepcopy(inv.arrays[0])
-        arr1.name = "Array 2"
-        inv.arrays.append(arr1)
-        return inv, PVSystem, True
-    else:
-        inv.arrays[0].tracking = single_axis_tracking
-        return inv, SingleAxisTracker, False
-
-
 def test_construct_pvsystem(either_tracker):
     inv, cls, multi = either_tracker
     out = pvmodeling.construct_pvsystem(inv)