Handle -99999 NaN for 6-character width fields in USCRN data (#775)

* handle -99999 for 6-character width fields

* add whatsnew

* link to pvlib issue

Author: lboeman

docs/source/whatsnew/1.0.12.rst

@@ -0,0 +1,17 @@
+.. _whatsnew_1012:
+
+.. py:currentmodule:: solarforecastarbiter
+
+
+1.0.12 (January ??, 2022)
+--------------------------
+
+Fixed
+~~~~~~~~~~~~
+* Fixed USCRN data fetch to handle NaN values of -99999. (:pull:`775`, :issue:`773`)
+
+Contributors
+~~~~~~~~~~~~
+
+* Will Holmgren (:ghuser:`wholmgren`)
+* Leland Boeman (:ghuser:`lboeman`)

docs/source/whatsnew/index.rst

@@ -9,6 +9,7 @@ These are new features and improvements of note in each release.
 .. toctree::
    :maxdepth: 2
 
+   1.0.12
    1.0.11
    1.0.10
    1.0.9

solarforecastarbiter/io/reference_observations/crn.py

@@ -2,6 +2,7 @@
 from urllib.error import URLError
 
 
+import numpy as np
 import pandas as pd
 from pvlib import iotools
 
@@ -72,6 +73,11 @@ def fetch(api, site, start, end):
         return pd.DataFrame()
     all_period_data = all_period_data.rename(
         columns={'temp_air': 'air_temperature'})
+
+    # Remove possible nans with value -99999. This can be removed
+    # if pvlib is updated to account for these values.
+    # See pvlib issue: https://github.com/pvlib/pvlib-python/issues/1372
+    all_period_data = all_period_data.replace(-99999, np.nan)
     return all_period_data
 
 

solarforecastarbiter/io/reference_observations/tests/test_crn.py

@@ -0,0 +1,41 @@
+import pandas as pd
+
+from solarforecastarbiter.io.reference_observations import crn
+from solarforecastarbiter.datamodel import Site
+
+
+test_site_dict = {
+    "extra_parameters": '{"network": "NOAA USCRN", "network_api_id": "NY_Millbrook_3_W", "network_api_abbreviation": "NY_Millbrook_3_W", "observation_interval_length": 5}',  # noqa: E501
+    "climate_zones": [
+      "Reference Region 5"
+    ],
+    "created_at": "2019-05-24T17:31:44+00:00",
+    "elevation": 126,
+    "latitude": 41.78,
+    "longitude": -73.74,
+    "modified_at": "2020-09-15T18:22:05+00:00",
+    "name": "NOAA USCRN Millbrook NY",
+    "provider": "Reference",
+    "site_id": "cc9f4180-7e49-11e9-b191-0a580a8003e9",
+    "timezone": "America/New_York"
+}
+
+test_site_object = Site.from_dict(test_site_dict)
+
+
+def test_fetch_fill_nan_99999(mocker):
+    mocker.patch(
+        'solarforecastarbiter.io.reference_observations.crn.iotools.read_crn',
+        return_value=pd.DataFrame(
+            data=[-99999],
+            index=pd.DatetimeIndex(['20220101T0000Z']),
+            columns=['ghi']
+        )
+    )
+    result = crn.fetch(
+        None,
+        test_site_object,
+        pd.Timestamp('20210101t0000z'),
+        pd.Timestamp('20210101t0000z'),
+    )
+    assert pd.isna(result.iloc[0]['ghi'])