Add extreme QCRAD QC limits

docs/whatsnew/0.2.2.rst

@@ -6,11 +6,16 @@
 
 Enhancements
 ~~~~~~~~~~~~
+* Added extreme limits option to
+  :py:func:`~pvanalytics.quality.irradiance.check_ghi_limits_qcrad`,
+  :py:func:`~pvanalytics.quality.irradiance.check_dhi_limits_qcrad`,
+  :py:func:`~pvanalytics.quality.irradiance.check_dni_limits_qcrad`, and
+  :py:func:`~pvanalytics.quality.irradiance.check_irradiance_limits_qcrad`.
+  (:pull:`190`)
 * Added optional keyword `outside_domain` to
   :py:func:`~pvanalytics.quality.irradiance.check_irradiance_consistency_qcrad`.
   (:pull:`214`)
 
-
 Bug Fixes
 ~~~~~~~~~
 

pvanalytics/quality/irradiance.py

@@ -10,10 +10,18 @@
 from pvanalytics import util
 
 
-QCRAD_LIMITS = {'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100},
-                'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50},
-                'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0},
-                'ghi_lb': -4, 'dhi_lb': -4, 'dni_lb': -4}
+# QCRAD limits are often also referred to as BSRN limits
+QCRAD_LIMITS_PHYSICAL = {  # Physically Possible Limits
+    'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100},
+    'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50},
+    'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0},
+    'ghi_lb': -4, 'dhi_lb': -4, 'dni_lb': -4}
+
+QCRAD_LIMITS_EXTREME = {  # Extremely Rare Limits
+    'ghi_ub': {'mult': 1.2, 'exp': 1.2, 'min': 50},
+    'dhi_ub': {'mult': 0.75, 'exp': 1.2, 'min': 30},
+    'dni_ub': {'mult': 0.95, 'exp': 0.2, 'min': 10},
+    'ghi_lb': -2, 'dhi_lb': -2, 'dni_lb': -2}
 
 QCRAD_CONSISTENCY = {
     'ghi_ratio': {
@@ -42,8 +50,8 @@ def _qcrad_ub(dni_extra, sza, lim):
     return lim['mult'] * dni_extra * cosd_sza**lim['exp'] + lim['min']
 
 
-def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None):
-    r"""Test for physical limits on GHI using the QCRad criteria.
+def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits='physical'):
+    r"""Test for lower and upper limits on GHI using the QCRad criteria.
 
     Test is applied to each GHI value. A GHI value passes if value >
     lower bound and value < upper bound. Lower bounds are constant for
@@ -60,10 +68,11 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None):
         Solar zenith angle in degrees
     dni_extra : Series
         Extraterrestrial normal irradiance in :math:`W/m^2`
-    limits : dict, default QCRAD_LIMITS
-        Must have keys 'ghi_ub' and 'ghi_lb'. For 'ghi_ub' value is a
-        dict with keys {'mult', 'exp', 'min'} and float values. For
-        'ghi_lb' value is a float.
+    limits : {'physical', 'extreme'} or dict, default 'physical'
+        If string, must be either 'physical' or 'extreme', corresponding to the
+        QCRAD QC limits. If dict, must have keys 'ghi_ub' and 'ghi_lb'. For
+        'ghi_ub' value is a dict with keys {'mult', 'exp', 'min'} and float
+        values. For 'ghi_lb' value is a float.
 
     Returns
     -------
@@ -79,17 +88,20 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None):
     for more information.
 
     """
-    if not limits:
-        limits = QCRAD_LIMITS
+    if limits == 'physical':
+        limits = QCRAD_LIMITS_PHYSICAL
+    elif limits == 'extreme':
+        limits = QCRAD_LIMITS_EXTREME
+
     ghi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['ghi_ub'])
 
     ghi_limit_flag = quality.util.check_limits(ghi, limits['ghi_lb'], ghi_ub)
 
     return ghi_limit_flag
 
 
-def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
-    r"""Test for physical limits on DHI using the QCRad criteria.
+def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits='physical'):
+    r"""Test for lower and upper limits on DHI using the QCRad criteria.
 
     Test is applied to each DHI value. A DHI value passes if value >
     lower bound and value < upper bound. Lower bounds are constant for
@@ -106,10 +118,11 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
         Solar zenith angle in degrees
     dni_extra : Series
         Extraterrestrial normal irradiance in :math:`W/m^2`
-    limits : dict, default QCRAD_LIMITS
-        Must have keys 'dhi_ub' and 'dhi_lb'. For 'dhi_ub' value is a
-        dict with keys {'mult', 'exp', 'min'} and float values. For
-        'dhi_lb' value is a float.
+    limits : {'physical', 'extreme'} or dict, default 'physical'
+        If string, must be either 'physical' or 'extreme', corresponding to the
+        QCRAD QC limits. If dict, must have keys 'dhi_ub' and 'dhi_lb'. For
+        'dhi_ub' value is a dict with keys {'mult', 'exp', 'min'} and float
+        values. For 'dhi_lb' value is a float.
 
     Returns
     -------
@@ -125,8 +138,10 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
     for more information.
 
     """
-    if not limits:
-        limits = QCRAD_LIMITS
+    if limits == 'physical':
+        limits = QCRAD_LIMITS_PHYSICAL
+    elif limits == 'extreme':
+        limits = QCRAD_LIMITS_EXTREME
 
     dhi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dhi_ub'])
 
@@ -135,8 +150,8 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
     return dhi_limit_flag
 
 
-def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
-    r"""Test for physical limits on DNI using the QCRad criteria.
+def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits='physical'):
+    r"""Test for lower and upper limits on DNI using the QCRad criteria.
 
     Test is applied to each DNI value. A DNI value passes if value >
     lower bound and value < upper bound. Lower bounds are constant for
@@ -153,10 +168,11 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
         Solar zenith angle in degrees
     dni_extra : Series
         Extraterrestrial normal irradiance in :math:`W/m^2`
-    limits : dict, default QCRAD_LIMITS
-        Must have keys 'dni_ub' and 'dni_lb'. For 'dni_ub' value is a
-        dict with keys {'mult', 'exp', 'min'} and float values. For
-        'dni_lb' value is a float.
+    limits : {'physical', 'extreme'} or dict, default 'physical'
+        If string, must be either 'physical' or 'extreme', corresponding to the
+        QCRAD QC limits. If dict, must have keys 'dni_ub' and 'dni_lb'. For
+        'dni_ub' value is a dict with keys {'mult', 'exp', 'min'} and float
+        values. For 'dni_lb' value is a float.
 
     Returns
     -------
@@ -172,8 +188,10 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
     for more information.
 
     """
-    if not limits:
-        limits = QCRAD_LIMITS
+    if limits == 'physical':
+        limits = QCRAD_LIMITS_PHYSICAL
+    elif limits == 'extreme':
+        limits = QCRAD_LIMITS_EXTREME
 
     dni_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dni_ub'])
 
@@ -183,10 +201,10 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
 
 
 def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None,
-                                  dni=None, limits=None):
+                                  dni=None, limits='physical'):
     r"""Test for physical limits on GHI, DHI or DNI using the QCRad criteria.
 
-    Criteria from [1]_ are used to determine physically plausible
+    Criteria from [1]_ and [2]_ are used to determine physically plausible
     lower and upper bounds. Each value is tested and a value passes if
     value > lower bound and value < upper bound. Lower bounds are
     constant for all tests. Upper bounds are calculated as
@@ -209,10 +227,13 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None,
         Diffuse horizontal irradiance in :math:`W/m^2`
     dni : Series or None, default None
         Direct normal irradiance in :math:`W/m^2`
-    limits : dict, default QCRAD_LIMITS
-        for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with
-        keys {'mult', 'exp', 'min'} and float values. For keys
-        'ghi_lb', 'dhi_lb', 'dni_lb', value is a float.
+    limits : {'physical', 'extreme'} or dict, default 'physical'
+        If string, must be either 'physical' or 'extreme', corresponding to the
+        QCRAD QC limits. If dict, must have keys:
+
+            * 'ghi_ub', 'dhi_ub', 'dni_ub': dicts with keys
+              {'mult', 'exp', 'min'} and float values.
+            * 'ghi_lb', 'dhi_lb', 'dni_lb': float values.
 
     Returns
     -------
@@ -233,13 +254,19 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None,
 
     References
     ----------
-    .. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control
-       Algorithm for Surface Radiation Measurements, The Open Atmospheric
-       Science Journal 2, pp. 23-37, 2008.
-
-    """
-    if not limits:
-        limits = QCRAD_LIMITS
+    .. [1] C. N. Long and Y. Shi, "An Automated Quality Assessment and Control
+       Algorithm for Surface Radiation Measurements," The Open Atmospheric
+       Science Journal, vol. 2, no. 1. Bentham Science Publishers Ltd.,
+       pp. 23–37, Apr. 18, 2008. :doi:`10.2174/1874282300802010023`.
+    .. [2]  C. N. Long and E. G. Dutton, "BSRN Global Network recommended QC
+       tests, V2.0," Baseline Surface Radiation Network (BSRN),
+       Accessed: Oct. 24, 2024. [Online.] Available:
+       `<https://bsrn.awi.de/fileadmin/user_upload/bsrn.awi.de/Publications/BSRN_recommended_QC_tests_V2.pdf>`_
+    """  # noqa: E501
+    if limits == 'physical':
+        limits = QCRAD_LIMITS_PHYSICAL
+    elif limits == 'extreme':
+        limits = QCRAD_LIMITS_EXTREME
 
     if ghi is not None:
         ghi_limit_flag = check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra,
@@ -289,8 +316,8 @@ def check_irradiance_consistency_qcrad(solar_zenith, ghi, dhi, dni,
                                        param=None, outside_domain=False):
     r"""Check consistency of GHI, DHI and DNI using QCRad criteria.
 
-    Uses criteria given in [1]_ to validate the ratio of irradiance
-    components.
+    Uses criteria given in [1]_ to validate the ratio of irradiance components.
+    These tests are equivalent to the BSRN comparison tests [2]_.
 
     .. warning:: Not valid for night time or low irradiance. When the input
        data fall outside the test domain, the returned value is set by the
@@ -342,11 +369,15 @@ def check_irradiance_consistency_qcrad(solar_zenith, ghi, dhi, dni,
 
     References
     ----------
-    .. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control
-       Algorithm for Surface Radiation Measurements, The Open Atmospheric
-       Science Journal 2, pp. 23-37, 2008.
-
-    """
+    .. [1] C. N. Long and Y. Shi, "An Automated Quality Assessment and Control
+       Algorithm for Surface Radiation Measurements," The Open Atmospheric
+       Science Journal, vol. 2, no. 1. Bentham Science Publishers Ltd.,
+       pp. 23–37, Apr. 18, 2008. :doi:`10.2174/1874282300802010023`.
+    .. [2]  C. N. Long and E. G. Dutton, "BSRN Global Network recommended QC
+       tests, V2.0," Baseline Surface Radiation Network (BSRN),
+       Accessed: Oct. 24, 2024. [Online.] Available:
+       https://bsrn.awi.de/fileadmin/user_upload/bsrn.awi.de/Publications/BSRN_recommended_QC_tests_V2.pdf
+    """  # noqa: E501
     if not param:
         param = QCRAD_CONSISTENCY
 

pvanalytics/tests/quality/test_irradiance.py

@@ -42,28 +42,31 @@ def irradiance_qcrad():
     output = pd.DataFrame(
         columns=['ghi', 'dhi', 'dni', 'solar_zenith', 'dni_extra',
                  'ghi_limit_flag', 'dhi_limit_flag', 'dni_limit_flag',
+                 'ghi_extreme_limit_flag', 'dhi_extreme_limit_flag',
+                 'dni_extreme_limit_flag',
                  'consistent_components', 'diffuse_ratio_limit',
                  'consistent_components_outside_domain',
                  'diffuse_ratio_limit_outside_domain',
                  ],
-        data=np.array([[-100, 100, 100, 30, 1370, 0, 1, 1, 0, 0, 0, 1],
-                       [100, -100, 100, 30, 1370, 1, 0, 1, 0, 0, 1, 0],
-                       [100, 100, -100, 30, 1370, 1, 1, 0, 0, 1, 1, 1],
-                       [1000, 100, 900, 0, 1370, 1, 1, 1, 1, 1, 1, 1],
-                       [1000, 200, 800, 15, 1370, 1, 1, 1, 1, 1, 1, 1],
-                       [1000, 200, 800, 60, 1370, 0, 1, 1, 0, 1, 0, 1],
-                       [1000, 300, 850, 80, 1370, 0, 0, 1, 0, 1, 0, 1],
-                       [1000, 500, 800, 90, 1370, 0, 0, 1, 0, 1, 0, 1],
-                       [500, 100, 1100, 0, 1370, 1, 1, 1, 0, 1, 0, 1],
-                       [1000, 300, 1200, 0, 1370, 1, 1, 1, 0, 1, 0, 1],
-                       [500, 600, 100, 60, 1370, 1, 1, 1, 0, 0, 0, 0],
-                       [500, 600, 400, 80, 1370, 0, 0, 1, 0, 0, 0, 0],
-                       [500, 500, 300, 80, 1370, 0, 0, 1, 1, 1, 1, 1],
-                       [0, 0, 0, 93, 1370, 1, 1, 1, 0, 0, 1, 1],
-                       [100, 100, 0, 95, 1370, 0, 0, 1, 0, 0, 1, 1],
+        data=np.array([[-100, 100, 100, 30, 1370, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1],  # noqa: E501
+                       [100, -100, 100, 30, 1370, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0],  # noqa: E501
+                       [100, 100, -100, 30, 1370, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1],  # noqa: E501
+                       [1000, 100, 900, 0, 1370, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                       [1000, 200, 800, 15, 1370, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],  # noqa: E501
+                       [1000, 200, 800, 60, 1370, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1],  # noqa: E501
+                       [1000, 300, 850, 80, 1370, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1],  # noqa: E501
+                       [1000, 500, 800, 90, 1370, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1],  # noqa: E501
+                       [500, 100, 1100, 0, 1370, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1],
+                       [1000, 300, 1200, 0, 1370, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1],  # noqa: E501
+                       [500, 600, 100, 60, 1370, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0],
+                       [500, 600, 400, 80, 1370, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0],
+                       [500, 500, 300, 80, 1370, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1],
+                       [0, 0, 0, 93, 1370, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1],
+                       [100, 100, 0, 95, 1370, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1]
                        ]))
     dtypes = ['float64', 'float64', 'float64', 'float64', 'float64',
-              'bool', 'bool', 'bool', 'bool', 'bool', 'bool', 'bool']
+              'bool', 'bool', 'bool', 'bool', 'bool', 'bool', 'bool', 'bool',
+              'bool', 'bool']
     for (col, typ) in zip(output.columns, dtypes):
         output[col] = output[col].astype(typ)
     return output
@@ -88,6 +91,13 @@ def test_check_ghi_limits_qcrad(irradiance_qcrad):
                                                 expected['dni_extra'])
     assert_series_equal(ghi_out, ghi_out_expected, check_names=False)
 
+    ghi_out_extreme = irradiance.check_ghi_limits_qcrad(
+        expected['ghi'], expected['solar_zenith'], expected['dni_extra'],
+        limits='extreme')
+    ghi_extreme_out_expected = expected['ghi_extreme_limit_flag']
+    assert_series_equal(
+        ghi_out_extreme, ghi_extreme_out_expected, check_names=False)
+
 
 def test_check_dhi_limits_qcrad(irradiance_qcrad):
     """Test that QCRad identifies out of bounds DHI values.
@@ -108,6 +118,13 @@ def test_check_dhi_limits_qcrad(irradiance_qcrad):
                                                 expected['dni_extra'])
     assert_series_equal(dhi_out, dhi_out_expected, check_names=False)
 
+    dhi_out_extreme = irradiance.check_dhi_limits_qcrad(
+        expected['dhi'], expected['solar_zenith'], expected['dni_extra'],
+        limits='extreme')
+    dhi_extreme_out_expected = expected['dhi_extreme_limit_flag']
+    assert_series_equal(
+        dhi_out_extreme, dhi_extreme_out_expected, check_names=False)
+
 
 def test_check_dni_limits_qcrad(irradiance_qcrad):
     """Test that QCRad identifies out of bounds DNI values.
@@ -128,6 +145,13 @@ def test_check_dni_limits_qcrad(irradiance_qcrad):
                                                 expected['dni_extra'])
     assert_series_equal(dni_out, dni_out_expected, check_names=False)
 
+    dni_out_extreme = irradiance.check_dni_limits_qcrad(
+        expected['dni'], expected['solar_zenith'], expected['dni_extra'],
+        limits='extreme')
+    dni_extreme_out_expected = expected['dni_extreme_limit_flag']
+    assert_series_equal(
+        dni_out_extreme, dni_extreme_out_expected, check_names=False)
+
 
 def test_check_irradiance_limits_qcrad(irradiance_qcrad):
     """Test different input combinations to check_irradiance_limits_qcrad.
@@ -162,6 +186,20 @@ def test_check_irradiance_limits_qcrad(irradiance_qcrad):
     assert_series_equal(dni_out, dni_out_expected, check_names=False)
 
 
+def test_check_irradiance_limits_qcrad_extreme(irradiance_qcrad):
+    """Test different input combinations to check_irradiance_limits_qcrad.
+    """
+    expected = irradiance_qcrad
+
+    ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad(
+        expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi'],
+        dhi=expected['dhi'], dni=expected['dni'], limits='extreme')
+
+    assert_series_equal(ghi_out, expected['ghi_extreme_limit_flag'], check_names=False)  # noqa: E501
+    assert_series_equal(dhi_out, expected['dhi_extreme_limit_flag'], check_names=False)  # noqa: E501
+    assert_series_equal(dni_out, expected['dni_extreme_limit_flag'], check_names=False)  # noqa: E501
+
+
 def test_check_irradiance_consistency_qcrad(irradiance_qcrad):
     """Test that QCRad identifies consistent irradiance measurements.