lookup_linke_turbidity speed improvement, mat to h5 (#442)

lookup_linke_turbidity speed improvement, mat to h5

Author: cedricleroy

appveyor.yml

@@ -28,7 +28,7 @@ install:
     - cmd: conda info -a
 
     # install depenencies
-    - cmd: conda create -n test_env --yes --quiet python=%PYTHON_VERSION% pip numpy scipy pandas nose pytest pytz ephem numba siphon -c conda-forge
+    - cmd: conda create -n test_env --yes --quiet python=%PYTHON_VERSION% pip numpy scipy pytables pandas nose pytest pytz ephem numba siphon -c conda-forge
     - cmd: activate test_env
     - cmd: python --version
     - cmd: conda list

ci/requirements-py27.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=2.7
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     - ephem

ci/requirements-py34.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=3.4
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     - ephem

ci/requirements-py35.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=3.5
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     - ephem

ci/requirements-py36.yml

@@ -6,6 +6,7 @@ dependencies:
     - python=3.6
     - numpy
     - scipy
+    - pytables
     - pandas
     - pytz
     #- ephem

docs/environment.yml

@@ -7,6 +7,7 @@ dependencies:
     - mock  # needed for local python 2.7 builds
     - numpy=1.11.2
     - scipy
+    - tables
     - pandas=0.19.1
     - pytz
     - ephem

docs/sphinx/source/clearsky.rst

@@ -136,24 +136,24 @@ the year. You could run it in a loop to create plots for all months.
 
     In [1]: import os
 
-    In [1]: import scipy.io
+    In [1]: import tables
 
     In [1]: pvlib_path = os.path.dirname(os.path.abspath(pvlib.clearsky.__file__))
 
-    In [1]: filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.mat')
-
-    In [1]: mat = scipy.io.loadmat(filepath)
+    In [1]: filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.h5')
 
     # data is in units of 20 x turbidity
-    In [1]: linke_turbidity_table = mat['LinkeTurbidity']  # / 20.   # crashes on rtd
+    In [1]: lt_h5_file = tables.open_file(filepath)
 
     In [1]: def plot_turbidity_map(month, vmin=1, vmax=100):
        ...:     plt.figure();
-       ...:     plt.imshow(linke_turbidity_table[:, :, month-1], vmin=vmin, vmax=vmax);
+       ...:     plt.imshow(lt_h5_file.root.LinkeTurbidity[:, :, month-1], vmin=vmin, vmax=vmax);
        ...:     plt.title('Linke turbidity x 20, ' + calendar.month_name[month]);
        ...:     plt.colorbar(shrink=0.5);
        ...:     plt.tight_layout();
 
+    In [1]: lt_h5_file.close()
+
     @savefig turbidity-1.png width=10in
     In [1]: plot_turbidity_map(1)
 

docs/sphinx/source/whatsnew/v0.5.2.rst

@@ -9,7 +9,8 @@ API Changes
 
 Enhancements
 ~~~~~~~~~~~~
-*
+* Improve clearsky.lookup_linke_turbidity speed, changing .mat source
+  file to .h5 (:issue:`437`)
 
 Bug fixes
 ~~~~~~~~~
@@ -42,5 +43,4 @@ Contributors
 * Will Holmgren
 * KonstantinTr
 * Anton Driesse
-
-
+* Cedric Leroy

pvlib/clearsky.py

@@ -153,7 +153,7 @@ def ineichen(apparent_zenith, airmass_absolute, linke_turbidity,
 def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
                            interp_turbidity=True):
     """
-    Look up the Linke Turibidity from the ``LinkeTurbidities.mat``
+    Look up the Linke Turibidity from the ``LinkeTurbidities.h5``
     data file supplied with pvlib.
 
     Parameters
@@ -165,18 +165,18 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
     longitude : float
 
     filepath : None or string, default None
-        The path to the ``.mat`` file.
+        The path to the ``.h5`` file.
 
     interp_turbidity : bool, default True
         If ``True``, interpolates the monthly Linke turbidity values
-        found in ``LinkeTurbidities.mat`` to daily values.
+        found in ``LinkeTurbidities.h5`` to daily values.
 
     Returns
     -------
     turbidity : Series
     """
 
-    # The .mat file 'LinkeTurbidities.mat' contains a single 2160 x 4320 x 12
+    # The .h5 file 'LinkeTurbidities.h5' contains a single 2160 x 4320 x 12
     # matrix of type uint8 called 'LinkeTurbidity'. The rows represent global
     # latitudes from 90 to -90 degrees; the columns represent global longitudes
     # from -180 to 180; and the depth (third dimension) represents months of
@@ -194,18 +194,15 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
     # 1st column: 179.9583 W, 2nd column: 179.875 W
 
     try:
-        import scipy.io
+        import tables
     except ImportError:
-        raise ImportError('The Linke turbidity lookup table requires scipy. '
+        raise ImportError('The Linke turbidity lookup table requires tables. '
                           'You can still use clearsky.ineichen if you '
                           'supply your own turbidities.')
 
     if filepath is None:
         pvlib_path = os.path.dirname(os.path.abspath(__file__))
-        filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.mat')
-
-    mat = scipy.io.loadmat(filepath)
-    linke_turbidity_table = mat['LinkeTurbidity']
+        filepath = os.path.join(pvlib_path, 'data', 'LinkeTurbidities.h5')
 
     latitude_index = (
         np.around(_linearly_scale(latitude, 90, -90, 0, 2160))
@@ -214,7 +211,14 @@ def lookup_linke_turbidity(time, latitude, longitude, filepath=None,
         np.around(_linearly_scale(longitude, -180, 180, 0, 4320))
         .astype(np.int64))
 
-    lts = linke_turbidity_table[latitude_index][longitude_index]
+    lt_h5_file = tables.open_file(filepath)
+    try:
+        lts = lt_h5_file.root.LinkeTurbidity[latitude_index, longitude_index, :]
+    except IndexError:
+        raise IndexError('Latitude should be between 90 and -90, '
+                         'longitude between -180 and 180.')
+    finally:
+        lt_h5_file.close()
 
     if interp_turbidity:
         linke_turbidity = _interpolate_turbidity(lts, time)