WIP: Some potential changes to fx to support estimates

Author: peterhbromley

zipline/data/fx/base.py

@@ -127,15 +127,18 @@ def get_rates_columnar(self, rate, quote, bases, dts):
             may appear multiple times.
         dts : np.DatetimeIndex
             Datetimes for which to load rates. The same value may appear
-            multiple times, but datetimes must be sorted in ascending order and
-            localized to UTC.
+            multiple times, and datetimes do not need to be sorted because
+            `np.unique` sorts in ascending order.
         """
         if len(bases) != len(dts):
             raise ValueError(
                 "len(bases) ({}) != len(dts) ({})".format(len(bases), len(dts))
             )
 
-        unique_bases, bases_ix = np.unique(bases, return_inverse=True)
+        # TODO: Casting `bases` to str here is a temporary fix for the bug
+        # where having any `None` in `bases` causes `np.unique` to error.
+        unique_bases, bases_ix = np.unique(bases.astype(str),
+                                           return_inverse=True)
         unique_dts, dts_ix = np.unique(dts.values, return_inverse=True)
         rates_2d = self.get_rates(
             rate,

zipline/data/fx/hdf5.py

@@ -189,7 +189,10 @@ def get_rates(self, rate, quote, bases, dts):
         if rate == DEFAULT_FX_RATE:
             rate = self._default_rate
 
-        self._check_dts(self.dts, dts)
+        # TODO: Commenting this _check_dts out for now to bypass the
+        # estimates loader date bounds issue.  Will need to address
+        # this before finalizing anything.
+        # self._check_dts(self.dts, dts)
 
         row_ixs = self.dts.searchsorted(dts, side='right') - 1
         col_ixs = self.currencies.get_indexer(bases)
@@ -212,7 +215,7 @@ def get_rates(self, rate, quote, bases, dts):
         # array, so it's easier to pull all columns and reindex in memory. For
         # rows, however, a quick and easy optimization is to pull just the
         # slice from min(row_ixs) to max(row_ixs).
-        min_row = row_ixs[0]
+        min_row = max(row_ixs[0], 0)
         max_row = row_ixs[-1]
         rows = dataset[min_row:max_row + 1]  # +1 to be inclusive of end
 
@@ -221,6 +224,11 @@ def get_rates(self, rate, quote, bases, dts):
         # get_indexer returns -1 for failed lookups. Fill these in with NaN.
         out[:, col_ixs == -1] = np.nan
 
+        # TODO: searchsorted also gives -1 for failed lookups. However, these
+        # failed lookups arise due to the estimates date bounds bug that we
+        # have not yet addressed, so this is a temporary fix.
+        out[row_ixs == -1, :] = np.nan
+
         return out
 
     def _check_dts(self, stored, requested):