Merge pull request #2617 from quantopian/columnar-fx-rates

ENH: Add get_rates_columnar method to FXRatesReader.

Author: Scott Sanderson

tests/data/test_fx.py

@@ -84,10 +84,13 @@ def test_scalar_lookup(self):
             expected = self.get_expected_fx_rate_scalar(rate, quote, base, dt)
             assert_equal(result_scalar, expected)
 
+            col_result = reader.get_rates_columnar(rate, quote, bases, dts)
+            assert_equal(col_result, result.ravel())
+
             alt_result_scalar = reader.get_rate_scalar(rate, quote, base, dt)
             assert_equal(result_scalar, alt_result_scalar)
 
-    def test_vectorized_lookup(self):
+    def test_2d_lookup(self):
         rand = np.random.RandomState(42)
 
         dates = pd.date_range(self.FX_RATES_START_DATE, self.FX_RATES_END_DATE)
@@ -113,6 +116,34 @@ def test_vectorized_lookup(self):
 
                 assert_equal(result, expected)
 
+    def test_columnar_lookup(self):
+        rand = np.random.RandomState(42)
+
+        dates = pd.date_range(self.FX_RATES_START_DATE, self.FX_RATES_END_DATE)
+        rates = self.FX_RATES_RATE_NAMES + [DEFAULT_FX_RATE]
+        currencies = self.FX_RATES_CURRENCIES
+        reader = self.reader
+
+        # For every combination of rate name and quote currency...
+        for rate, quote in itertools.product(rates, currencies):
+            for N in 1, 2, 10, 200:
+                # Choose N (date, base) pairs randomly with replacement.
+                dts_raw = rand.choice(dates, N, replace=True)
+                dts = pd.DatetimeIndex(dts_raw, tz='utc').sort_values()
+                bases = rand.choice(currencies, N, replace=True)
+
+                # ... And check that we get the expected result when querying
+                # for those dates/currencies.
+                result = reader.get_rates_columnar(rate, quote, bases, dts)
+                expected = self.get_expected_fx_rates_columnar(
+                    rate,
+                    quote,
+                    bases,
+                    dts,
+                )
+
+                assert_equal(result, expected)
+
     def test_load_everything(self):
         # Sanity check for the randomized tests above: check that we get
         # exactly the rates we set up in make_fx_rates if we query for their

zipline/data/fx/base.py

@@ -9,22 +9,64 @@
 
 
 class FXRateReader(Interface):
+    """
+    Interface for reading foreign exchange (fx) rates.
+
+    An FX rate reader contains one or more distinct "rates", each of which
+    corresponds to a collection of mappings from (quote, base, dt) ->
+    float. The value produced for a given (quote, base, dt) triple is the
+    exchange rate to use when converting from ``base`` to ``quote`` on ``dt``.
+
+    The specific set of rates contained in a particular reader is
+    user-defined. We infer no particular semantics from their names, other than
+    that they are distinct rates. Examples of possible rate names might be
+    things like "bid", "mid", and "ask", or "london_close", "tokyo_close",
+    "nyse_close".
+
+    Implementations of :class:`FXRateReader` must provide at least one method::
+
+        def get_rates(self, rate, quote, bases, dts):
+
+    which takes a rate, a quote currency, an array of base currencies, and an
+    array of dts, and produces a (len(dts), len(base))-shape array containing a
+    conversion rates for all pairs in the cartesian product of bases and dts.
+
+    Given a definition of :meth:`get_rates`, this interface automatically
+    generates two additional methods::
+
+        def get_rates_scalar(self, rate, quote, base, dt):
+
+    and::
+
+        def get_rates_columnar(self, rate, quote, bases, dts):
+
+    :meth:`get_rates_scalar` takes scalar-valued ``base`` and ``dt`` values,
+    and returns a scalar float value for the requested fx rate.
+
+    :meth:`get_rates_columnar` takes parallel arrays of ``bases`` and ``dts``
+    and returns a same-length array of fx rates by performing a lookup on the
+    (base, dt) pairs drawn from zipping together ``bases``, and ``dts``. In
+    other words, its behavior is equivalent to::
+
+        def get_rates_columnnar(self, rate, quote, bases, dts):
+            out = []
+            for base, dt in zip(bases, dts):
+                out.append(self.get_rate_scalar(rate, quote, base, dt))
+            return np.array(out)
+    """
 
     def get_rates(self, rate, quote, bases, dts):
         """
-        Get rates to convert ``bases`` into ``quote``.
+        Load a 2D array of fx rates.
 
         Parameters
         ----------
         rate : str
-            Rate type to load. Readers intended for use with the Pipeline API
-            should support at least ``zipline.data.fx.DEFAULT_FX_RATE``, which
-            will be used by default for Pipeline API terms that don't specify a
-            specific rate.
+            Name of the rate to load.
         quote : str
             Currency code of the currency to convert into.
         bases : np.array[object]
-            Array of codes of the currencies to convert from. A single currency
+            Array of codes of the currencies to convert from. The same currency
             may appear multiple times.
         dts : pd.DatetimeIndex
             Datetimes for which to load rates. Must be sorted in ascending
@@ -42,15 +84,13 @@ def get_rates(self, rate, quote, bases, dts):
 
     @default
     def get_rate_scalar(self, rate, quote, base, dt):
-        """Scalar version of ``get_rates``.
+        """
+        Load a scalar FX rate value.
 
         Parameters
         ----------
         rate : str
-            Rate type to load. Readers intended for use with the Pipeline API
-            should support at least ``zipline.data.fx.DEFAULT_FX_RATE``, which
-            will be used by default for Pipeline API terms that don't specify a
-            specific rate.
+            Name of the rate to load.
         quote : str
             Currency code of the currency to convert into.
         base : str
@@ -63,10 +103,44 @@ def get_rate_scalar(self, rate, quote, base, dt):
         rate : np.float64
             Exchange rate from base -> quote on dt.
         """
-        rates_array = self.get_rates(
+        rates_2d = self.get_rates(
             rate,
             quote,
             bases=np.array([base], dtype=object),
             dts=pd.DatetimeIndex([dt], tz='UTC'),
         )
-        return rates_array[0, 0]
+        return rates_2d[0, 0]
+
+    @default
+    def get_rates_columnar(self, rate, quote, bases, dts):
+        """
+        Load a 1D array of FX rates.
+
+        Parameters
+        ----------
+        rate : str
+            Name of the rate to load.
+        quote : str
+            Currency code of the currency to convert into.
+        bases : np.array[object]
+            Array of codes of the currencies to convert from. The same currency
+            may appear multiple times.
+        dts : np.DatetimeIndex
+            Datetimes for which to load rates. The same value may appear
+            multiple times, but datetimes must be sorted in ascending order and
+            localized to UTC.
+        """
+        if len(bases) != len(dts):
+            raise ValueError(
+                "len(bases) ({}) != len(dts) ({})".format(len(bases), len(dts))
+            )
+
+        unique_bases, bases_ix = np.unique(bases, return_inverse=True)
+        unique_dts, dts_ix = np.unique(dts.values, return_inverse=True)
+        rates_2d = self.get_rates(
+            rate,
+            quote,
+            unique_bases,
+            pd.DatetimeIndex(unique_dts, tz='utc')
+        )
+        return rates_2d[dts_ix, bases_ix]

zipline/testing/fixtures.py

@@ -2222,6 +2222,15 @@ def get_expected_fx_rates(cls, rate, quote, bases, dts):
 
         return out
 
+    @classmethod
+    def get_expected_fx_rates_columnar(cls, rate, quote, bases, dts):
+        assert len(bases) == len(dts)
+        rates = [
+            cls.get_expected_fx_rate_scalar(rate, quote, base, dt)
+            for base, dt in zip(bases, dts)
+        ]
+        return np.array(rates, dtype='float64')
+
 
 def fast_get_loc_ffilled(dts, dt):
     """