Merge pull request #2612 from quantopian/last-currency-fix

More Currency Fixes

Author: Scott Sanderson

tests/data/test_daily_bars.py

@@ -36,7 +36,6 @@
 from toolz import merge
 from trading_calendars import get_calendar
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.bar_reader import (
     NoDataAfterDate,
     NoDataBeforeDate,
@@ -188,7 +187,7 @@ def make_equity_daily_bar_currency_codes(cls, country_code, sids):
         # Evenly distribute choices among ``sids``.
         choices = cls.DAILY_BARS_TEST_CURRENCIES[country_code]
         codes = list(islice(cycle(choices), len(sids)))
-        return Series(index=sids, data=np.array(codes, dtype='S3'))
+        return Series(index=sids, data=np.array(codes, dtype=object))
 
     @classproperty
     def holes(cls):
@@ -531,6 +530,10 @@ def test_listing_currency(self):
         ).values
         assert_equal(all_results, all_expected)
 
+        self.assertEqual(all_results.dtype, np.dtype(object))
+        for code in all_results:
+            self.assertIsInstance(code, str)
+
         # Check all possible subsets of assets.
         for indices in map(list, powerset(range(len(all_assets)))):
             # Empty queries aren't currently supported.
@@ -557,10 +560,7 @@ def test_listing_currency_for_nonexistent_asset(self):
         # queries.
         mixed = np.array(invalid_sids + [valid_sid])
         result = self.daily_bar_reader.currency_codes(mixed)
-        expected = np.array(
-            [MISSING_CURRENCY_CODE] * 2 + [valid_currency],
-            dtype='S3'
-        )
+        expected = np.array([None] * 2 + [valid_currency])
         assert_equal(result, expected)
 
 

tests/data/test_fx.py

@@ -146,7 +146,7 @@ def test_read_before_start_date(self):
 
             for rate in self.FX_RATES_RATE_NAMES:
                 quote = 'USD'
-                bases = np.array(['CAD'], dtype='S3')
+                bases = np.array(['CAD'], dtype=object)
                 dts = pd.DatetimeIndex([bad_date])
                 with self.assertRaises(ValueError):
                     self.reader.get_rates(rate, quote, bases, dts)
@@ -157,7 +157,7 @@ def test_read_after_end_date(self):
 
             for rate in self.FX_RATES_RATE_NAMES:
                 quote = 'USD'
-                bases = np.array(['CAD'], dtype='S3')
+                bases = np.array(['CAD'], dtype=object)
                 dts = pd.DatetimeIndex([bad_date])
                 with self.assertRaises(ValueError):
                     self.reader.get_rates(rate, quote, bases, dts)

zipline/currency.py

@@ -4,10 +4,6 @@
 _ALL_CURRENCIES = {}
 
 
-# Special sentinel used to represent unknown or missing currencies.
-MISSING_CURRENCY_CODE = 'XXX'
-
-
 @total_ordering
 class Currency(object):
     """A currency identifier, as defined by ISO-4217.
@@ -28,8 +24,7 @@ def __new__(cls, code):
         try:
             return _ALL_CURRENCIES[code]
         except KeyError:
-            # This isn't a real
-            if code == MISSING_CURRENCY_CODE:
+            if code is None:
                 name = "NO CURRENCY"
             else:
                 try:

zipline/data/bcolz_daily_bars.py

@@ -34,7 +34,6 @@
 from toolz import compose
 from trading_calendars import get_calendar
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.session_bars import CurrencyAwareSessionBarReader
 from zipline.data.bar_reader import (
     NoDataAfterDate,
@@ -708,13 +707,13 @@ def get_value(self, sid, dt, field):
 
     def currency_codes(self, sids):
         # XXX: This is pretty inefficient. This reader doesn't really support
-        # country codes, so we always either return USD or
-        # MISSING_CURRENCY_CODE if we don't know about the sid at all.
+        # country codes, so we always either return USD or None if we don't
+        # know about the sid at all.
         first_rows = self._first_rows
         out = []
         for sid in sids:
             if sid in first_rows:
                 out.append('USD')
             else:
-                out.append(MISSING_CURRENCY_CODE)
-        return np.array(out, dtype='S3')
+                out.append(None)
+        return np.array(out, dtype=object)

zipline/data/fx/hdf5.py

@@ -99,9 +99,9 @@
 from logbook import Logger
 import numpy as np
 import pandas as pd
-import six
 
 from zipline.utils.memoize import lazyval
+from zipline.utils.numpy_utils import bytes_array_to_native_str_object_array
 
 from .base import FXRateReader, DEFAULT_FX_RATE
 
@@ -177,13 +177,9 @@ def currencies(self):
         """
         # Currencies are stored as fixed-length bytes in the file, but we want
         # `str` objects in memory.
-        byte_strings = self._group[INDEX][CURRENCIES][:]
-        if six.PY3:
-            values = [c.decode('ascii') for c in byte_strings]
-        else:
-            values = byte_strings.astype(object)
-
-        return pd.Index(values)
+        bytes_array = self._group[INDEX][CURRENCIES][:]
+        objects = bytes_array_to_native_str_object_array(bytes_array)
+        return pd.Index(objects)
 
     def get_rates(self, rate, quote, bases, dts):
         """Get rates to convert ``bases`` into ``quote``.
@@ -281,6 +277,13 @@ def _write_metadata(self):
     def _write_index_group(self, dts, currencies):
         """Write content of /index.
         """
+        if not is_sorted_ascending(dts):
+            raise ValueError("dts is not sorted")
+
+        for c in currencies:
+            if not isinstance(c, str) or len(c) != 3:
+                raise ValueError("Invalid currency: {!r}".format(c))
+
         index_group = self._group.create_group(INDEX)
 
         self._log_writing(INDEX, DTS)

zipline/data/hdf5_daily_bars.py

@@ -107,7 +107,6 @@
 from six import iteritems, raise_from, viewkeys
 from six.moves import reduce
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.bar_reader import (
     NoDataAfterDate,
     NoDataBeforeDate,
@@ -116,6 +115,7 @@
 )
 from zipline.data.session_bars import CurrencyAwareSessionBarReader
 from zipline.utils.memoize import lazyval
+from zipline.utils.numpy_utils import bytes_array_to_native_str_object_array
 from zipline.utils.pandas_utils import check_indexes_all_same
 
 
@@ -696,7 +696,8 @@ def asset_end_dates(self):
 
     @lazyval
     def _currency_codes(self):
-        return self._country_group[CURRENCY][CODE][:]
+        bytes_array = self._country_group[CURRENCY][CODE][:]
+        return bytes_array_to_native_str_object_array(bytes_array)
 
     def currency_codes(self, sids):
         """Get currencies in which prices are quoted for the requested sids.
@@ -708,7 +709,7 @@ def currency_codes(self, sids):
 
         Returns
         -------
-        currency_codes : np.array[S3]
+        currency_codes : np.array[object]
             Array of currency codes for listing currencies of ``sids``.
         """
         # Find the index of requested sids in our stored sids.
@@ -720,7 +721,7 @@ def currency_codes(self, sids):
         # fails. Fill these sids with the special "missing" sentinel.
         not_found = (self.sids[ixs] != sids)
 
-        result[not_found] = MISSING_CURRENCY_CODE
+        result[not_found] = None
 
         return result
 

zipline/data/session_bars.py

@@ -39,7 +39,8 @@ class CurrencyAwareSessionBarReader(SessionBarReader):
 
     @abstractmethod
     def currency_codes(self, sids):
-        """Get currencies in which prices are quoted for the requested sids.
+        """
+        Get currencies in which prices are quoted for the requested sids.
 
         Assumes that a sid's prices are always quoted in a single currency.
 
@@ -50,9 +51,8 @@ def currency_codes(self, sids):
 
         Returns
         -------
-        currency_codes : np.array[S3]
+        currency_codes : np.array[object]
             Array of currency codes for listing currencies of
-            ``sids``. Implementations should return
-            zipline.currency.MISSING_CURRENCY_CODE for sids whose currency is
-            unknown.
+            ``sids``. Implementations should return None for sids whose
+            currency is unknown.
         """

zipline/pipeline/loaders/equity_pricing_loader.py

@@ -16,13 +16,9 @@
 from interface import implements
 from numpy import iinfo, uint32, multiply
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.fx import ExplodingFXRateReader
 from zipline.lib.adjusted_array import AdjustedArray
-from zipline.utils.numpy_utils import (
-    repeat_first_axis,
-    bytes_array_to_native_str_object_array,
-)
+from zipline.utils.numpy_utils import repeat_first_axis
 
 from .base import PipelineLoader
 from .utils import shift_dates
@@ -123,12 +119,7 @@ def load_adjusted_array(self, domain, columns, dates, sids, mask):
             )
 
         for c in currency_cols:
-            codes_1d = bytes_array_to_native_str_object_array(
-                self.raw_price_reader.currency_codes(sids)
-            )
-            # XXX: Should this just be the contract of `currency_codes`?
-            codes_1d[codes_1d == MISSING_CURRENCY_CODE] = None
-
+            codes_1d = self.raw_price_reader.currency_codes(sids)
             codes = repeat_first_axis(codes_1d, len(dates))
             out[c] = AdjustedArray(
                 codes,
@@ -141,8 +132,8 @@ def load_adjusted_array(self, domain, columns, dates, sids, mask):
     @property
     def currency_aware(self):
         # Tell the pipeline engine that this loader supports currency
-        # conversion.
-        return True
+        # conversion if we have a non-dummy fx rates reader.
+        return not isinstance(self.fx_reader, ExplodingFXRateReader)
 
     def _inplace_currency_convert(self, columns, arrays, dates, sids):
         """

zipline/testing/fixtures.py

@@ -2182,7 +2182,7 @@ def write_h5_fx_rates(cls, path):
 
             writer.write(
                 dts=sessions.values,
-                currencies=np.array(cls.FX_RATES_CURRENCIES, dtype='S3'),
+                currencies=np.array(cls.FX_RATES_CURRENCIES, dtype=object),
                 data=fx_data,
             )