MAINT: Use object arrays with None for currency codes.

Rather than trying to use S3s everywhere, which is annoying in Python 3 and
makes it harder to represent missing data, just use object arrays with None as
the missing value. This is the representation we want anyway for loading
currency data in pipelines, and the main downsides are performance (which
doesn't appear to be meaningfully affected) and difficulty with sorting, which
we don't need to do (at least right now).

Author: Scott Sanderson

tests/data/test_daily_bars.py

@@ -36,7 +36,6 @@
 from toolz import merge
 from trading_calendars import get_calendar
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.bar_reader import (
     NoDataAfterDate,
     NoDataBeforeDate,
@@ -188,7 +187,7 @@ def make_equity_daily_bar_currency_codes(cls, country_code, sids):
         # Evenly distribute choices among ``sids``.
         choices = cls.DAILY_BARS_TEST_CURRENCIES[country_code]
         codes = list(islice(cycle(choices), len(sids)))
-        return Series(index=sids, data=np.array(codes, dtype='S3'))
+        return Series(index=sids, data=np.array(codes, dtype=object))
 
     @classproperty
     def holes(cls):
@@ -531,6 +530,10 @@ def test_listing_currency(self):
         ).values
         assert_equal(all_results, all_expected)
 
+        self.assertEqual(all_results.dtype, np.dtype(object))
+        for code in all_results:
+            self.assertIsInstance(code, str)
+
         # Check all possible subsets of assets.
         for indices in map(list, powerset(range(len(all_assets)))):
             # Empty queries aren't currently supported.
@@ -557,10 +560,7 @@ def test_listing_currency_for_nonexistent_asset(self):
         # queries.
         mixed = np.array(invalid_sids + [valid_sid])
         result = self.daily_bar_reader.currency_codes(mixed)
-        expected = np.array(
-            [MISSING_CURRENCY_CODE] * 2 + [valid_currency],
-            dtype='S3'
-        )
+        expected = np.array([None] * 2 + [valid_currency])
         assert_equal(result, expected)
 
 

tests/data/test_fx.py

@@ -146,7 +146,7 @@ def test_read_before_start_date(self):
 
             for rate in self.FX_RATES_RATE_NAMES:
                 quote = 'USD'
-                bases = np.array(['CAD'], dtype='S3')
+                bases = np.array(['CAD'], dtype=object)
                 dts = pd.DatetimeIndex([bad_date])
                 with self.assertRaises(ValueError):
                     self.reader.get_rates(rate, quote, bases, dts)
@@ -157,7 +157,7 @@ def test_read_after_end_date(self):
 
             for rate in self.FX_RATES_RATE_NAMES:
                 quote = 'USD'
-                bases = np.array(['CAD'], dtype='S3')
+                bases = np.array(['CAD'], dtype=object)
                 dts = pd.DatetimeIndex([bad_date])
                 with self.assertRaises(ValueError):
                     self.reader.get_rates(rate, quote, bases, dts)

zipline/currency.py

@@ -4,10 +4,6 @@
 _ALL_CURRENCIES = {}
 
 
-# Special sentinel used to represent unknown or missing currencies.
-MISSING_CURRENCY_CODE = 'XXX'
-
-
 @total_ordering
 class Currency(object):
     """A currency identifier, as defined by ISO-4217.
@@ -28,8 +24,7 @@ def __new__(cls, code):
         try:
             return _ALL_CURRENCIES[code]
         except KeyError:
-            # This isn't a real
-            if code == MISSING_CURRENCY_CODE:
+            if code is None:
                 name = "NO CURRENCY"
             else:
                 try:

zipline/data/bcolz_daily_bars.py

@@ -34,7 +34,6 @@
 from toolz import compose
 from trading_calendars import get_calendar
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.session_bars import CurrencyAwareSessionBarReader
 from zipline.data.bar_reader import (
     NoDataAfterDate,
@@ -708,13 +707,13 @@ def get_value(self, sid, dt, field):
 
     def currency_codes(self, sids):
         # XXX: This is pretty inefficient. This reader doesn't really support
-        # country codes, so we always either return USD or
-        # MISSING_CURRENCY_CODE if we don't know about the sid at all.
+        # country codes, so we always either return USD or None if we don't
+        # know about the sid at all.
         first_rows = self._first_rows
         out = []
         for sid in sids:
             if sid in first_rows:
                 out.append('USD')
             else:
-                out.append(MISSING_CURRENCY_CODE)
-        return np.array(out, dtype='S3')
+                out.append(None)
+        return np.array(out, dtype=object)

zipline/data/fx/hdf5.py

@@ -99,9 +99,9 @@
 from logbook import Logger
 import numpy as np
 import pandas as pd
-import six
 
 from zipline.utils.memoize import lazyval
+from zipline.utils.numpy_utils import bytes_array_to_native_str_object_array
 
 from .base import FXRateReader, DEFAULT_FX_RATE
 
@@ -177,13 +177,9 @@ def currencies(self):
         """
         # Currencies are stored as fixed-length bytes in the file, but we want
         # `str` objects in memory.
-        byte_strings = self._group[INDEX][CURRENCIES][:]
-        if six.PY3:
-            values = [c.decode('ascii') for c in byte_strings]
-        else:
-            values = byte_strings.astype(object)
-
-        return pd.Index(values)
+        bytes_array = self._group[INDEX][CURRENCIES][:]
+        objects = bytes_array_to_native_str_object_array(bytes_array)
+        return pd.Index(objects)
 
     def get_rates(self, rate, quote, bases, dts):
         """Get rates to convert ``bases`` into ``quote``.
@@ -281,6 +277,13 @@ def _write_metadata(self):
     def _write_index_group(self, dts, currencies):
         """Write content of /index.
         """
+        if not is_sorted_ascending(dts):
+            raise ValueError("dts is not sorted")
+
+        for c in currencies:
+            if not isinstance(c, str) or len(c) != 3:
+                raise ValueError("Invalid currency: {!r}".format(c))
+
         index_group = self._group.create_group(INDEX)
 
         self._log_writing(INDEX, DTS)

zipline/data/hdf5_daily_bars.py

@@ -107,7 +107,6 @@
 from six import iteritems, raise_from, viewkeys
 from six.moves import reduce
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.bar_reader import (
     NoDataAfterDate,
     NoDataBeforeDate,
@@ -116,6 +115,7 @@
 )
 from zipline.data.session_bars import CurrencyAwareSessionBarReader
 from zipline.utils.memoize import lazyval
+from zipline.utils.numpy_utils import bytes_array_to_native_str_object_array
 from zipline.utils.pandas_utils import check_indexes_all_same
 
 
@@ -696,7 +696,8 @@ def asset_end_dates(self):
 
     @lazyval
     def _currency_codes(self):
-        return self._country_group[CURRENCY][CODE][:]
+        bytes_array = self._country_group[CURRENCY][CODE][:]
+        return bytes_array_to_native_str_object_array(bytes_array)
 
     def currency_codes(self, sids):
         """Get currencies in which prices are quoted for the requested sids.
@@ -708,7 +709,7 @@ def currency_codes(self, sids):
 
         Returns
         -------
-        currency_codes : np.array[S3]
+        currency_codes : np.array[object]
             Array of currency codes for listing currencies of ``sids``.
         """
         # Find the index of requested sids in our stored sids.
@@ -720,7 +721,7 @@ def currency_codes(self, sids):
         # fails. Fill these sids with the special "missing" sentinel.
         not_found = (self.sids[ixs] != sids)
 
-        result[not_found] = MISSING_CURRENCY_CODE
+        result[not_found] = None
 
         return result
 

zipline/data/session_bars.py

@@ -39,7 +39,8 @@ class CurrencyAwareSessionBarReader(SessionBarReader):
 
     @abstractmethod
     def currency_codes(self, sids):
-        """Get currencies in which prices are quoted for the requested sids.
+        """
+        Get currencies in which prices are quoted for the requested sids.
 
         Assumes that a sid's prices are always quoted in a single currency.
 
@@ -50,9 +51,8 @@ def currency_codes(self, sids):
 
         Returns
         -------
-        currency_codes : np.array[S3]
+        currency_codes : np.array[object]
             Array of currency codes for listing currencies of
-            ``sids``. Implementations should return
-            zipline.currency.MISSING_CURRENCY_CODE for sids whose currency is
-            unknown.
+            ``sids``. Implementations should return None for sids whose
+            currency is unknown.
         """

zipline/pipeline/loaders/equity_pricing_loader.py

@@ -16,13 +16,9 @@
 from interface import implements
 from numpy import iinfo, uint32, multiply
 
-from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.fx import ExplodingFXRateReader
 from zipline.lib.adjusted_array import AdjustedArray
-from zipline.utils.numpy_utils import (
-    repeat_first_axis,
-    bytes_array_to_native_str_object_array,
-)
+from zipline.utils.numpy_utils import repeat_first_axis
 
 from .base import PipelineLoader
 from .utils import shift_dates
@@ -123,12 +119,7 @@ def load_adjusted_array(self, domain, columns, dates, sids, mask):
             )
 
         for c in currency_cols:
-            codes_1d = bytes_array_to_native_str_object_array(
-                self.raw_price_reader.currency_codes(sids)
-            )
-            # XXX: Should this just be the contract of `currency_codes`?
-            codes_1d[codes_1d == MISSING_CURRENCY_CODE] = None
-
+            codes_1d = self.raw_price_reader.currency_codes(sids)
             codes = repeat_first_axis(codes_1d, len(dates))
             out[c] = AdjustedArray(
                 codes,

zipline/testing/fixtures.py

@@ -2182,7 +2182,7 @@ def write_h5_fx_rates(cls, path):
 
             writer.write(
                 dts=sessions.values,
-                currencies=np.array(cls.FX_RATES_CURRENCIES, dtype='S3'),
+                currencies=np.array(cls.FX_RATES_CURRENCIES, dtype=object),
                 data=fx_data,
             )