BUG: Handle currency conversions for unknown sids.

Don't crash on queries for currency codes of possibly-unknown sids on calls to
`SessionBarReader.currency_codes`. When a currency code is requested for an
unknown sid, we return zipline.currency.MISSING_CURRENCY_CODE for that sid.

Author: Scott Sanderson

tests/data/test_daily_bars.py

@@ -36,6 +36,7 @@
 from toolz import merge
 from trading_calendars import get_calendar
 
+from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.bar_reader import (
     NoDataAfterDate,
     NoDataBeforeDate,
@@ -541,6 +542,27 @@ def test_listing_currency(self):
 
             assert_equal(results, expected)
 
+    def test_listing_currency_for_nonexistent_asset(self):
+        reader = self.daily_bar_reader
+
+        valid_sid = max(self.assets)
+        valid_currency = reader.currency_codes(np.array([valid_sid]))[0]
+        invalid_sids = [-1, -2]
+
+        # XXX: We currently require at least one valid sid here, because the
+        # MultiCountryDailyBarReader needs one valid sid to be able to dispatch
+        # to a child reader. We could probably make that work, but there are no
+        # real-world cases where we expect to get all-invalid currency queries,
+        # so it's unclear whether we should do work to explicitly support such
+        # queries.
+        mixed = np.array(invalid_sids + [valid_sid])
+        result = self.daily_bar_reader.currency_codes(mixed)
+        expected = np.array(
+            [MISSING_CURRENCY_CODE] * 2 + [valid_currency],
+            dtype='S3'
+        )
+        assert_equal(result, expected)
+
 
 class BcolzDailyBarTestCase(WithBcolzEquityDailyBarReader, _DailyBarsTestCase):
     EQUITY_DAILY_BAR_COUNTRY_CODES = ['US']

zipline/currency.py

@@ -1,31 +1,11 @@
-from functools import partial, total_ordering
-
+from functools import total_ordering
 from iso4217 import Currency as ISO4217Currency
 
-import numpy as np
-
 _ALL_CURRENCIES = {}
 
 
-def strs_to_sids(strs, category_num):
-    """TODO: Improve this.
-    """
-    out = np.full(len(strs), category_num << 50, dtype='i8')
-    casted_buffer = np.ndarray(
-        shape=out.shape,
-        dtype='S6',
-        buffer=out,
-        strides=out.strides,
-    )
-    casted_buffer[:] = np.array(strs, dtype='S6')
-    return out
-
-
-def str_to_sid(str_, category_num):
-    return strs_to_sids([str_], category_num)[0]
-
-
-iso_currency_to_sid = partial(str_to_sid, category_num=3)
+# Special sentinel used to represent unknown or missing currencies.
+MISSING_CURRENCY_CODE = 'XXX'
 
 
 @total_ordering
@@ -48,15 +28,20 @@ def __new__(cls, code):
         try:
             return _ALL_CURRENCIES[code]
         except KeyError:
-            try:
-                iso_currency = ISO4217Currency(code)
-            except ValueError:
-                raise ValueError(
-                    "{!r} is not a valid currency code.".format(code)
-                )
+            # This isn't a real
+            if code == MISSING_CURRENCY_CODE:
+                name = "NO CURRENCY"
+            else:
+                try:
+                    name = ISO4217Currency(code).currency_name
+                except ValueError:
+                    raise ValueError(
+                        "{!r} is not a valid currency code.".format(code)
+                    )
+
             obj = _ALL_CURRENCIES[code] = super(Currency, cls).__new__(cls)
-            obj._currency = iso_currency
-            obj._sid = iso_currency_to_sid(iso_currency.value)
+            obj._code = code
+            obj._name = name
             return obj
 
     @property
@@ -67,7 +52,7 @@ def code(self):
         -------
         code : str
         """
-        return self._currency.value
+        return self._code
 
     @property
     def name(self):
@@ -77,13 +62,7 @@ def name(self):
         -------
         name : str
         """
-        return self._currency.currency_name
-
-    @property
-    def sid(self):
-        """Unique integer identifier for this currency.
-        """
-        return self._sid
+        return self._name
 
     def __eq__(self, other):
         if type(self) != type(other):

zipline/data/bcolz_daily_bars.py

@@ -34,6 +34,7 @@
 from toolz import compose
 from trading_calendars import get_calendar
 
+from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.session_bars import CurrencyAwareSessionBarReader
 from zipline.data.bar_reader import (
     NoDataAfterDate,
@@ -706,5 +707,14 @@ def get_value(self, sid, dt, field):
             return price
 
     def currency_codes(self, sids):
-        # TODO: Better handling for this.
-        return np.full(len(sids), b'USD', dtype='S3')
+        # XXX: This is pretty inefficient. This reader doesn't really support
+        # country codes, so we always either return USD or
+        # MISSING_CURRENCY_CODE if we don't know about the sid at all.
+        first_rows = self._first_rows
+        out = []
+        for sid in sids:
+            if sid in first_rows:
+                out.append('USD')
+            else:
+                out.append(MISSING_CURRENCY_CODE)
+        return np.array(out, dtype='S3')

zipline/data/fx/hdf5.py

@@ -97,6 +97,7 @@
 from interface import implements
 import h5py
 from logbook import Logger
+import numpy as np
 import pandas as pd
 import six
 
@@ -176,17 +177,9 @@ def get_rates(self, rate, quote, bases, dts):
 
         self._check_dts(self.dts, dts)
 
-        date_ixs = self.dts.searchsorted(dts, side='right') - 1
-        currency_ixs = self.currencies.get_indexer(bases)
+        row_ixs = self.dts.searchsorted(dts, side='right') - 1
+        col_ixs = self.currencies.get_indexer(bases)
 
-        return self._read_rate_block(
-            rate,
-            quote,
-            row_ixs=date_ixs,
-            col_ixs=currency_ixs,
-        )
-
-    def _read_rate_block(self, rate, quote, row_ixs, col_ixs):
         try:
             dataset = self._group[DATA][rate][quote][RATES]
         except KeyError:
@@ -209,7 +202,12 @@ def _read_rate_block(self, rate, quote, row_ixs, col_ixs):
         max_row = row_ixs[-1]
         rows = dataset[min_row:max_row + 1]  # +1 to be inclusive of end
 
-        return rows[row_ixs - min_row][:, col_ixs]
+        out = rows[row_ixs - min_row][:, col_ixs]
+
+        # get_indexer returns -1 for failed lookups. Fill these in with NaN.
+        out[:, col_ixs == -1] = np.nan
+
+        return out
 
     def _check_dts(self, stored, requested):
         """Validate that requested dates are in bounds for what we have stored.

zipline/data/hdf5_daily_bars.py

@@ -107,6 +107,7 @@
 from six import iteritems, raise_from, viewkeys
 from six.moves import reduce
 
+from zipline.currency import MISSING_CURRENCY_CODE
 from zipline.data.bar_reader import (
     NoDataAfterDate,
     NoDataBeforeDate,
@@ -713,18 +714,15 @@ def currency_codes(self, sids):
         # Find the index of requested sids in our stored sids.
         ixs = self.sids.searchsorted(sids, side='left')
 
+        result = self._currency_codes[ixs]
+
         # searchsorted returns the index of the next lowest sid if the lookup
-        # fails. Check for this case and raise an error.
+        # fails. Fill these sids with the special "missing" sentinel.
         not_found = (self.sids[ixs] != sids)
 
-        if not_found.any():
-            # TODO: Should we return an unknown sentinel here?
-            missing = sids[not_found]
-            raise ValueError(
-                "Unable to find currency codes for sids:\n{}".format(missing)
-            )
+        result[not_found] = MISSING_CURRENCY_CODE
 
-        return self._currency_codes[ixs]
+        return result
 
     @property
     def last_available_dt(self):

zipline/data/session_bars.py

@@ -51,5 +51,8 @@ def currency_codes(self, sids):
         Returns
         -------
         currency_codes : np.array[S3]
-            Array of currency codes for listing currencies of ``sids``.
+            Array of currency codes for listing currencies of
+            ``sids``. Implementations should return
+            zipline.currency.MISSING_CURRENCY_CODE for sids whose currency is
+            unknown.
         """