MOD: Integrate DBN 0.5.0

Author: cjdsellers

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 
 ## 0.12.0 - TBD
+- Upgraded `databento-dbn` to 0.5.0
 - Renamed the `cost` field to `cost_usd` for `batch.submit_job` and `batch.list_jobs` (value now expressed as US dollars)
+- Removed `bad` condition variant from `batch.get_dataset_condition`
+- Added `degraded`, `pending` and `missing` condition variants for `batch.get_dataset_condition`
+- Added `last_modified_date` field to `batch.get_dataset_condition` response
 
 ## 0.11.0 - 2023-04-13
 - Changed `end` and `end_date` to optional to support new forward-fill behaviour

README.md

@@ -31,7 +31,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.7 and
 The minimum dependencies as found in the `requirements.txt` are also listed below:
 - Python (>=3.7)
 - aiohttp (>=3.7.2)
-- databento-dbn (==0.4.3)
+- databento-dbn (==0.5.0)
 - numpy (>=1.17.0)
 - pandas (>=1.1.3)
 - requests (>=2.24.0)

databento/common/dbnstore.py

@@ -31,8 +31,8 @@
 )
 from databento.common.enums import Compression, Schema, SType
 from databento.common.error import BentoError
-from databento.common.metadata import MetadataDecoder
 from databento.common.symbology import InstrumentIdMappingInterval
+from databento_dbn import Metadata
 
 
 logger = logging.getLogger(__name__)
@@ -195,7 +195,7 @@ class MemoryDataSource(DataSource):
 
     """
 
-    def __init__(self, source: Union[BytesIO, bytes]):
+    def __init__(self, source: Union[BytesIO, bytes, IO[bytes]]):
         initial_data = source if isinstance(source, bytes) else source.read()
         if len(initial_data) == 0:
             raise ValueError(
@@ -334,7 +334,7 @@ def __init__(self, data_source: DataSource) -> None:
         metadata_bytes.write(buffer.read(metadata_length))
 
         # Read metadata
-        self._metadata: Dict[str, Any] = MetadataDecoder().decode_to_json(
+        self._metadata: Metadata = Metadata.decode(
             metadata_bytes.getvalue(),
         )
 
@@ -408,7 +408,7 @@ def _build_instrument_id_index(self) -> Dict[dt.date, Dict[int, str]]:
                     ),
                 )
 
-        product_id_index: Dict[dt.date, Dict[int, str]] = {}
+        instrument_id_index: Dict[dt.date, Dict[int, str]] = {}
         for interval in intervals:
             for ts in pd.date_range(
                 start=interval.start_date,
@@ -417,12 +417,12 @@ def _build_instrument_id_index(self) -> Dict[dt.date, Dict[int, str]]:
                 **{"inclusive" if pd.__version__ >= "1.4.0" else "closed": "left"},
             ):
                 d: dt.date = ts.date()
-                date_map: Dict[int, str] = product_id_index.get(d, {})
+                date_map: Dict[int, str] = instrument_id_index.get(d, {})
                 if not date_map:
-                    product_id_index[d] = date_map
+                    instrument_id_index[d] = date_map
                 date_map[interval.instrument_id] = interval.raw_symbol
 
-        return product_id_index
+        return instrument_id_index
 
     def _prepare_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
         # Setup column ordering and index
@@ -495,7 +495,7 @@ def dataset(self) -> str:
         str
 
         """
-        return str(self._metadata["dataset"])
+        return str(self._metadata.dataset)
 
     @property
     def dtype(self) -> np.dtype[Any]:
@@ -523,7 +523,7 @@ def end(self) -> pd.Timestamp:
         The data timestamps will not occur after `end`.
 
         """
-        return pd.Timestamp(self._metadata["end"], tz="UTC")
+        return pd.Timestamp(self._metadata.end, tz="UTC")
 
     @property
     def limit(self) -> Optional[int]:
@@ -535,7 +535,7 @@ def limit(self) -> Optional[int]:
         int or None
 
         """
-        return self._metadata["limit"]
+        return self._metadata.limit
 
     @property
     def nbytes(self) -> int:
@@ -559,16 +559,16 @@ def mappings(self) -> Dict[str, List[Dict[str, Any]]]:
         Dict[str, List[Dict[str, Any]]]
 
         """
-        return self._metadata["mappings"]
+        return self._metadata.mappings
 
     @property
-    def metadata(self) -> Dict[str, Any]:
+    def metadata(self) -> Metadata:
         """
         Return the metadata for the data.
 
         Returns
         -------
-        Dict[str, Any]
+        Metadata
 
         """
         return self._metadata
@@ -624,7 +624,7 @@ def schema(self) -> Schema:
         Schema
 
         """
-        return Schema(self._metadata["schema"])
+        return Schema(self._metadata.schema)
 
     @property
     def start(self) -> pd.Timestamp:
@@ -640,7 +640,7 @@ def start(self) -> pd.Timestamp:
         The data timestamps will not occur prior to `start`.
 
         """
-        return pd.Timestamp(self._metadata["start"], tz="UTC")
+        return pd.Timestamp(self._metadata.start, tz="UTC")
 
     @property
     def record_size(self) -> int:
@@ -664,7 +664,7 @@ def stype_in(self) -> SType:
         SType
 
         """
-        return SType(self._metadata["stype_in"])
+        return SType(self._metadata.stype_in)
 
     @property
     def stype_out(self) -> SType:
@@ -676,7 +676,7 @@ def stype_out(self) -> SType:
         SType
 
         """
-        return SType(self._metadata["stype_out"])
+        return SType(self._metadata.stype_out)
 
     @property
     def symbology(self) -> Dict[str, Any]:
@@ -694,8 +694,8 @@ def symbology(self) -> Dict[str, Any]:
             "stype_out": str(self.stype_out),
             "start_date": str(self.start.date()),
             "end_date": str(self.end.date()),
-            "partial": self._metadata["partial"],
-            "not_found": self._metadata["not_found"],
+            "partial": self._metadata.partial,
+            "not_found": self._metadata.not_found,
             "mappings": self.mappings,
         }
 
@@ -709,7 +709,7 @@ def symbols(self) -> List[str]:
         List[str]
 
         """
-        return self._metadata["symbols"]
+        return self._metadata.symbols
 
     @classmethod
     def from_file(cls, path: Union[PathLike[str], str]) -> "DBNStore":

databento/common/enums.py

@@ -253,16 +253,15 @@ class RecordFlags(StringyMixin, IntFlag):  # type: ignore
     """Represents record flags.
 
     F_LAST
-        Last message in the packet from the venue for a given `product_id`
+        Last message in the packet from the venue for a given `instrument_id`.
     F_SNAPSHOT
-        Message sourced from a replay, such as a snapshot server
+        Message sourced from a replay, such as a snapshot server.
     F_MBP
-        Aggregated price level message, not an individual order
+        Aggregated price level message, not an individual order.
     F_BAD_TS_RECV
-        The `ts_recv` value is inaccurate (clock issues or reordering)
+        The `ts_recv` value is inaccurate (clock issues or reordering).
 
     Other bits are reserved and have no current meaning.
-
     """
 
     F_LAST = 128

databento/common/metadata.py

@@ -142,7 +142,7 @@ def _(symbols: str, stype_in: SType) -> str:
         symbol_list = symbols.strip().strip(",").split(",")
         return ",".join(map(symbol_to_string, symbol_list))
 
-    # TODO(cs): Temporary mapping until new DBN released
+    # TODO(cs): Temporary mapping until past stype rename deprecation period
     if stype_in in (SType.SMART, SType.CONTINUOUS):
         return validate_smart_symbol(symbols)
     return symbols.strip().upper()

databento/common/parsing.py

@@ -79,7 +79,7 @@ def submit_job(
         dataset : Dataset or str
             The dataset code (string identifier) for the request.
         symbols : List[Union[str, int]] or str
-            The product symbols to filter for. Takes up to 2,000 symbols per request.
+            The instrument symbols to filter for. Takes up to 2,000 symbols per request.
             If more than 1 symbol is specified, the data is merged and sorted by time.
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa

databento/historical/api/batch.py

@@ -299,7 +299,7 @@ def get_record_count(
             Values are forward filled based on the resolution provided.
             Defaults to the same value as `start`.
         symbols : List[Union[str, int]] or str, optional
-            The product symbols to filter for. Takes up to 2,000 symbols per request.
+            The instrument symbols to filter for. Takes up to 2,000 symbols per request.
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
@@ -368,7 +368,7 @@ def get_billable_size(
             Values are forward filled based on the resolution provided.
             Defaults to the same value as `start`.
         symbols : List[Union[str, int]] or str, optional
-            The product symbols to filter for. Takes up to 2,000 symbols per request.
+            The instrument symbols to filter for. Takes up to 2,000 symbols per request.
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
@@ -440,7 +440,7 @@ def get_cost(
         mode : FeedMode or str {'live', 'historical-streaming', 'historical'}, default 'historical-streaming'
             The data feed mode for the request.
         symbols : List[Union[str, int]] or str, optional
-            The product symbols to filter for. Takes up to 2,000 symbols per request.
+            The instrument symbols to filter for. Takes up to 2,000 symbols per request.
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.

databento/historical/api/metadata.py

@@ -97,7 +97,7 @@ def get_range(
             Values are forward filled based on the resolution provided.
             Defaults to the same value as `start`.
         symbols : List[Union[str, int]] or str, optional
-            The product symbols to filter for. Takes up to 2,000 symbols per request.
+            The instrument symbols to filter for. Takes up to 2,000 symbols per request.
             If more than 1 symbol is specified, the data is merged and sorted by time.
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
@@ -237,7 +237,7 @@ async def get_range_async(
             Values are forward filled based on the resolution provided.
             Defaults to the same value as `start`.
         symbols : List[Union[str, int]] or str, optional
-            The product symbols to filter for. Takes up to 2,000 symbols per request.
+            The instrument symbols to filter for. Takes up to 2,000 symbols per request.
             If more than 1 symbol is specified, the data is merged and sorted by time.
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa

databento/historical/api/timeseries.py

@@ -1,5 +1,5 @@
 aiohttp>=3.7.2,<4.0.0
-databento-dbn==0.4.3
+databento-dbn==0.5.0
 numpy>=1.17.0
 pandas>=1.1.3
 requests>=2.24.0