VER: Release 0.23.0

See release notes.

Author: nmacholl

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.23.0 - 2023-10-26
+
+#### Enhancements
+- Added `map_symbols_csv` function to the `databento` module for using `symbology.json` files to map a symbol column onto a CSV file
+- Added `map_symbols_json` function to the `databento` module for using `symbology.json` files to add a symbol key to a file of JSON records
+- Added new publisher values in preparation for IFEU.IMPACT and NDEX.IMPACT datasets
+
+#### Bug fixes
+- Fixed issue where a large unreadable symbol subscription message could be sent
+- Fixed an issue where `DBNStore.to_df` with `pretty_ts=True` was very slow
+
 ## 0.22.1 - 2023-10-24
 
 #### Bug fixes

databento/__init__.py

@@ -19,6 +19,7 @@
 from databento_dbn import TradeMsg
 
 from databento.common import bentologging
+from databento.common import symbology
 from databento.common.dbnstore import DBNStore
 from databento.common.enums import Delivery
 from databento.common.enums import FeedMode
@@ -35,6 +36,7 @@
 from databento.common.publishers import Dataset
 from databento.common.publishers import Publisher
 from databento.common.publishers import Venue
+from databento.common.symbology import InstrumentMap
 from databento.historical.api import API_VERSION
 from databento.historical.client import Historical
 from databento.live import DBNRecord
@@ -60,6 +62,7 @@
     "RecordFlags",
     "Historical",
     "HistoricalGateway",
+    "InstrumentMap",
     "Live",
     "Packaging",
     "RollRule",
@@ -91,3 +94,5 @@
 # Convenience imports
 enable_logging = bentologging.enable_logging
 from_dbn = DBNStore.from_file
+map_symbols_csv = symbology.map_symbols_csv
+map_symbols_json = symbology.map_symbols_json

databento/common/bentologging.py

@@ -6,7 +6,7 @@
 def enable_logging(level: int | str = logging.INFO) -> None:
     """
     Enable logging for the Databento module. This function should be used for
-    simple applications and examples. It is advisible to configure your own
+    simple applications and examples. It is advisable to configure your own
     logging for serious applications.
 
     Parameters

databento/common/constants.py

@@ -0,0 +1 @@
+ALL_SYMBOLS = "ALL_SYMBOLS"

databento/common/dbnstore.py

@@ -7,7 +7,6 @@
 import warnings
 from collections.abc import Generator
 from collections.abc import Iterator
-from functools import partial
 from io import BytesIO
 from os import PathLike
 from pathlib import Path
@@ -1112,8 +1111,8 @@ def _transcode(
             compression=compression,
             pretty_px=pretty_px,
             pretty_ts=pretty_ts,
-            map_symbols=map_symbols,
             has_metadata=True,
+            map_symbols=map_symbols,
             symbol_map=symbol_map,  # type: ignore [arg-type]
             schema=schema,
         )
@@ -1246,9 +1245,7 @@ def _format_px(
 
     def _format_pretty_ts(self, df: pd.DataFrame) -> None:
         for field in self._struct._timestamp_fields:
-            df[field] = df[field].apply(
-                partial(pd.to_datetime, utc=True, errors="coerce"),
-            )
+            df[field] = pd.to_datetime(df[field], utc=True, errors="coerce")
 
     def _format_set_index(self, df: pd.DataFrame) -> None:
         index_column = (

databento/common/parsing.py

@@ -9,7 +9,7 @@
 import pandas as pd
 from databento_dbn import SType
 
-from databento.common.symbology import ALL_SYMBOLS
+from databento.common.constants import ALL_SYMBOLS
 from databento.common.validation import validate_smart_symbol
 
 

databento/common/publishers.py

@@ -91,6 +91,10 @@ class Venue(StringyMixin, str, Enum):
         Cboe BZX Options Exchange.
     MXOP
         MEMX LLC Options.
+    IFEU
+        ICE Futures Europe (Commodities).
+    NDEX
+        ICE Endex.
 
     """
 
@@ -131,6 +135,8 @@ class Venue(StringyMixin, str, Enum):
     XPHL = "XPHL"
     BATO = "BATO"
     MXOP = "MXOP"
+    IFEU = "IFEU"
+    NDEX = "NDEX"
 
     @classmethod
     def from_int(cls, value: int) -> Venue:
@@ -211,6 +217,10 @@ def from_int(cls, value: int) -> Venue:
             return Venue.BATO
         if value == 37:
             return Venue.MXOP
+        if value == 38:
+            return Venue.IFEU
+        if value == 39:
+            return Venue.NDEX
         raise ValueError(f"Integer value {value} does not correspond with any Venue variant")
 
     def to_int(self) -> int:
@@ -291,6 +301,10 @@ def to_int(self) -> int:
             return 36
         if self == Venue.MXOP:
             return 37
+        if self == Venue.IFEU:
+            return 38
+        if self == Venue.NDEX:
+            return 39
         raise ValueError("Invalid Venue")
 
     @property
@@ -372,6 +386,10 @@ def description(self) -> str:
             return "Cboe BZX Options Exchange"
         if self == Venue.MXOP:
             return "MEMX LLC Options"
+        if self == Venue.IFEU:
+            return "ICE Futures Europe (Commodities)"
+        if self == Venue.NDEX:
+            return "ICE Endex"
         raise ValueError("Unexpected Venue value")
 
 @unique
@@ -434,6 +452,10 @@ class Dataset(StringyMixin, str, Enum):
         Nasdaq QBBO.
     XNAS_NLS
         Nasdaq NLS.
+    IFEU_IMPACT
+        ICE Futures Europe (Commodities) iMpact.
+    NDEX_IMPACT
+        ICE Endex iMpact.
 
     """
 
@@ -464,6 +486,8 @@ class Dataset(StringyMixin, str, Enum):
     XNYS_TRADES = "XNYS.TRADES"
     XNAS_QBBO = "XNAS.QBBO"
     XNAS_NLS = "XNAS.NLS"
+    IFEU_IMPACT = "IFEU.IMPACT"
+    NDEX_IMPACT = "NDEX.IMPACT"
 
     @classmethod
     def from_int(cls, value: int) -> Dataset:
@@ -524,6 +548,10 @@ def from_int(cls, value: int) -> Dataset:
             return Dataset.XNAS_QBBO
         if value == 27:
             return Dataset.XNAS_NLS
+        if value == 28:
+            return Dataset.IFEU_IMPACT
+        if value == 29:
+            return Dataset.NDEX_IMPACT
         raise ValueError(f"Integer value {value} does not correspond with any Dataset variant")
 
     def to_int(self) -> int:
@@ -584,6 +612,10 @@ def to_int(self) -> int:
             return 26
         if self == Dataset.XNAS_NLS:
             return 27
+        if self == Dataset.IFEU_IMPACT:
+            return 28
+        if self == Dataset.NDEX_IMPACT:
+            return 29
         raise ValueError("Invalid Dataset")
 
     @property
@@ -645,6 +677,10 @@ def description(self) -> str:
             return "Nasdaq QBBO"
         if self == Dataset.XNAS_NLS:
             return "Nasdaq NLS"
+        if self == Dataset.IFEU_IMPACT:
+            return "ICE Futures Europe (Commodities) iMpact"
+        if self == Dataset.NDEX_IMPACT:
+            return "ICE Endex iMpact"
         raise ValueError("Unexpected Dataset value")
 
 @unique
@@ -765,6 +801,10 @@ class Publisher(StringyMixin, str, Enum):
         DBEQ Plus - FINRA/Nasdaq TRF Carteret.
     DBEQ_PLUS_FINC
         DBEQ Plus - FINRA/Nasdaq TRF Chicago.
+    IFEU_IMPACT_IFEU
+        ICE Futures Europe (Commodities).
+    NDEX_IMPACT_NDEX
+        ICE Endex.
 
     """
 
@@ -824,6 +864,8 @@ class Publisher(StringyMixin, str, Enum):
     DBEQ_PLUS_FINN = "DBEQ.PLUS.FINN"
     DBEQ_PLUS_FINY = "DBEQ.PLUS.FINY"
     DBEQ_PLUS_FINC = "DBEQ.PLUS.FINC"
+    IFEU_IMPACT_IFEU = "IFEU.IMPACT.IFEU"
+    NDEX_IMPACT_NDEX = "NDEX.IMPACT.NDEX"
 
     @classmethod
     def from_int(cls, value: int) -> Publisher:
@@ -942,6 +984,10 @@ def from_int(cls, value: int) -> Publisher:
             return Publisher.DBEQ_PLUS_FINY
         if value == 56:
             return Publisher.DBEQ_PLUS_FINC
+        if value == 57:
+            return Publisher.IFEU_IMPACT_IFEU
+        if value == 58:
+            return Publisher.NDEX_IMPACT_NDEX
         raise ValueError(f"Integer value {value} does not correspond with any Publisher variant")
 
     def to_int(self) -> int:
@@ -1060,6 +1106,10 @@ def to_int(self) -> int:
             return 55
         if self == Publisher.DBEQ_PLUS_FINC:
             return 56
+        if self == Publisher.IFEU_IMPACT_IFEU:
+            return 57
+        if self == Publisher.NDEX_IMPACT_NDEX:
+            return 58
         raise ValueError("Invalid Publisher")
     @property
     def venue(self) -> Venue:
@@ -1178,6 +1228,10 @@ def venue(self) -> Venue:
             return Venue.FINY
         if self == Publisher.DBEQ_PLUS_FINC:
             return Venue.FINC
+        if self == Publisher.IFEU_IMPACT_IFEU:
+            return Venue.IFEU
+        if self == Publisher.NDEX_IMPACT_NDEX:
+            return Venue.NDEX
         raise ValueError("Unexpected Publisher value")
     @property
     def dataset(self) -> Dataset:
@@ -1296,6 +1350,10 @@ def dataset(self) -> Dataset:
             return Dataset.DBEQ_PLUS
         if self == Publisher.DBEQ_PLUS_FINC:
             return Dataset.DBEQ_PLUS
+        if self == Publisher.IFEU_IMPACT_IFEU:
+            return Dataset.IFEU_IMPACT
+        if self == Publisher.NDEX_IMPACT_NDEX:
+            return Dataset.NDEX_IMPACT
         raise ValueError("Unexpected Publisher value")
 
     @property
@@ -1415,4 +1473,8 @@ def description(self) -> str:
             return "DBEQ Plus - FINRA/Nasdaq TRF Carteret"
         if self == Publisher.DBEQ_PLUS_FINC:
             return "DBEQ Plus - FINRA/Nasdaq TRF Chicago"
+        if self == Publisher.IFEU_IMPACT_IFEU:
+            return "ICE Futures Europe (Commodities)"
+        if self == Publisher.NDEX_IMPACT_NDEX:
+            return "ICE Endex"
         raise ValueError("Unexpected Publisher value")