FIX: Reduce FP error in Python client's pretty_px

Author: threecgreen

CHANGELOG.md

@@ -9,6 +9,7 @@
 
 #### Bug fixes
 - Fixed issue where exception messages were displaying JSON encoded data
+- Reduced floating error when converting prices to floats with `pretty_px=True`
 
 ## 0.18.1 - 2023-08-16
 

databento/common/data.py

@@ -51,36 +51,6 @@
     Schema.STATISTICS: StatMsg._dtypes,
 }
 
-DEFINITION_CHARARRAY_COLUMNS = [
-    "currency",
-    "settl_currency",
-    "secsubtype",
-    "raw_symbol",
-    "group",
-    "exchange",
-    "asset",
-    "cfi",
-    "security_type",
-    "unit_of_measure",
-    "underlying",
-    "strike_price_currency",
-    "instrument_class",
-    "match_algorithm",
-    "security_update_action",
-    "user_defined_instrument",
-]
-
-DEFINITION_PRICE_COLUMNS = [
-    "min_price_increment",
-    "high_limit_price",
-    "low_limit_price",
-    "max_price_variation",
-    "trading_reference_price",
-    "min_price_increment_amount",
-    "price_ratio",
-    "strike_price",
-]
-
 DEFINITION_TYPE_MAX_MAP = {
     x[0]: np.iinfo(x[1]).max
     for x in InstrumentDefMsg._dtypes

databento/common/dbnstore.py

@@ -18,6 +18,7 @@
 import numpy as np
 import pandas as pd
 import zstandard
+from databento_dbn import FIXED_PRICE_SCALE
 from databento_dbn import Compression
 from databento_dbn import DBNDecoder
 from databento_dbn import ErrorMsg
@@ -27,8 +28,6 @@
 from databento_dbn import SymbolMappingMsg
 from databento_dbn import SystemMsg
 
-from databento.common.data import DEFINITION_CHARARRAY_COLUMNS
-from databento.common.data import DEFINITION_PRICE_COLUMNS
 from databento.common.data import DEFINITION_TYPE_MAX_MAP
 from databento.common.data import DERIV_SCHEMAS
 from databento.common.data import SCHEMA_COLUMNS
@@ -49,7 +48,6 @@
 ]
 
 INT64_NULL = 9223372036854775807
-NAN = float("NaN")
 
 
 logger = logging.getLogger(__name__)
@@ -398,36 +396,16 @@ def __repr__(self) -> str:
 
     def _apply_pretty_ts(self, df: pd.DataFrame) -> pd.DataFrame:
         df.index = pd.to_datetime(df.index, utc=True)
-        for column in df.columns:
-            if column.startswith("ts_") and "delta" not in column:
-                df[column] = pd.to_datetime(df[column], errors="coerce", utc=True)
-
-        if self.schema == Schema.DEFINITION:
-            df["expiration"] = pd.to_datetime(
-                df["expiration"],
-                errors="coerce",
-                utc=True,
-            )
-            df["activation"] = pd.to_datetime(
-                df["activation"],
-                errors="coerce",
-                utc=True,
-            )
+        for column in SCHEMA_STRUCT_MAP[self.schema]._timestamp_fields:
+            if df.index.name == column:
+                continue
+            df[column] = pd.to_datetime(df[column], errors="coerce", utc=True)
 
         return df
 
     def _apply_pretty_px(self, df: pd.DataFrame) -> pd.DataFrame:
-        for column in list(df.columns):
-            if (
-                column in ("price", "open", "high", "low", "close")
-                or column.startswith("bid_px")  # MBP
-                or column.startswith("ask_px")  # MBP
-            ):
-                df[column] = df[column].replace(INT64_NULL, NAN) * 1e-9
-
-        if self.schema == Schema.DEFINITION:
-            for column in DEFINITION_PRICE_COLUMNS:
-                df[column] = df[column].replace(INT64_NULL, NAN) * 1e-9
+        for column in SCHEMA_STRUCT_MAP[self.schema]._price_fields:
+            df[column] = df[column].replace(INT64_NULL, np.nan) / FIXED_PRICE_SCALE
 
         return df
 
@@ -468,13 +446,14 @@ def _prepare_dataframe(
         df: pd.DataFrame,
         schema: Schema,
     ) -> pd.DataFrame:
+        # char array columns
+        hidden_fields = SCHEMA_STRUCT_MAP[self.schema]._hidden_fields
+        for column, dtype  in SCHEMA_DTYPES_MAP[self.schema]:
+            if dtype.startswith("S") and column not in hidden_fields:
+                df[column] = df[column].str.decode("utf-8")
         if schema == Schema.MBO or schema in DERIV_SCHEMAS:
             df["flags"] = df["flags"] & 0xFF  # Apply bitmask
-            df["side"] = df["side"].str.decode("utf-8")
-            df["action"] = df["action"].str.decode("utf-8")
         elif schema == Schema.DEFINITION:
-            for column in DEFINITION_CHARARRAY_COLUMNS:
-                df[column] = df[column].str.decode("utf-8")
             for column, type_max in DEFINITION_TYPE_MAX_MAP.items():
                 if column in df.columns:
                     df[column] = df[column].where(df[column] != type_max, np.nan)
@@ -715,7 +694,7 @@ def symbology(self) -> dict[str, Any]:
             "stype_in": str(self.stype_in),
             "stype_out": str(self.stype_out),
             "start_date": str(self.start.date()),
-            "end_date": str(self.end.date()),
+            "end_date": str(self.end.date()) if self.end else None,
             "partial": self._metadata.partial,
             "not_found": self._metadata.not_found,
             "mappings": self.mappings,
@@ -862,7 +841,7 @@ def request_symbology(self, client: Historical) -> dict[str, Any]:
             stype_in=self.stype_in,
             stype_out=self.stype_out,
             start_date=self.start.date(),
-            end_date=self.end.date(),
+            end_date=self.end.date() if self.end else None,
         )
 
     def to_csv(

tests/test_historical_bento.py

@@ -572,14 +572,14 @@ def test_mbp_1_to_csv_with_all_options_writes_expected_file_to_disk(
         b"ts_recv,ts_event,rtype,publisher_id,instrument_id,action,side,depth,price,size,flags"
         b",ts_in_delta,sequence,bid_px_00,ask_px_00,bid_sz_00,ask_sz_00,bid_ct_00,ask_ct_00,sy"
         b"mbol\n2020-12-28 13:00:00.006136329+00:00,2020-12-28 13:00:00.006001487+00:00,1,1"
-        b",5482,A,A,0,3720.5000000000005,1,128,17214,1170362,3720.2500000000005,3720.500000000"
-        b"0005,24,11,15,9,ESH1\n2020-12-28 13:00:00.006246513+00:00,2020-12-28 13:00:00.006"
-        b"146661+00:00,1,1,5482,A,A,0,3720.5000000000005,1,128,18858,1170364,3720.250000000000"
-        b"5,3720.5000000000005,24,12,15,10,ESH1\n2020-12-28 13:00:00.007159323+00:00,2020-1"
-        b"2-28 13:00:00.007044577+00:00,1,1,5482,A,B,0,3720.2500000000005,2,128,18115,1170365,"
-        b"3720.2500000000005,3720.5000000000005,26,12,16,10,ESH1\n2020-12-28 13:00:00.00726"
-        b"0967+00:00,2020-12-28 13:00:00.007169135+00:00,1,1,5482,C,A,0,3720.5000000000005,1,1"
-        b"28,17361,1170366,3720.2500000000005,3720.5000000000005,26,11,16,9,ESH1\n"
+        b",5482,A,A,0,3720.5,1,128,17214,1170362,3720.25,3720.5"
+        b",24,11,15,9,ESH1\n2020-12-28 13:00:00.006246513+00:00,2020-12-28 13:00:00.006"
+        b"146661+00:00,1,1,5482,A,A,0,3720.5,1,128,18858,1170364,3720.25"
+        b",3720.5,24,12,15,10,ESH1\n2020-12-28 13:00:00.007159323+00:00,2020-1"
+        b"2-28 13:00:00.007044577+00:00,1,1,5482,A,B,0,3720.25,2,128,18115,1170365,"
+        b"3720.25,3720.5,26,12,16,10,ESH1\n2020-12-28 13:00:00.00726"
+        b"0967+00:00,2020-12-28 13:00:00.007169135+00:00,1,1,5482,C,A,0,3720.5,1,1"
+        b"28,17361,1170366,3720.25,3720.5,26,11,16,9,ESH1\n"
     )
 
     if sys.platform == "win32":