MOD: Update definition schema struct map

Author: cjdsellers

databento/common/bento.py

@@ -6,7 +6,12 @@
 import numpy as np
 import pandas as pd
 import zstandard
-from databento.common.data import COLUMNS, DERIV_SCHEMAS, STRUCT_MAP
+from databento.common.data import (
+    COLUMNS,
+    DEFINITION_CHARARRAY_COLUMNS,
+    DERIV_SCHEMAS,
+    STRUCT_MAP,
+)
 from databento.common.enums import Compression, Encoding, Schema, SType
 from databento.common.logging import log_debug
 from databento.common.metadata import MetadataDecoder
@@ -445,13 +450,20 @@ def to_df(
             df["flags"] = df["flags"] & 0xFF  # Apply bitmask
             df["side"] = df["side"].str.decode("utf-8")
             df["action"] = df["action"].str.decode("utf-8")
+        elif self.schema == Schema.DEFINITION:
+            for column in DEFINITION_CHARARRAY_COLUMNS:
+                df[column] = df[column].str.decode("utf-8")
 
         if pretty_ts:
             df.index = pd.to_datetime(df.index, utc=True)
             for column in df.columns:
                 if column.startswith("ts_") and "delta" not in column:
                     df[column] = pd.to_datetime(df[column], utc=True)
 
+            if self.schema == Schema.DEFINITION:
+                df["expiration"] = pd.to_datetime(df["expiration"], utc=True)
+                df["activation"] = pd.to_datetime(df["activation"], utc=True)
+
         if pretty_px:
             for column in list(df.columns):
                 if (

databento/common/data.py

@@ -143,18 +143,18 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
         ("maturity_year", np.uint16),
         ("decay_start_date", np.uint16),
         ("channel_id", np.uint16),
-        ("currency", "S1"),  # 1 byte chararray
-        ("settl_currency", "S1"),  # 1 byte chararray
-        ("secsubtype", "S1"),  # 1 byte chararray
-        ("symbol", "S1"),  # 1 byte chararray
-        ("group", "S1"),  # 1 byte chararray
-        ("exchange", "S1"),  # 1 byte chararray
-        ("asset", "S1"),  # 1 byte chararray
-        ("cfi", "S1"),  # 1 byte chararray
-        ("security_type", "S1"),  # 1 byte chararray
-        ("unit_of_measure", "S1"),  # 1 byte chararray
-        ("underlying", "S1"),  # 1 byte chararray
-        ("related", "S1"),  # 1 byte chararray
+        ("currency", "S4"),  # 4 byte chararray
+        ("settl_currency", "S4"),  # 4 byte chararray
+        ("secsubtype", "S6"),  # 6 byte chararray
+        ("symbol", "S22"),  # 22 byte chararray
+        ("group", "S21"),  # 21 byte chararray
+        ("exchange", "S5"),  # 5 byte chararray
+        ("asset", "S7"),  # 7 byte chararray
+        ("cfi", "S7"),  # 7 byte chararray
+        ("security_type", "S7"),  # 7 byte chararray
+        ("unit_of_measure", "S31"),  # 31 byte chararray
+        ("underlying", "S21"),  # 21 byte chararray
+        ("related", "S21"),  # 21 byte chararray
         ("match_algorithm", "S1"),  # 1 byte chararray
         ("md_security_trading_status", np.uint8),
         ("main_fraction", np.uint8),
@@ -170,7 +170,7 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
         ("contract_multiplier_unit", np.int8),
         ("flow_schedule_type", np.int8),
         ("tick_rule", np.uint8),
-        ("dummy", "S1"),  # 1 byte chararray
+        ("dummy", "S3"),  # 3 byte chararray (Adjustment filler for 8-bytes alignment)
     ],
     Schema.GATEWAY_ERROR: RECORD_HEADER
     + [
@@ -186,6 +186,23 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ],
 }
 
+DEFINITION_CHARARRAY_COLUMNS = [
+    "currency",
+    "settl_currency",
+    "secsubtype",
+    "symbol",
+    "group",
+    "exchange",
+    "asset",
+    "cfi",
+    "security_type",
+    "unit_of_measure",
+    "underlying",
+    "related",
+    "match_algorithm",
+    "security_update_action",
+    "user_defined_instrument",
+]
 
 ################################################################################
 # DBZ fields