ADD: Add DataFrame chunk reading

Author: nmacholl

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 0.19.0 - TBD
 
 #### Enhancements
+- Added `count` parameter to `DBNStore.to_df` and `DBNStore.to_ndarray` to help process large files incrementally
 - Added the `Publisher`, `Venue`, and `Dataset` enums
 - Replace null prices with `NaN` when `pretty_px=True` in `DBNStore.to_df()`
 - Upgraded `databento-dbn` to 0.8.3
@@ -11,6 +12,9 @@
 - Fixed issue where exception messages were displaying JSON encoded data
 - Reduced floating error when converting prices to floats with `pretty_px=True`
 
+#### Breaking changes
+- `DBNStore.to_df` now always utf-8 decodes string fields
+
 ## 0.18.1 - 2023-08-16
 
 #### Bug fixes

databento/common/data.py

@@ -14,6 +14,12 @@
 from databento.live import DBNRecord
 
 
+DEFINITION_TYPE_MAX_MAP = {
+    x[0]: np.iinfo(x[1]).max
+    for x in InstrumentDefMsg._dtypes
+    if not isinstance(x[1], str)
+}
+
 SCHEMA_STRUCT_MAP: dict[Schema, type[DBNRecord]] = {
     Schema.DEFINITION: InstrumentDefMsg,
     Schema.IMBALANCE: ImbalanceMsg,
@@ -51,12 +57,6 @@
     Schema.STATISTICS: StatMsg._dtypes,
 }
 
-DEFINITION_TYPE_MAX_MAP = {
-    x[0]: np.iinfo(x[1]).max
-    for x in InstrumentDefMsg._dtypes
-    if not isinstance(x[1], str)
-}
-
 SCHEMA_COLUMNS = {
     Schema.MBO: MBOMsg._ordered_fields,
     Schema.MBP_1: MBP1Msg._ordered_fields,