FIX: Fix to_ndarray implementation

Author: nmacholl

CHANGELOG.md

@@ -5,6 +5,9 @@
 #### Breaking changes
 - Renamed the `TimeSeriesHttpAPI` class to `TimeseriesHttpAPI`
 
+#### Bug Fixes
+- Fixed an issue where `DBNStore.to_csv()`, `DBNStore.to_df()`, `DBNStore.to_json()`, and `DBNStore.to_ndarray()` would consume large amounts of memory.
+
 ## 0.17.0 - 2023-08-10
 
 This release includes improvements to the ergonomics of the clients metadata API, you can read more about the changes [here](https://databento.com/blog/api-improvements-august-2023).

databento/common/dbnstore.py

@@ -2,7 +2,6 @@
 
 import abc
 import datetime as dt
-import functools
 import logging
 from collections.abc import Generator
 from io import BytesIO
@@ -1069,15 +1068,16 @@ def to_ndarray(
                 raise ValueError("a schema must be specified for mixed DBN data")
             schema = self.schema
 
-        schema_records = filter(
-            lambda r: isinstance(r, SCHEMA_STRUCT_MAP[schema]),  # type: ignore
-            self,
-        )
-
-        decoder = functools.partial(np.frombuffer, dtype=SCHEMA_DTYPES_MAP[schema])
-        result = tuple(map(decoder, map(bytes, schema_records)))
+        record_buffer = BytesIO()
+        num_records = 0
+        for record in filter(lambda r: isinstance(r, SCHEMA_STRUCT_MAP[schema]), self):  # type: ignore [arg-type]
+            num_records += 1
+            record_buffer.write(bytes(record))
 
-        if not result:
-            return np.empty(shape=(0, 1), dtype=SCHEMA_DTYPES_MAP[schema])
+        result = np.frombuffer(
+            record_buffer.getvalue(),
+            dtype=SCHEMA_DTYPES_MAP[schema],
+            count=num_records,
+        )
 
-        return np.ravel(result)
+        return result