FIX: Fix passing map_symbols to Transcoder

Author: nmacholl

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.22.1 - TBD
+
+#### Bug fixes
+- Fixed an issue where `DBNStore.to_csv` and `DBNStore.to_json` were mapping symbols even when `map_symbols` was set to `False`
+
 ## 0.22.0 - 2023-10-23
 
 #### Enhancements

databento/common/dbnstore.py

@@ -36,7 +36,6 @@
 from databento.common.error import BentoError
 from databento.common.iterator import chunk
 from databento.common.symbology import InstrumentMap
-from databento.common.symbology import SymbolInterval
 from databento.common.validation import validate_enum
 from databento.common.validation import validate_file_write_path
 from databento.common.validation import validate_maybe_enum
@@ -812,23 +811,13 @@ def to_csv(
                 raise ValueError("a schema must be specified for mixed DBN data")
             schema = self.schema
 
-        record_type = SCHEMA_STRUCT_MAP[schema]
-        record_iter = filter(lambda r: isinstance(r, record_type), self)
-
-        if map_symbols:
-            self._instrument_map.insert_metadata(self.metadata)
-            symbol_map = self._instrument_map._data
-        else:
-            symbol_map = None
-
         with open(path, "xb") as output:
             self._transcode(
                 output=output,
-                records_iter=record_iter,
                 encoding=Encoding.CSV,
                 pretty_px=pretty_px,
                 pretty_ts=pretty_ts,
-                symbol_map=symbol_map,
+                map_symbols=map_symbols,
                 compression=compression,
                 schema=schema,
             )
@@ -1025,23 +1014,13 @@ def to_json(
                 raise ValueError("a schema must be specified for mixed DBN data")
             schema = self.schema
 
-        record_type = SCHEMA_STRUCT_MAP[schema]
-        record_iter = filter(lambda r: isinstance(r, record_type), self)
-
-        if map_symbols:
-            self._instrument_map.insert_metadata(self.metadata)
-            symbol_map = self._instrument_map._data
-        else:
-            symbol_map = None
-
         with open(path, "xb") as output:
             self._transcode(
                 output=output,
-                records_iter=record_iter,
                 encoding=Encoding.JSON,
                 pretty_px=pretty_px,
                 pretty_ts=pretty_ts,
-                symbol_map=symbol_map,
+                map_symbols=map_symbols,
                 compression=compression,
                 schema=schema,
             )
@@ -1114,27 +1093,33 @@ def to_ndarray(
     def _transcode(
         self,
         output: BinaryIO,
-        records_iter: Iterator[DBNRecord],
         encoding: Encoding,
         pretty_px: bool,
         pretty_ts: bool,
-        symbol_map: dict[int, list[SymbolInterval]] | None,
+        map_symbols: bool,
         compression: Compression,
         schema: Schema,
     ) -> None:
+        if map_symbols:
+            self._instrument_map.insert_metadata(self.metadata)
+            symbol_map = self._instrument_map._data
+        else:
+            symbol_map = None
+
         transcoder = Transcoder(
             file=output,
             encoding=encoding,
             compression=compression,
             pretty_px=pretty_px,
             pretty_ts=pretty_ts,
+            map_symbols=map_symbols,
             has_metadata=True,
             symbol_map=symbol_map,  # type: ignore [arg-type]
             schema=schema,
         )
 
         transcoder.write(bytes(self.metadata))
-        for records in chunk(records_iter, 2**16):
+        for records in chunk(self, 2**16):
             for record in records:
                 transcoder.write(bytes(record))
             transcoder.flush()

tests/test_historical_bento.py

@@ -488,11 +488,11 @@ def test_mbo_to_csv_writes_expected_file_to_disk(
     # Assert
     written = path.read_text()
     expected = (
-        "ts_recv,ts_event,rtype,publisher_id,instrument_id,action,side,price,size,channel_id,order_id,flags,ts_in_delta,sequence,symbol\n"
-        "1609160400000704060,1609160400000429831,160,1,5482,C,A,3722750000000,1,0,647784973705,128,22993,1170352,ESH1\n"
-        "1609160400000711344,1609160400000431665,160,1,5482,C,A,3723000000000,1,0,647784973631,128,19621,1170353,ESH1\n"
-        "1609160400000728600,1609160400000433051,160,1,5482,C,A,3723250000000,1,0,647784973427,128,16979,1170354,ESH1\n"
-        "1609160400000740248,1609160400000434353,160,1,5482,C,A,3723500000000,1,0,647784973094,128,17883,1170355,ESH1\n"
+        "ts_recv,ts_event,rtype,publisher_id,instrument_id,action,side,price,size,channel_id,order_id,flags,ts_in_delta,sequence\n"
+        "1609160400000704060,1609160400000429831,160,1,5482,C,A,3722750000000,1,0,647784973705,128,22993,1170352\n"
+        "1609160400000711344,1609160400000431665,160,1,5482,C,A,3723000000000,1,0,647784973631,128,19621,1170353\n"
+        "1609160400000728600,1609160400000433051,160,1,5482,C,A,3723250000000,1,0,647784973427,128,16979,1170354\n"
+        "1609160400000740248,1609160400000434353,160,1,5482,C,A,3723500000000,1,0,647784973094,128,17883,1170355\n"
     )
 
     assert written == expected
@@ -518,11 +518,11 @@ def test_mbp_1_to_csv_with_no_options_writes_expected_file_to_disk(
     # Assert
     written = path.read_text()
     expected = (
-        "ts_recv,ts_event,rtype,publisher_id,instrument_id,action,side,depth,price,size,flags,ts_in_delta,sequence,bid_px_00,ask_px_00,bid_sz_00,ask_sz_00,bid_ct_00,ask_ct_00,symbol\n"
-        "1609160400006136329,1609160400006001487,1,1,5482,A,A,0,3720500000000,1,128,17214,1170362,3720250000000,3720500000000,24,11,15,9,ESH1\n"
-        "1609160400006246513,1609160400006146661,1,1,5482,A,A,0,3720500000000,1,128,18858,1170364,3720250000000,3720500000000,24,12,15,10,ESH1\n"
-        "1609160400007159323,1609160400007044577,1,1,5482,A,B,0,3720250000000,2,128,18115,1170365,3720250000000,3720500000000,26,12,16,10,ESH1\n"
-        "1609160400007260967,1609160400007169135,1,1,5482,C,A,0,3720500000000,1,128,17361,1170366,3720250000000,3720500000000,26,11,16,9,ESH1\n"
+        "ts_recv,ts_event,rtype,publisher_id,instrument_id,action,side,depth,price,size,flags,ts_in_delta,sequence,bid_px_00,ask_px_00,bid_sz_00,ask_sz_00,bid_ct_00,ask_ct_00\n"
+        "1609160400006136329,1609160400006001487,1,1,5482,A,A,0,3720500000000,1,128,17214,1170362,3720250000000,3720500000000,24,11,15,9\n"
+        "1609160400006246513,1609160400006146661,1,1,5482,A,A,0,3720500000000,1,128,18858,1170364,3720250000000,3720500000000,24,12,15,10\n"
+        "1609160400007159323,1609160400007044577,1,1,5482,A,B,0,3720250000000,2,128,18115,1170365,3720250000000,3720500000000,26,12,16,10\n"
+        "1609160400007260967,1609160400007169135,1,1,5482,C,A,0,3720500000000,1,128,17361,1170366,3720250000000,3720500000000,26,11,16,9\n"
     )
 
     assert written == expected
@@ -578,10 +578,10 @@ def test_mbo_to_json_with_no_options_writes_expected_file_to_disk(
     # Assert
     written = path.read_text()
     assert written == (
-        '{"ts_recv":"1609160400000704060","hd":{"ts_event":"1609160400000429831","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3722750000000","size":1,"channel_id":0,"order_id":"647784973705","flags":128,"ts_in_delta":22993,"sequence":1170352,"symbol":"ESH1"}\n'
-        '{"ts_recv":"1609160400000711344","hd":{"ts_event":"1609160400000431665","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3723000000000","size":1,"channel_id":0,"order_id":"647784973631","flags":128,"ts_in_delta":19621,"sequence":1170353,"symbol":"ESH1"}\n'
-        '{"ts_recv":"1609160400000728600","hd":{"ts_event":"1609160400000433051","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3723250000000","size":1,"channel_id":0,"order_id":"647784973427","flags":128,"ts_in_delta":16979,"sequence":1170354,"symbol":"ESH1"}\n'
-        '{"ts_recv":"1609160400000740248","hd":{"ts_event":"1609160400000434353","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3723500000000","size":1,"channel_id":0,"order_id":"647784973094","flags":128,"ts_in_delta":17883,"sequence":1170355,"symbol":"ESH1"}\n'
+        '{"ts_recv":"1609160400000704060","hd":{"ts_event":"1609160400000429831","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3722750000000","size":1,"channel_id":0,"order_id":"647784973705","flags":128,"ts_in_delta":22993,"sequence":1170352}\n'
+        '{"ts_recv":"1609160400000711344","hd":{"ts_event":"1609160400000431665","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3723000000000","size":1,"channel_id":0,"order_id":"647784973631","flags":128,"ts_in_delta":19621,"sequence":1170353}\n'
+        '{"ts_recv":"1609160400000728600","hd":{"ts_event":"1609160400000433051","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3723250000000","size":1,"channel_id":0,"order_id":"647784973427","flags":128,"ts_in_delta":16979,"sequence":1170354}\n'
+        '{"ts_recv":"1609160400000740248","hd":{"ts_event":"1609160400000434353","rtype":160,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","price":"3723500000000","size":1,"channel_id":0,"order_id":"647784973094","flags":128,"ts_in_delta":17883,"sequence":1170355}\n'
     )
 
 
@@ -632,10 +632,10 @@ def test_mbp_1_to_json_with_no_options_writes_expected_file_to_disk(
     # Assert
     written = path.read_text()
     assert written == (
-        '{"ts_recv":"1609160400006136329","hd":{"ts_event":"1609160400006001487","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"A","side":"A","depth":0,"price":"3720500000000","size":1,"flags":128,"ts_in_delta":17214,"sequence":1170362,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":24,"ask_sz":11,"bid_ct":15,"ask_ct":9}],"symbol":"ESH1"}\n'
-        '{"ts_recv":"1609160400006246513","hd":{"ts_event":"1609160400006146661","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"A","side":"A","depth":0,"price":"3720500000000","size":1,"flags":128,"ts_in_delta":18858,"sequence":1170364,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":24,"ask_sz":12,"bid_ct":15,"ask_ct":10}],"symbol":"ESH1"}\n'
-        '{"ts_recv":"1609160400007159323","hd":{"ts_event":"1609160400007044577","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"A","side":"B","depth":0,"price":"3720250000000","size":2,"flags":128,"ts_in_delta":18115,"sequence":1170365,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":26,"ask_sz":12,"bid_ct":16,"ask_ct":10}],"symbol":"ESH1"}\n'
-        '{"ts_recv":"1609160400007260967","hd":{"ts_event":"1609160400007169135","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","depth":0,"price":"3720500000000","size":1,"flags":128,"ts_in_delta":17361,"sequence":1170366,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":26,"ask_sz":11,"bid_ct":16,"ask_ct":9}],"symbol":"ESH1"}\n'
+        '{"ts_recv":"1609160400006136329","hd":{"ts_event":"1609160400006001487","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"A","side":"A","depth":0,"price":"3720500000000","size":1,"flags":128,"ts_in_delta":17214,"sequence":1170362,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":24,"ask_sz":11,"bid_ct":15,"ask_ct":9}]}\n'
+        '{"ts_recv":"1609160400006246513","hd":{"ts_event":"1609160400006146661","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"A","side":"A","depth":0,"price":"3720500000000","size":1,"flags":128,"ts_in_delta":18858,"sequence":1170364,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":24,"ask_sz":12,"bid_ct":15,"ask_ct":10}]}\n'
+        '{"ts_recv":"1609160400007159323","hd":{"ts_event":"1609160400007044577","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"A","side":"B","depth":0,"price":"3720250000000","size":2,"flags":128,"ts_in_delta":18115,"sequence":1170365,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":26,"ask_sz":12,"bid_ct":16,"ask_ct":10}]}\n'
+        '{"ts_recv":"1609160400007260967","hd":{"ts_event":"1609160400007169135","rtype":1,"publisher_id":1,"instrument_id":5482},"action":"C","side":"A","depth":0,"price":"3720500000000","size":1,"flags":128,"ts_in_delta":17361,"sequence":1170366,"levels":[{"bid_px":"3720250000000","ask_px":"3720500000000","bid_sz":26,"ask_sz":11,"bid_ct":16,"ask_ct":9}]}\n'
     )