Skip to content

Commit afe85ee

Browse files
nmachollnmacholl
committed
FIX: Fix missing index in DBNStore.to_json
1 parent 8db51b0 commit afe85ee

File tree

3 files changed

+71
-49
lines changed

3 files changed

+71
-49
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Changelog
22

3+
## 0.19.1 - TBD
4+
5+
#### Bug fixes
6+
- Fixed an issue where the index column was not serialized with `DBNStore.to_json`
7+
38
## 0.19.0 - 2023-08-25
49

510
This release includes improvements to handling large DBN data and adds support for future datasets.

‎databento/common/dbnstore.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,6 +1046,7 @@ def to_json(
10461046

10471047
with open(path, "x") as json_path:
10481048
for frame in df_iter:
1049+
frame.reset_index(inplace=True)
10491050
frame.to_json(
10501051
json_path,
10511052
orient="records",

‎tests/test_historical_bento.py‎

Lines changed: 65 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -596,15 +596,20 @@ def test_mbo_to_json_with_no_options_writes_expected_file_to_disk(
596596
# Assert
597597
written = path.read_text()
598598
assert written.strip() == (
599-
'{"ts_event":1609160400000429831,"rtype":160,"publisher_id":1,"instrument_id":5482,"a'
600-
'ction":"C","side":"A","price":3722750000000,"size":1,"channel_id":0,"order_id":64778'
601-
'4973705,"flags":128,"ts_in_delta":22993,"sequence":1170352}\n{"ts_event":16091604'
602-
'00000431665,"rtype":160,"publisher_id":1,"instrument_id":5482,"action":"C","side":"A'
603-
'","price":3723000000000,"size":1,"channel_id":0,"order_id":647784973631,"flags":128,'
604-
'"ts_in_delta":19621,"sequence":1170353}\n{"ts_event":1609160400000433051,"rtype":'
599+
'{"ts_recv":1609160400000704060,"ts_event":1609160400000429831,"rtype":160,'
600+
'"publisher_id":1,"instrument_id":5482,"action":"C","side":"A","price":3722750000000,'
601+
'"size":1,"channel_id":0,"order_id":647784973705,"flags":128,"ts_in_delta":22993,"sequence":1170352}'
602+
"\n"
603+
'{"ts_recv":1609160400000711344,"ts_event":1609160400000431665,"rtype":160,"publisher_id":1,"instrument_id":5482,'
604+
'"action":"C","side":"A","price":3723000000000,"size":1,"channel_id":0,"order_id":647784973631,'
605+
'"flags":128,"ts_in_delta":19621,"sequence":1170353}'
606+
"\n"
607+
'{"ts_recv":1609160400000728600,"ts_event":1609160400000433051,"rtype":'
605608
'160,"publisher_id":1,"instrument_id":5482,"action":"C","side":"A","price":3723250000'
606609
'000,"size":1,"channel_id":0,"order_id":647784973427,"flags":128,"ts_in_delta":16979,'
607-
'"sequence":1170354}\n{"ts_event":1609160400000434353,"rtype":160,"publisher_id":1'
610+
'"sequence":1170354}'
611+
"\n"
612+
'{"ts_recv":1609160400000740248,"ts_event":1609160400000434353,"rtype":160,"publisher_id":1'
608613
',"instrument_id":5482,"action":"C","side":"A","price":3723500000000,"size":1,"channe'
609614
'l_id":0,"order_id":647784973094,"flags":128,"ts_in_delta":17883,"sequence":11703'
610615
"55}"
@@ -631,18 +636,21 @@ def test_mbo_to_json_with_all_options_writes_expected_file_to_disk(
631636
# Assert
632637
written = path.read_text()
633638
assert written.strip() == (
634-
'{"ts_event":1609160400000,"rtype":160,"publisher_id":1,"instrument_id":5482,"action"'
635-
':"C","side":"A","price":3722.75,"size":1,"channel_id":0,"order_id":647784973705,"fla'
636-
'gs":128,"ts_in_delta":22993,"sequence":1170352,"symbol":"ESH1"}\n{"ts_event":1609'
637-
'160400000,"rtype":160,"publisher_id":1,"instrument_id":5482,"action":"C","side":"A",'
638-
'"price":3723.0,"size":1,"channel_id":0,"order_id":647784973631,"flags":128,"ts_in_de'
639-
'lta":19621,"sequence":1170353,"symbol":"ESH1"}\n{"ts_event":1609160400000,"rtype"'
640-
':160,"publisher_id":1,"instrument_id":5482,"action":"C","side":"A","price":3723.25,"'
641-
'size":1,"channel_id":0,"order_id":647784973427,"flags":128,"ts_in_delta":16979,"sequ'
642-
'ence":1170354,"symbol":"ESH1"}\n{"ts_event":1609160400000,"rtype":160,"publisher_'
643-
'id":1,"instrument_id":5482,"action":"C","side":"A","price":3723.5,"size":1,"channel_'
644-
'id":0,"order_id":647784973094,"flags":128,"ts_in_delta":17883,"sequence":1170355,"sy'
645-
'mbol":"ESH1"}'
639+
'{"ts_recv":1609160400000,"ts_event":1609160400000,"rtype":160,"publisher_id":1,"instrument_id":5482,'
640+
'"action":"C","side":"A","price":3722.75,"size":1,"channel_id":0,"order_id":647784973705,"flags":128,'
641+
'"ts_in_delta":22993,"sequence":1170352,"symbol":"ESH1"}'
642+
"\n"
643+
'{"ts_recv":1609160400000,"ts_event":1609160400000,"rtype":160,"publisher_id":1,"instrument_id":5482,'
644+
'"action":"C","side":"A","price":3723.0,"size":1,"channel_id":0,"order_id":647784973631,"flags":128,'
645+
'"ts_in_delta":19621,"sequence":1170353,"symbol":"ESH1"}'
646+
"\n"
647+
'{"ts_recv":1609160400000,"ts_event":1609160400000,"rtype":160,"publisher_id":1,"instrument_id":5482,'
648+
'"action":"C","side":"A","price":3723.25,"size":1,"channel_id":0,"order_id":647784973427,"flags":128,'
649+
'"ts_in_delta":16979,"sequence":1170354,"symbol":"ESH1"}'
650+
"\n"
651+
'{"ts_recv":1609160400000,"ts_event":1609160400000,"rtype":160,"publisher_id":1,"instrument_id":5482,'
652+
'"action":"C","side":"A","price":3723.5,"size":1,"channel_id":0,"order_id":647784973094,"flags":128,'
653+
'"ts_in_delta":17883,"sequence":1170355,"symbol":"ESH1"}'
646654
)
647655

648656

@@ -666,21 +674,25 @@ def test_mbp_1_to_json_with_no_options_writes_expected_file_to_disk(
666674
# Assert
667675
written = path.read_text()
668676
assert written.strip() == (
669-
'{"ts_event":1609160400006001487,"rtype":1,"publisher_id":1,"instrument_id":5482,"act'
670-
'ion":"A","side":"A","depth":0,"price":3720500000000,"size":1,"flags":128,"ts_in_delt'
671-
'a":17214,"sequence":1170362,"bid_px_00":3720250000000,"ask_px_00":3720500000000,"bid'
672-
'_sz_00":24,"ask_sz_00":11,"bid_ct_00":15,"ask_ct_00":9}\n{"ts_event":160916040000'
673-
'6146661,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"A","side":"A","dep'
674-
'th":0,"price":3720500000000,"size":1,"flags":128,"ts_in_delta":18858,"sequence":1170'
675-
'364,"bid_px_00":3720250000000,"ask_px_00":3720500000000,"bid_sz_00":24,"ask_sz_00":1'
676-
'2,"bid_ct_00":15,"ask_ct_00":10}\n{"ts_event":1609160400007044577,"rtype":1,"publ'
677-
'isher_id":1,"instrument_id":5482,"action":"A","side":"B","depth":0,"price":372025000'
678-
'0000,"size":2,"flags":128,"ts_in_delta":18115,"sequence":1170365,"bid_px_00":3720250'
679-
'000000,"ask_px_00":3720500000000,"bid_sz_00":26,"ask_sz_00":12,"bid_ct_00":16,"ask_c'
680-
't_00":10}\n{"ts_event":1609160400007169135,"rtype":1,"publisher_id":1,"instrument'
681-
'_id":5482,"action":"C","side":"A","depth":0,"price":3720500000000,"size":1,"flags":1'
682-
'28,"ts_in_delta":17361,"sequence":1170366,"bid_px_00":3720250000000,"ask_px_00":3720'
683-
'500000000,"bid_sz_00":26,"ask_sz_00":11,"bid_ct_00":16,"ask_ct_00":9}'
677+
'{"ts_recv":1609160400006136329,"ts_event":1609160400006001487,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"A",'
678+
'"side":"A","depth":0,"price":3720500000000,"size":1,"flags":128,"ts_in_delta":17214,"sequence":1170362,'
679+
'"bid_px_00":3720250000000,"ask_px_00":3720500000000,"bid_sz_00":24,"ask_sz_00":11,"bid_ct_00":15,'
680+
'"ask_ct_00":9}'
681+
"\n"
682+
'{"ts_recv":1609160400006246513,"ts_event":1609160400006146661,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"A",'
683+
'"side":"A","depth":0,"price":3720500000000,"size":1,"flags":128,"ts_in_delta":18858,"sequence":1170364,'
684+
'"bid_px_00":3720250000000,"ask_px_00":3720500000000,"bid_sz_00":24,"ask_sz_00":12,"bid_ct_00":15,'
685+
'"ask_ct_00":10}'
686+
"\n"
687+
'{"ts_recv":1609160400007159323,"ts_event":1609160400007044577,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"A",'
688+
'"side":"B","depth":0,"price":3720250000000,"size":2,"flags":128,"ts_in_delta":18115,"sequence":1170365,'
689+
'"bid_px_00":3720250000000,"ask_px_00":3720500000000,"bid_sz_00":26,"ask_sz_00":12,"bid_ct_00":16,'
690+
'"ask_ct_00":10}'
691+
"\n"
692+
'{"ts_recv":1609160400007260967,"ts_event":1609160400007169135,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"C",'
693+
'"side":"A","depth":0,"price":3720500000000,"size":1,"flags":128,"ts_in_delta":17361,"sequence":1170366,'
694+
'"bid_px_00":3720250000000,"ask_px_00":3720500000000,"bid_sz_00":26,"ask_sz_00":11,"bid_ct_00":16,'
695+
'"ask_ct_00":9}'
684696
)
685697

686698

@@ -704,21 +716,25 @@ def test_mbp_1_to_json_with_all_options_writes_expected_file_to_disk(
704716
# Assert
705717
written = path.read_text()
706718
assert written.strip() == (
707-
'{"ts_event":1609160400006,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"'
708-
'A","side":"A","depth":0,"price":3720.5,"size":1,"flags":128,"ts_in_delta":17214,"seq'
709-
'uence":1170362,"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":24,"ask_sz_00":11,'
710-
'"bid_ct_00":15,"ask_ct_00":9,"symbol":"ESH1"}\n{"ts_event":1609160400006,"rtype":'
711-
'1,"publisher_id":1,"instrument_id":5482,"action":"A","side":"A","depth":0,"price":37'
712-
'20.5,"size":1,"flags":128,"ts_in_delta":18858,"sequence":1170364,"bid_px_00":3720.25'
713-
',"ask_px_00":3720.5,"bid_sz_00":24,"ask_sz_00":12,"bid_ct_00":15,"ask_ct_00":10,"sym'
714-
'bol":"ESH1"}\n{"ts_event":1609160400007,"rtype":1,"publisher_id":1,"instrument_id'
715-
'":5482,"action":"A","side":"B","depth":0,"price":3720.25,"size":2,"flags":128,"ts_in'
716-
'_delta":18115,"sequence":1170365,"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":'
717-
'26,"ask_sz_00":12,"bid_ct_00":16,"ask_ct_00":10,"symbol":"ESH1"}\n{"ts_event":160'
718-
'9160400007,"rtype":1,"publisher_id":1,"instrument_id":5482,"action":"C","side":"A","'
719-
'depth":0,"price":3720.5,"size":1,"flags":128,"ts_in_delta":17361,"sequence":1170366,'
720-
'"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":26,"ask_sz_00":11,"bid_ct_00":16,'
721-
'"ask_ct_00":9,"symbol":"ESH1"}'
719+
'{"ts_recv":1609160400006,"ts_event":1609160400006,"rtype":1,"publisher_id":1,"instrument_id":5482,'
720+
'"action":"A","side":"A","depth":0,"price":3720.5,"size":1,"flags":128,"ts_in_delta":17214,'
721+
'"sequence":1170362,"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":24,"ask_sz_00":11,'
722+
'"bid_ct_00":15,"ask_ct_00":9,"symbol":"ESH1"}'
723+
"\n"
724+
'{"ts_recv":1609160400006,"ts_event":1609160400006,"rtype":1,"publisher_id":1,"instrument_id":5482,'
725+
'"action":"A","side":"A","depth":0,"price":3720.5,"size":1,"flags":128,"ts_in_delta":18858,'
726+
'"sequence":1170364,"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":24,"ask_sz_00":12,'
727+
'"bid_ct_00":15,"ask_ct_00":10,"symbol":"ESH1"}'
728+
"\n"
729+
'{"ts_recv":1609160400007,"ts_event":1609160400007,"rtype":1,"publisher_id":1,"instrument_id":5482,'
730+
'"action":"A","side":"B","depth":0,"price":3720.25,"size":2,"flags":128,"ts_in_delta":18115,'
731+
'"sequence":1170365,"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":26,"ask_sz_00":12,'
732+
'"bid_ct_00":16,"ask_ct_00":10,"symbol":"ESH1"}'
733+
"\n"
734+
'{"ts_recv":1609160400007,"ts_event":1609160400007,"rtype":1,"publisher_id":1,"instrument_id":5482,'
735+
'"action":"C","side":"A","depth":0,"price":3720.5,"size":1,"flags":128,"ts_in_delta":17361,'
736+
'"sequence":1170366,"bid_px_00":3720.25,"ask_px_00":3720.5,"bid_sz_00":26,"ask_sz_00":11,'
737+
'"bid_ct_00":16,"ask_ct_00":9,"symbol":"ESH1"}'
722738
)
723739

724740

0 commit comments

Comments
 (0)