FIX: Invalid symbol mappings from tz offset

Author: nmacholl

CHANGELOG.md

@@ -3,10 +3,13 @@
 ## 0.30.0 - TBD
 
 #### Enhancements
-- Changed how `SymbolMappingMsg` objects are ingested by `InstrumentMap` to single source the timestamp parsing from the `dbn` package.
+- Changed how `SymbolMappingMsg` objects are ingested by `InstrumentMap` to single source the timestamp parsing from the `databento-dbn` package
+
+#### Bug fixes
+- Fixed an issue where setting a timezone in `DBNStore.to_df` could cause invalid symbol mappings
 
 #### Breaking changes
-- Changed `Live.add_stream` to use the exclusive write mode when handling file paths so existing files won't be overwritten.
+- Changed `Live.add_stream` to use the exclusive write mode when handling file paths so existing files won't be overwritten
 
 ## 0.29.0 - 2024-02-13
 

databento/common/dbnstore.py

@@ -1378,16 +1378,19 @@ def __next__(self) -> pd.DataFrame:
 
         self._format_hidden_fields(df)
 
-        self._format_px(df, self._price_type)
-
         if self._pretty_ts:
             self._format_pretty_ts(df)
 
-        self._format_set_index(df)
-
         if self._map_symbols:
             self._format_map_symbols(df)
 
+        if self._pretty_ts:
+            self._format_timezone(df)
+
+        self._format_px(df, self._price_type)
+
+        self._format_set_index(df)
+
         return df
 
     def _format_definition_fields(self, df: pd.DataFrame) -> None:
@@ -1402,13 +1405,20 @@ def _format_hidden_fields(self, df: pd.DataFrame) -> None:
                 df[column] = df[column].str.decode("utf-8")
 
     def _format_map_symbols(self, df: pd.DataFrame) -> None:
-        df_index = df.index if self._pretty_ts else pd.to_datetime(df.index, utc=True)
+        # the first ordered field will be ts_recv or ts_event when appropriate
+        ts_name = self._struct_type._ordered_fields[0]
+
+        df_index = df[ts_name] if self._pretty_ts else pd.to_datetime(df[ts_name], utc=True)
         dates = [ts.date() for ts in df_index]
         df["symbol"] = [
             self._instrument_map.resolve(inst, dates[i])
             for i, inst in enumerate(df["instrument_id"])
         ]
 
+    def _format_timezone(self, df: pd.DataFrame) -> None:
+        for field in self._struct_type._timestamp_fields:
+            df[field] = df[field].dt.tz_convert(self._tz)
+
     def _format_px(
         self,
         df: pd.DataFrame,
@@ -1429,8 +1439,9 @@ def _format_px(
 
     def _format_pretty_ts(self, df: pd.DataFrame) -> None:
         for field in self._struct_type._timestamp_fields:
-            df[field] = pd.to_datetime(df[field], utc=True, errors="coerce").dt.tz_convert(self._tz)
+            df[field] = pd.to_datetime(df[field], utc=True, errors="coerce")
 
     def _format_set_index(self, df: pd.DataFrame) -> None:
+        # the first ordered field will be ts_recv or ts_event when appropriate
         index_column = self._struct_type._ordered_fields[0]
         df.set_index(index_column, inplace=True)