MOD: Rename STypes across bento (python+multidocs)

Author: cjdsellers

databento/common/data.py

@@ -40,7 +40,7 @@ def get_deriv_ba_types(level: int) -> List[Tuple[str, Union[type, str]]]:
     ("length", np.uint8),
     ("rtype", np.uint8),
     ("publisher_id", np.uint16),
-    ("product_id", np.uint32),
+    ("instrument_id", np.uint32),
     ("ts_event", np.uint64),
 ]
 
@@ -271,7 +271,7 @@ def get_deriv_ba_fields(level: int) -> List[str]:
     "ts_event",
     "ts_in_delta",
     "publisher_id",
-    "product_id",
+    "instrument_id",
     "action",
     "side",
     "depth",
@@ -283,7 +283,7 @@ def get_deriv_ba_fields(level: int) -> List[str]:
 
 OHLCV_HEADER_COLUMNS = [
     "publisher_id",
-    "product_id",
+    "instrument_id",
     "open",
     "high",
     "low",
@@ -330,7 +330,7 @@ def get_deriv_ba_fields(level: int) -> List[str]:
         "ts_in_delta",
         "publisher_id",
         "channel_id",
-        "product_id",
+        "instrument_id",
         "order_id",
         "action",
         "side",

databento/common/dbnstore.py

@@ -32,7 +32,7 @@
 from databento.common.enums import Compression, Schema, SType
 from databento.common.error import BentoError
 from databento.common.metadata import MetadataDecoder
-from databento.common.symbology import ProductIdMappingInterval
+from databento.common.symbology import InstrumentIdMappingInterval
 
 
 logger = logging.getLogger(__name__)
@@ -339,7 +339,7 @@ def __init__(self, data_source: DataSource) -> None:
         )
 
         # This is populated when _map_symbols is called
-        self._product_id_index: Dict[
+        self._instrument_id_index: Dict[
             dt.date,
             Dict[int, str],
         ] = {}
@@ -392,19 +392,19 @@ def _apply_pretty_px(self, df: pd.DataFrame) -> pd.DataFrame:
 
         return df
 
-    def _build_product_id_index(self) -> Dict[dt.date, Dict[int, str]]:
-        intervals: List[ProductIdMappingInterval] = []
-        for native, i in self.mappings.items():
+    def _build_instrument_id_index(self) -> Dict[dt.date, Dict[int, str]]:
+        intervals: List[InstrumentIdMappingInterval] = []
+        for raw_symbol, i in self.mappings.items():
             for row in i:
                 symbol = row["symbol"]
                 if symbol == "":
                     continue
                 intervals.append(
-                    ProductIdMappingInterval(
+                    InstrumentIdMappingInterval(
                         start_date=row["start_date"],
                         end_date=row["end_date"],
-                        native=native,
-                        product_id=int(row["symbol"]),
+                        raw_symbol=raw_symbol,
+                        instrument_id=int(row["symbol"]),
                     ),
                 )
 
@@ -420,7 +420,7 @@ def _build_product_id_index(self) -> Dict[dt.date, Dict[int, str]]:
                 date_map: Dict[int, str] = product_id_index.get(d, {})
                 if not date_map:
                     product_id_index[d] = date_map
-                date_map[interval.product_id] = interval.native
+                date_map[interval.instrument_id] = interval.raw_symbol
 
         return product_id_index
 
@@ -457,17 +457,17 @@ def _get_index_column(self) -> str:
         )
 
     def _map_symbols(self, df: pd.DataFrame, pretty_ts: bool) -> pd.DataFrame:
-        # Build product ID index
-        if not self._product_id_index:
-            self._product_id_index = self._build_product_id_index()
+        # Build instrument ID index
+        if not self._instrument_id_index:
+            self._instrument_id_index = self._build_instrument_id_index()
 
-        # Map product IDs to native symbols
-        if self._product_id_index:
+        # Map instrument IDs to raw symbols
+        if self._instrument_id_index:
             df_index = df.index if pretty_ts else pd.to_datetime(df.index, utc=True)
             dates = [ts.date() for ts in df_index]
             df["symbol"] = [
-                self._product_id_index[dates[i]][p]
-                for i, p in enumerate(df["product_id"])
+                self._instrument_id_index[dates[i]][p]
+                for i, p in enumerate(df["instrument_id"])
             ]
 
         return df
@@ -864,7 +864,7 @@ def to_csv(
             the correct scale (using the fixed precision scalar 1e-9).
         map_symbols : bool, default True
             If symbology mappings from the metadata should be used to create
-            a 'symbol' column, mapping the product ID to its native symbol for
+            a 'symbol' column, mapping the instrument ID to its native symbol for
             every record.
 
         Notes
@@ -897,7 +897,7 @@ def to_df(
             the correct scale (using the fixed precision scalar 1e-9).
         map_symbols : bool, default True
             If symbology mappings from the metadata should be used to create
-            a 'symbol' column, mapping the product ID to its native symbol for
+            a 'symbol' column, mapping the instrument ID to its native symbol for
             every record.
 
         Returns
@@ -955,7 +955,7 @@ def to_json(
             the correct scale (using the fixed precision scalar 1e-9).
         map_symbols : bool, default True
             If symbology mappings from the metadata should be used to create
-            a 'symbol' column, mapping the product ID to its native symbol for
+            a 'symbol' column, mapping the instrument ID to its native symbol for
             every record.
 
         Notes

databento/common/enums.py

@@ -211,9 +211,13 @@ class Delivery(StringyMixin, str, Enum):
 class SType(StringyMixin, str, Enum):
     """Represents a symbology type."""
 
-    PRODUCT_ID = "product_id"
-    NATIVE = "native"
-    SMART = "smart"
+    PRODUCT_ID = "product_id"  # Deprecated for `instrument_id`
+    NATIVE = "native"  # Deprecated for `raw_symbol`
+    SMART = "smart"  # Deprecated for `parent` and `continuous`
+    INSTRUMENT_ID = "instrument_id"
+    RAW_SYMBOL = "raw_symbol"
+    PARENT = "parent"
+    CONTINUOUS = "continuous"
 
 
 @unique

databento/common/metadata.py

@@ -1,5 +1,6 @@
 from typing import Dict
 
+from databento.common.enums import SType
 from databento_dbn import decode_metadata
 
 
@@ -38,4 +39,25 @@ def decode_to_json(raw_metadata: bytes) -> Dict[str, object]:
                 value = conversion_mapping[field](value)
             metadata_dict[field] = value
 
+        # TODO(cs): Temporary mappings until new DBN released
+        stype_in = metadata_dict["stype_in"]
+        stype_out = metadata_dict["stype_out"]
+
+        if stype_in == SType.NATIVE.value:
+            stype_in = SType.RAW_SYMBOL.value
+        elif stype_in == SType.PRODUCT_ID.value:
+            stype_in = SType.INSTRUMENT_ID.value
+        else:
+            stype_in = SType.SMART.value
+
+        if stype_out == SType.NATIVE.value:
+            stype_out = SType.INSTRUMENT_ID.value
+        elif stype_out == SType.PRODUCT_ID.value:
+            stype_out = SType.INSTRUMENT_ID.value
+        else:
+            stype_out = SType.SMART.value
+
+        metadata_dict["stype_in"] = stype_in
+        metadata_dict["stype_out"] = stype_out
+
         return metadata_dict

databento/common/parsing.py

@@ -109,11 +109,11 @@ def _(symbols: int, stype_in: SType) -> str:
     optional_symbols_list_to_string
 
     """
-    if stype_in == SType.PRODUCT_ID:
+    if stype_in == SType.INSTRUMENT_ID:
         return str(symbols)
     raise ValueError(
         f"value `{symbols}` is not a valid symbol for stype {stype_in}; "
-        "did you mean to use `product_id`?",
+        "did you mean to use `instrument_id`?",
     )
 
 
@@ -142,7 +142,8 @@ def _(symbols: str, stype_in: SType) -> str:
         symbol_list = symbols.strip().strip(",").split(",")
         return ",".join(map(symbol_to_string, symbol_list))
 
-    if stype_in == SType.SMART:
+    # TODO(cs): Temporary mapping until new DBN released
+    if stype_in in (SType.SMART, SType.CONTINUOUS):
         return validate_smart_symbol(symbols)
     return symbols.strip().upper()
 

databento/common/symbology.py

@@ -6,9 +6,9 @@
 
 
 @dataclass(frozen=True)
-class ProductIdMappingInterval:
+class InstrumentIdMappingInterval:
     """
-    Represents a product ID to native symbol mapping over a start and end date
+    Represents an instrument ID to raw symbol mapping over a start and end date
     range interval.
 
     Parameters
@@ -17,13 +17,13 @@ class ProductIdMappingInterval:
         The start of the mapping period.
     end_date : dt.date
         The end of the mapping period.
-    native : str
+    raw_symbol : str
         The native symbol value.
-    product_id : int
-        The product ID value.
+    instrument_id : int
+        The instrument ID value.
     """
 
     start_date: dt.date
     end_date: dt.date
-    native: str
-    product_id: int
+    raw_symbol: str
+    instrument_id: int

databento/historical/api/batch.py

@@ -65,8 +65,8 @@ def submit_job(
         split_size: Optional[int] = None,
         packaging: Optional[Union[Packaging, str]] = None,
         delivery: Union[Delivery, str] = "download",
-        stype_in: Union[SType, str] = "native",
-        stype_out: Union[SType, str] = "product_id",
+        stype_in: Union[SType, str] = "raw_symbol",
+        stype_out: Union[SType, str] = "instrument_id",
         limit: Optional[int] = None,
     ) -> Dict[str, Any]:
         """
@@ -107,9 +107,9 @@ def submit_job(
             The archive type to package all batched data files in.
         delivery : Delivery or str {'download', 's3', 'disk'}, default 'download'
             The delivery mechanism for the processed batched data files.
-        stype_in : SType or str, default 'native'
+        stype_in : SType or str, default 'raw_symbol'
             The input symbology type to resolve from.
-        stype_out : SType or str, default 'product_id'
+        stype_out : SType or str, default 'instrument_id'
             The output symbology type to resolve to.
         limit : int, optional
             The maximum number of records to return. If `None` then no limit.

databento/historical/api/metadata.py

@@ -276,7 +276,7 @@ def get_record_count(
         end: Optional[Union[pd.Timestamp, date, str, int]] = None,
         symbols: Optional[Union[List[str], str]] = None,
         schema: Union[Schema, str] = "trades",
-        stype_in: Optional[Union[SType, str]] = "native",
+        stype_in: Optional[Union[SType, str]] = "raw_symbol",
         limit: Optional[int] = None,
     ) -> int:
         """
@@ -303,7 +303,7 @@ def get_record_count(
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
-        stype_in : SType or str, default 'native'
+        stype_in : SType or str, default 'raw_symbol'
             The input symbology type to resolve from.
         limit : int, optional
             The maximum number of records to return. If `None` then no limit.
@@ -344,7 +344,7 @@ def get_billable_size(
         end: Optional[Union[pd.Timestamp, date, str, int]] = None,
         symbols: Optional[Union[List[str], str]] = None,
         schema: Union[Schema, str] = "trades",
-        stype_in: Optional[Union[SType, str]] = "native",
+        stype_in: Optional[Union[SType, str]] = "raw_symbol",
         limit: Optional[int] = None,
     ) -> int:
         """
@@ -372,7 +372,7 @@ def get_billable_size(
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
-        stype_in : SType or str, default 'native'
+        stype_in : SType or str, default 'raw_symbol'
             The input symbology type to resolve from.
         limit : int, optional
             The maximum number of records to return. If `None` then no limit.
@@ -392,7 +392,7 @@ def get_billable_size(
             ("symbols", symbols_list),
             ("schema", str(validate_enum(schema, Schema, "schema"))),
             ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(SType.PRODUCT_ID)),
+            ("stype_out", str(SType.INSTRUMENT_ID)),
         ]
 
         if limit is not None:
@@ -414,7 +414,7 @@ def get_cost(
         mode: Union[FeedMode, str] = "historical-streaming",
         symbols: Optional[Union[List[str], str]] = None,
         schema: Union[Schema, str] = "trades",
-        stype_in: Optional[Union[SType, str]] = "native",
+        stype_in: Optional[Union[SType, str]] = "raw_symbol",
         limit: Optional[int] = None,
     ) -> float:
         """
@@ -444,7 +444,7 @@ def get_cost(
             If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
-        stype_in : SType or str, default 'native'
+        stype_in : SType or str, default 'raw_symbol'
             The input symbology type to resolve from.
         limit : int, optional
             The maximum number of records to return. If `None` then no limit.
@@ -464,7 +464,7 @@ def get_cost(
             ("symbols", symbols_list),
             ("schema", str(validate_enum(schema, Schema, "schema"))),
             ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(SType.PRODUCT_ID)),
+            ("stype_out", str(SType.INSTRUMENT_ID)),
             ("mode", validate_enum(mode, FeedMode, "mode")),
         ]
 

databento/historical/api/symbology.py

@@ -43,9 +43,9 @@ def resolve(
             The dataset code (string identifier) for the request.
         symbols : List[Union[str, int]] or str, optional
             The symbols to resolve. Takes up to 2,000 symbols per request.
-        stype_in : SType or str, default 'native'
+        stype_in : SType or str, default 'raw_symbol'
             The input symbology type to resolve from.
-        stype_out : SType or str, default 'product_id'
+        stype_out : SType or str, default 'instrument_id'
             The output symbology type to resolve to.
         start_date : date or str
             The start date (UTC) of the request time range (inclusive).