MOD: Update symbols parsing for smart symbology

Author: cjdsellers

databento/common/parsing.py

@@ -178,6 +178,7 @@ def maybe_values_list_to_string(
 
 def maybe_symbols_list_to_string(
     symbols: Optional[Union[Iterable[str], str]],
+    stype_in: SType,
 ) -> Optional[str]:
     """
     Concatenate a symbols string or iterable of symbol strings (if not None).
@@ -186,21 +187,31 @@ def maybe_symbols_list_to_string(
     ----------
     symbols : iterable of str or str, optional
         The symbols to concatenate.
+    stype_in : SType
+        The input symbology type for the request.
 
     Returns
     -------
     str or ``None``
 
     """
     if symbols is None:
-        return None  # All symbols
-
-    if isinstance(symbols, str):
-        return symbols.strip().rstrip(",").upper()
-    elif isinstance(symbols, Iterable):
-        return ",".join(symbols).strip().upper()
-    else:
-        raise TypeError(f"invalid symbols type, was {type(symbols)}")
+        return None  # Full universe
+
+    symbols_list = symbols.split(",") if isinstance(symbols, str) else list(symbols)
+    cleaned_symbols: List[str] = []
+    for symbol in symbols_list:
+        if not symbol:
+            continue
+        symbol = symbol.strip().upper()
+        if stype_in == SType.SMART:
+            pieces: List[str] = symbol.split(".")
+            if len(pieces) == 3:
+                symbol = f"{pieces[0]}.{pieces[1].lower()}.{pieces[2]}"
+
+        cleaned_symbols.append(symbol)
+
+    return ",".join(cleaned_symbols)
 
 
 def maybe_date_to_string(value: Optional[Union[date, str]]) -> Optional[str]:

databento/historical/api/metadata.py

@@ -331,7 +331,7 @@ def get_record_count(
 
         params: List[Tuple[str, Optional[str]]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("symbols", maybe_symbols_list_to_string(symbols)),
+            ("symbols", maybe_symbols_list_to_string(symbols, SType(stype_in))),
             ("schema", Schema(schema).value),
             ("start", maybe_datetime_to_string(start)),
             ("end", maybe_datetime_to_string(end)),

databento/historical/api/symbology.py

@@ -63,7 +63,7 @@ def resolve(
         """
         params: List[Tuple[str, Optional[str]]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("symbols", maybe_symbols_list_to_string(symbols)),
+            ("symbols", maybe_symbols_list_to_string(symbols, SType(stype_in))),
             ("stype_in", enum_or_str_lowercase(stype_in, "stype_in")),
             ("stype_out", enum_or_str_lowercase(stype_out, "stype_out")),
             ("start_date", str(pd.to_datetime(start_date).date())),

databento/historical/api/timeseries.py

@@ -246,7 +246,7 @@ def _pre_check_data_size(  # noqa (prefer not to make static)
 
 def _is_large_number_of_symbols(symbols: Optional[Union[List[str], str]]) -> bool:
     if not symbols:
-        return True  # All symbols
+        return True  # Full universe
 
     if isinstance(symbols, str):
         symbols = symbols.split(",")

databento/historical/http.py

@@ -1,7 +1,7 @@
 import sys
 from datetime import date
 from json.decoder import JSONDecodeError
-from typing import Any, BinaryIO, List, Optional, Sequence, Tuple, Union
+from typing import Any, BinaryIO, List, Optional, Tuple, Union
 
 import aiohttp
 import pandas as pd
@@ -54,7 +54,7 @@ def _timeseries_params(
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
-            ("symbols", maybe_symbols_list_to_string(symbols) or "*"),
+            ("symbols", maybe_symbols_list_to_string(symbols, SType(stype_in)) or "*"),
             ("schema", schema.value),
             ("stype_in", stype_in.value),
             ("stype_out", stype_out.value),
@@ -179,7 +179,7 @@ def _stream(
     async def _stream_async(
         self,
         url: str,
-        params: Sequence[Tuple[str, Optional[str]]],
+        params: List[Tuple[str, Optional[str]]],
         basic_auth: bool,
         bento: Bento,
     ) -> None:

tests/test_common_parsing.py

@@ -5,7 +5,7 @@
 import numpy as np
 import pandas as pd
 import pytest
-from databento.common.enums import Dataset, Flags
+from databento.common.enums import Dataset, Flags, SType
 from databento.common.parsing import (
     enum_or_str_lowercase,
     enum_or_str_uppercase,
@@ -159,17 +159,19 @@ def test_maybe_symbols_list_to_string_given_invalid_input_raises_type_error(
     ) -> None:
         # Arrange, Act, Assert
         with pytest.raises(TypeError):
-            maybe_symbols_list_to_string(INCORRECT_TYPE)
+            maybe_symbols_list_to_string(INCORRECT_TYPE, SType.NATIVE)
 
     @pytest.mark.parametrize(
         "symbols, expected",
         [
             [None, None],
+            ["ES.fut", "ES.FUT"],
             ["ES,CL", "ES,CL"],
             ["ES,CL,", "ES,CL"],
             ["es,cl,", "ES,CL"],
             [["ES", "CL"], "ES,CL"],
             [["es", "cl"], "ES,CL"],
+            [["ES.N.0", "CL.n.0"], "ES.n.0,CL.n.0"],
         ],
     )
     def test_maybe_symbols_list_to_string_given_valid_inputs_returns_expected(
@@ -178,7 +180,7 @@ def test_maybe_symbols_list_to_string_given_valid_inputs_returns_expected(
         expected: str,
     ) -> None:
         # Arrange, Act
-        result: Optional[str] = maybe_symbols_list_to_string(symbols)
+        result: Optional[str] = maybe_symbols_list_to_string(symbols, SType.SMART)
 
         # Assert
         assert result == expected

tests/test_historical_timeseries.py

@@ -54,7 +54,8 @@ def test_stream_sends_expected_request(self, mocker: MockerFixture) -> None:
         # Act
         self.client.timeseries.stream(
             dataset="GLBX.MDP3",
-            symbols="ESH1",
+            symbols="ES.c.0",
+            stype_in="smart",
             schema="trades",
             start="2020-12-28T12:00",
             end="2020-12-29",
@@ -75,9 +76,9 @@ def test_stream_sends_expected_request(self, mocker: MockerFixture) -> None:
             ("dataset", "GLBX.MDP3"),
             ("start", "2020-12-28T12:00:00"),
             ("end", "2020-12-29T00:00:00"),
-            ("symbols", "ESH1"),
+            ("symbols", "ES.c.0"),
             ("schema", "trades"),
-            ("stype_in", "native"),
+            ("stype_in", "smart"),
             ("stype_out", "product_id"),
             ("encoding", "dbz"),
         ]