MOD: Batch large symbol subscriptions

Author: nmacholl

CHANGELOG.md

@@ -4,14 +4,19 @@
 
 #### Enhancements
 - Added `symbology_map` property to `Live` client
+- Added `optional_symbols_list_to_list` parsing function
 - Changed `Live.add_callback` and `Live.add_stream` to accept an exception callback
 - Changed `Live.__iter__()` and `Live.__aiter__()` to send the session start message if the session is connected but not started
 - Upgraded `databento-dbn` to 0.7.1
 - Removed `Encoding`, `Compression`, `Schema`, and `SType` enums as they are now exposed by `databento-dbn`
 - Removed exception chaining from exceptions emitted by the library
 
+#### Bug fixes
+- Fixed issue where a large unreadable symbol subscription message could be sent
+
 #### Breaking changes
 - Renamed `func` parameter to `record_callback` for `Live.add_callback` and `Live.add_stream`
+- Removed `optional_symbols_list_to_string` parsing function
 
 ## 0.14.1 - 2023-06-16
 

databento/common/parsing.py

@@ -58,12 +58,13 @@ def optional_values_list_to_string(
 
 
 @singledispatch
-def optional_symbols_list_to_string(
+def optional_symbols_list_to_list(
     symbols: Iterable[str] | Iterable[Number] | str | Number | None,
     stype_in: SType,
-) -> str:
+) -> list[str]:
     """
-    Concatenate a symbols string or iterable of symbol strings (if not None).
+    Create a list from a symbols string or iterable of symbol strings (if not
+    None).
 
     Parameters
     ----------
@@ -74,11 +75,11 @@ def optional_symbols_list_to_string(
 
     Returns
     -------
-    str
+    list[str]
 
     Notes
     -----
-    If None is given, ALL_SYMBOLS is returned.
+    If None is given, [ALL_SYMBOLS] is returned.
 
     """
     raise TypeError(
@@ -87,48 +88,48 @@ def optional_symbols_list_to_string(
     )
 
 
-@optional_symbols_list_to_string.register
-def _(_: None, __: SType) -> str:
+@optional_symbols_list_to_list.register
+def _(_: None, __: SType) -> list[str]:
     """
-    Dispatch method for optional_symbols_list_to_string. Handles None which
-    defaults to ALL_SYMBOLS.
+    Dispatch method for optional_symbols_list_to_list. Handles None which
+    defaults to [ALL_SYMBOLS].
 
     See Also
     --------
-    optional_symbols_list_to_string
+    optional_symbols_list_to_list
 
     """
-    return ALL_SYMBOLS
+    return [ALL_SYMBOLS]
 
 
-@optional_symbols_list_to_string.register
-def _(symbols: Number, stype_in: SType) -> str:
+@optional_symbols_list_to_list.register
+def _(symbols: Number, stype_in: SType) -> list[str]:
     """
-    Dispatch method for optional_symbols_list_to_string. Handles numerical
-    types, alerting when an integer is given for STypes that expect strings.
+    Dispatch method for optional_symbols_list_to_list. Handles numerical types,
+    alerting when an integer is given for STypes that expect strings.
 
     See Also
     --------
-    optional_symbols_list_to_string
+    optional_symbols_list_to_list
 
     """
     if stype_in == SType.INSTRUMENT_ID:
-        return str(symbols)
+        return [str(symbols)]
     raise ValueError(
         f"value `{symbols}` is not a valid symbol for stype {stype_in}; "
         "did you mean to use `instrument_id`?",
     )
 
 
-@optional_symbols_list_to_string.register
-def _(symbols: str, stype_in: SType) -> str:
+@optional_symbols_list_to_list.register
+def _(symbols: str, stype_in: SType) -> list[str]:
     """
-    Dispatch method for optional_symbols_list_to_string. Handles str, splitting
+    Dispatch method for optional_symbols_list_to_list. Handles str, splitting
     on commas and validating smart symbology.
 
     See Also
     --------
-    optional_symbols_list_to_string
+    optional_symbols_list_to_list
 
     """
     if not symbols:
@@ -137,35 +138,33 @@ def _(symbols: str, stype_in: SType) -> str:
             "an empty string is not allowed",
         )
 
-    if "," in symbols:
-        symbol_to_string = partial(
-            optional_symbols_list_to_string,
-            stype_in=stype_in,
-        )
-        symbol_list = symbols.strip().strip(",").split(",")
-        return ",".join(map(symbol_to_string, symbol_list))
+    symbol_list = symbols.strip().strip(",").split(",")
 
     if stype_in in (SType.PARENT, SType.CONTINUOUS):
-        return validate_smart_symbol(symbols)
-    return symbols.strip().upper()
+        return list(map(str.strip, map(validate_smart_symbol, symbol_list)))
+
+    return list(map(str.upper, map(str.strip, symbol_list)))
 
 
-@optional_symbols_list_to_string.register(cls=Iterable)
-def _(symbols: Iterable[str] | Iterable[int], stype_in: SType) -> str:
+@optional_symbols_list_to_list.register(cls=Iterable)
+def _(symbols: Iterable[str] | Iterable[int], stype_in: SType) -> list[str]:
     """
-    Dispatch method for optional_symbols_list_to_string. Handles Iterables by
+    Dispatch method for optional_symbols_list_to_list. Handles Iterables by
     dispatching the individual members.
 
     See Also
     --------
-    optional_symbols_list_to_string
+    optional_symbols_list_to_list
 
     """
-    symbol_to_string = partial(
-        optional_symbols_list_to_string,
+    symbol_to_list = partial(
+        optional_symbols_list_to_list,
         stype_in=stype_in,
     )
-    return ",".join(map(symbol_to_string, symbols))
+    aggregated: list[str] = []
+    for sym in map(symbol_to_list, symbols):
+        aggregated.extend(sym)
+    return aggregated
 
 
 def optional_date_to_string(value: date | str | None) -> str | None:

databento/historical/api/batch.py

@@ -22,7 +22,7 @@
 from databento.common.enums import SplitDuration
 from databento.common.parsing import datetime_to_string
 from databento.common.parsing import optional_datetime_to_string
-from databento.common.parsing import optional_symbols_list_to_string
+from databento.common.parsing import optional_symbols_list_to_list
 from databento.common.parsing import optional_values_list_to_string
 from databento.common.validation import validate_enum
 from databento.common.validation import validate_path
@@ -118,12 +118,12 @@ def submit_job(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         data: dict[str, object | None] = {
             "dataset": validate_semantic_string(dataset, "dataset"),
             "start": datetime_to_string(start),
             "end": optional_datetime_to_string(end),
-            "symbols": str(symbols_list),
+            "symbols": ",".join(symbols_list),
             "schema": str(validate_enum(schema, Schema, "schema")),
             "stype_in": str(stype_in_valid),
             "stype_out": str(validate_enum(stype_out, SType, "stype_out")),

databento/historical/api/metadata.py

@@ -14,7 +14,7 @@
 from databento.common.parsing import datetime_to_string
 from databento.common.parsing import optional_date_to_string
 from databento.common.parsing import optional_datetime_to_string
-from databento.common.parsing import optional_symbols_list_to_string
+from databento.common.parsing import optional_symbols_list_to_list
 from databento.common.validation import validate_enum
 from databento.common.validation import validate_maybe_enum
 from databento.common.validation import validate_semantic_string
@@ -318,10 +318,10 @@ def get_record_count(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         params: list[tuple[str, str | None]] = [
             ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("symbols", symbols_list),
+            ("symbols", ",".join(symbols_list)),
             ("schema", str(validate_enum(schema, Schema, "schema"))),
             ("start", optional_datetime_to_string(start)),
             ("end", optional_datetime_to_string(end)),
@@ -387,12 +387,12 @@ def get_billable_size(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         params: list[tuple[str, str | None]] = [
             ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", optional_datetime_to_string(end)),
-            ("symbols", symbols_list),
+            ("symbols", ",".join(symbols_list)),
             ("schema", str(validate_enum(schema, Schema, "schema"))),
             ("stype_in", str(stype_in_valid)),
             ("stype_out", str(SType.INSTRUMENT_ID)),
@@ -459,12 +459,12 @@ def get_cost(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         params: list[tuple[str, str | None]] = [
             ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", optional_datetime_to_string(end)),
-            ("symbols", symbols_list),
+            ("symbols", ",".join(symbols_list)),
             ("schema", str(validate_enum(schema, Schema, "schema"))),
             ("stype_in", str(stype_in_valid)),
             ("stype_out", str(SType.INSTRUMENT_ID)),

databento/historical/api/symbology.py

@@ -9,7 +9,7 @@
 from databento.common.enums import Dataset
 from databento.common.parsing import datetime_to_date_string
 from databento.common.parsing import optional_date_to_string
-from databento.common.parsing import optional_symbols_list_to_string
+from databento.common.parsing import optional_symbols_list_to_list
 from databento.common.validation import validate_enum
 from databento.common.validation import validate_semantic_string
 from databento.historical.api import API_VERSION
@@ -65,10 +65,10 @@ def resolve(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         data: dict[str, object | None] = {
             "dataset": validate_semantic_string(dataset, "dataset"),
-            "symbols": symbols_list,
+            "symbols": ",".join(symbols_list),
             "stype_in": str(stype_in_valid),
             "stype_out": str(validate_enum(stype_out, SType, "stype_out")),
             "start_date": datetime_to_date_string(start_date),

databento/historical/api/timeseries.py

@@ -13,7 +13,7 @@
 from databento.common.enums import Dataset
 from databento.common.parsing import datetime_to_string
 from databento.common.parsing import optional_datetime_to_string
-from databento.common.parsing import optional_symbols_list_to_string
+from databento.common.parsing import optional_symbols_list_to_list
 from databento.common.validation import validate_enum
 from databento.common.validation import validate_semantic_string
 from databento.historical.api import API_VERSION
@@ -95,15 +95,15 @@ def get_range(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         schema_valid = validate_enum(schema, Schema, "schema")
         start_valid = datetime_to_string(start)
         end_valid = optional_datetime_to_string(end)
         data: dict[str, object | None] = {
             "dataset": validate_semantic_string(dataset, "dataset"),
             "start": start_valid,
             "end": end_valid,
-            "symbols": symbols_list,
+            "symbols": ",".join(symbols_list),
             "schema": str(schema_valid),
             "stype_in": str(stype_in_valid),
             "stype_out": str(validate_enum(stype_out, SType, "stype_out")),
@@ -189,15 +189,15 @@ async def get_range_async(
 
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
-        symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
+        symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
         schema_valid = validate_enum(schema, Schema, "schema")
         start_valid = datetime_to_string(start)
         end_valid = optional_datetime_to_string(end)
         data: dict[str, object | None] = {
             "dataset": validate_semantic_string(dataset, "dataset"),
             "start": start_valid,
             "end": end_valid,
-            "symbols": symbols_list,
+            "symbols": ",".join(symbols_list),
             "schema": str(schema_valid),
             "stype_in": str(stype_in_valid),
             "stype_out": str(validate_enum(stype_out, SType, "stype_out")),

databento/live/client.py

@@ -18,7 +18,6 @@
 from databento.common.enums import Dataset
 from databento.common.error import BentoError
 from databento.common.parsing import optional_datetime_to_unix_nanoseconds
-from databento.common.parsing import optional_symbols_list_to_string
 from databento.common.symbology import ALL_SYMBOLS
 from databento.common.validation import validate_enum
 from databento.common.validation import validate_semantic_string
@@ -453,7 +452,6 @@ def subscribe(
         dataset = validate_semantic_string(dataset, "dataset")
         schema = validate_enum(schema, Schema, "schema")
         stype_in = validate_enum(stype_in, SType, "stype_in")
-        symbols = optional_symbols_list_to_string(symbols, stype_in)
         start = optional_datetime_to_unix_nanoseconds(start)
 
         if not self.dataset:

databento/live/gateway.py

@@ -18,7 +18,6 @@
 
 T = TypeVar("T", bound="GatewayControl")
 
-
 @dataclasses.dataclass
 class GatewayControl:
     """