MOD: Standardize ALL_SYMBOLS representation

Author: cjdsellers

databento/common/parsing.py

@@ -4,6 +4,7 @@
 
 import pandas as pd
 from databento.common.enums import Compression, Encoding, Flags, Schema, SType
+from databento.common.symbology import ALL_SYMBOLS
 
 
 def enum_or_str_lowercase(
@@ -39,7 +40,7 @@ def enum_or_str_lowercase(
     raise TypeError(f"invalid `{param}` type, was {type(value)}.")
 
 
-def maybe_enum_or_str_lowercase(
+def optional_enum_or_str_lowercase(
     value: Optional[Union[Enum, str]],
     param: str,
 ) -> Optional[str]:
@@ -101,7 +102,7 @@ def enum_or_str_uppercase(
     raise TypeError(f"invalid `{param}` type, was {type(value)}.")
 
 
-def maybe_enum_or_str_uppercase(
+def optional_enum_or_str_uppercase(
     value: Optional[Union[Enum, str]],
     param: str,
 ) -> Optional[str]:
@@ -154,7 +155,7 @@ def values_list_to_string(
         raise TypeError(f"invalid values type, was {type(values)}")
 
 
-def maybe_values_list_to_string(
+def optional_values_list_to_string(
     values: Optional[Union[Iterable[str], str]],
 ) -> Optional[str]:
     """
@@ -176,10 +177,10 @@ def maybe_values_list_to_string(
     return values_list_to_string(values)
 
 
-def maybe_symbols_list_to_string(
+def optional_symbols_list_to_string(
     symbols: Optional[Union[Iterable[str], str]],
     stype_in: SType,
-) -> Optional[str]:
+) -> str:
     """
     Concatenate a symbols string or iterable of symbol strings (if not None).
 
@@ -192,11 +193,11 @@ def maybe_symbols_list_to_string(
 
     Returns
     -------
-    str or ``None``
+    str
 
     """
     if symbols is None:
-        return None  # Full universe
+        return ALL_SYMBOLS
 
     symbols_list = symbols.split(",") if isinstance(symbols, str) else list(symbols)
     cleaned_symbols: List[str] = []
@@ -214,7 +215,7 @@ def maybe_symbols_list_to_string(
     return ",".join(cleaned_symbols)
 
 
-def maybe_date_to_string(value: Optional[Union[date, str]]) -> Optional[str]:
+def optional_date_to_string(value: Optional[Union[date, str]]) -> Optional[str]:
     """
     Return a valid date string from the given value (if not None).
 
@@ -251,7 +252,7 @@ def datetime_to_string(value: Union[pd.Timestamp, date, str, int]) -> str:
     return str(pd.to_datetime(value)).replace(" ", "T")
 
 
-def maybe_datetime_to_string(
+def optional_datetime_to_string(
     value: Optional[Union[pd.Timestamp, date, str, int]],
 ) -> Optional[str]:
     """

databento/common/symbology.py

@@ -2,6 +2,9 @@
 from dataclasses import dataclass
 
 
+ALL_SYMBOLS = "ALL_SYMBOLS"
+
+
 @dataclass(frozen=True)
 class ProductIdMappingInterval:
     """

databento/historical/api/__init__.py

@@ -1 +1,2 @@
 API_VERSION = 0
+ALL_SYMBOLS = "ALL_SYMBOLS"

databento/historical/api/batch.py

@@ -13,8 +13,8 @@
     SType,
 )
 from databento.common.parsing import (
-    maybe_datetime_to_string,
-    maybe_values_list_to_string,
+    optional_datetime_to_string,
+    optional_values_list_to_string,
 )
 from databento.common.validation import validate_enum
 from databento.historical.api import API_VERSION
@@ -67,7 +67,7 @@ def submit_job(
         symbols : List[Union[str, int]] or str
             The product symbols to filter for. Takes up to 2,000 symbols per request.
             If more than 1 symbol is specified, the data is merged and sorted by time.
-            If `*` or ``None`` then will be for **all** symbols.
+            If 'ALL_SYMBOLS' or ``None`` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
         encoding : Encoding or str {'dbn', 'csv', 'json'}, default 'dbn'
@@ -163,8 +163,8 @@ def list_jobs(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("states", maybe_values_list_to_string(states)),
-            ("since", maybe_datetime_to_string(since)),
+            ("states", optional_values_list_to_string(states)),
+            ("since", optional_datetime_to_string(since)),
         ]
 
         return self._get(

databento/historical/api/metadata.py

@@ -5,10 +5,10 @@
 from databento.common.enums import Dataset, Encoding, FeedMode, Schema, SType
 from databento.common.parsing import (
     enum_or_str_uppercase,
-    maybe_date_to_string,
-    maybe_datetime_to_string,
-    maybe_enum_or_str_lowercase,
-    maybe_symbols_list_to_string,
+    optional_date_to_string,
+    optional_datetime_to_string,
+    optional_enum_or_str_lowercase,
+    optional_symbols_list_to_string,
 )
 from databento.common.validation import validate_enum, validate_maybe_enum
 from databento.historical.api import API_VERSION
@@ -70,8 +70,8 @@ def list_datasets(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("start_date", maybe_date_to_string(start_date)),
-            ("end_date", maybe_date_to_string(end_date)),
+            ("start_date", optional_date_to_string(start_date)),
+            ("end_date", optional_date_to_string(end_date)),
         ]
 
         response: Response = self._get(
@@ -108,8 +108,8 @@ def list_schemas(
         """
         params: List[Tuple[str, Optional[str]]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("start_date", maybe_date_to_string(start_date)),
-            ("end_date", maybe_date_to_string(end_date)),
+            ("start_date", optional_date_to_string(start_date)),
+            ("end_date", optional_date_to_string(end_date)),
         ]
 
         response: Response = self._get(
@@ -153,8 +153,8 @@ def list_fields(
 
         params: List[Tuple[str, str]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("schema", maybe_enum_or_str_lowercase(schema, "schema")),
-            ("encoding", maybe_enum_or_str_lowercase(encoding, "encoding")),
+            ("schema", optional_enum_or_str_lowercase(schema, "schema")),
+            ("encoding", optional_enum_or_str_lowercase(encoding, "encoding")),
         ]
 
         response: Response = self._get(
@@ -230,8 +230,8 @@ def list_unit_prices(
 
         params: List[Tuple[str, Optional[str]]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("mode", maybe_enum_or_str_lowercase(mode, "mode")),
-            ("schema", maybe_enum_or_str_lowercase(schema, "schema")),
+            ("mode", optional_enum_or_str_lowercase(mode, "mode")),
+            ("schema", optional_enum_or_str_lowercase(schema, "schema")),
         ]
 
         response: Response = self._get(
@@ -308,7 +308,7 @@ def get_record_count(
             If an integer is passed, then this represents nanoseconds since UNIX epoch.
         symbols : List[Union[str, int]] or str, optional
             The product symbols to filter for. Takes up to 2,000 symbols per request.
-            If `*` or ``None`` then will be for **all** symbols.
+            If 'ALL_SYMBOLS' or ``None`` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
         stype_in : SType or str, default 'native'
@@ -328,10 +328,10 @@ def get_record_count(
 
         params: List[Tuple[str, Optional[str]]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("symbols", maybe_symbols_list_to_string(symbols, SType(stype_in))),
+            ("symbols", optional_symbols_list_to_string(symbols, SType(stype_in))),
             ("schema", Schema(schema).value),
-            ("start", maybe_datetime_to_string(start)),
-            ("end", maybe_datetime_to_string(end)),
+            ("start", optional_datetime_to_string(start)),
+            ("end", optional_datetime_to_string(end)),
             ("stype_in", SType(stype_in).value),
         ]
         if limit is not None:
@@ -373,7 +373,7 @@ def get_billable_size(
             If an integer is passed, then this represents nanoseconds since UNIX epoch.
         symbols : List[Union[str, int]] or str, optional
             The product symbols to filter for. Takes up to 2,000 symbols per request.
-            If `*` or ``None`` then will be for **all** symbols.
+            If 'ALL_SYMBOLS' or ``None`` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
         stype_in : SType or str, default 'native'
@@ -439,7 +439,7 @@ def get_cost(
             The data feed mode for the request.
         symbols : List[Union[str, int]] or str, optional
             The product symbols to filter for. Takes up to 2,000 symbols per request.
-            If `*` or ``None`` then will be for **all** symbols.
+            If 'ALL_SYMBOLS' or ``None`` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
         stype_in : SType or str, default 'native'

databento/historical/api/symbology.py

@@ -6,7 +6,7 @@
 from databento.common.parsing import (
     enum_or_str_lowercase,
     enum_or_str_uppercase,
-    maybe_symbols_list_to_string,
+    optional_symbols_list_to_string,
 )
 from databento.historical.api import API_VERSION
 from databento.historical.http import BentoHttpAPI
@@ -63,7 +63,7 @@ def resolve(
         """
         params: List[Tuple[str, Optional[str]]] = [
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
-            ("symbols", maybe_symbols_list_to_string(symbols, SType(stype_in))),
+            ("symbols", optional_symbols_list_to_string(symbols, SType(stype_in))),
             ("stype_in", enum_or_str_lowercase(stype_in, "stype_in")),
             ("stype_out", enum_or_str_lowercase(stype_out, "stype_out")),
             ("start_date", str(pd.to_datetime(start_date).date())),

databento/historical/api/timeseries.py

@@ -55,7 +55,7 @@ def stream(
         symbols : List[Union[str, int]] or str, optional
             The product symbols to filter for. Takes up to 2,000 symbols per request.
             If more than 1 symbol is specified, the data is merged and sorted by time.
-            If `*` or ``None`` then will be for **all** symbols.
+            If 'ALL_SYMBOLS' or ``None`` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
         stype_in : SType or str, default 'native'
@@ -154,7 +154,7 @@ async def stream_async(
         symbols : List[Union[str, int]] or str, optional
             The product symbols to filter for. Takes up to 2,000 symbols per request.
             If more than 1 symbol is specified, the data is merged and sorted by time.
-            If `*` or ``None`` then will be for **all** symbols.
+            If 'ALL_SYMBOLS' or ``None`` then will be for **all** symbols.
         schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'  # noqa
             The data record schema for the request.
         stype_in : SType or str, default 'native'

databento/historical/http.py

@@ -13,7 +13,7 @@
 from databento.common.parsing import (
     datetime_to_string,
     enum_or_str_uppercase,
-    maybe_symbols_list_to_string,
+    optional_symbols_list_to_string,
 )
 from databento.historical.error import BentoClientError, BentoServerError
 from databento.version import __version__
@@ -54,7 +54,7 @@ def _timeseries_params(
             ("dataset", enum_or_str_uppercase(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
-            ("symbols", maybe_symbols_list_to_string(symbols, SType(stype_in)) or "*"),
+            ("symbols", optional_symbols_list_to_string(symbols, SType(stype_in))),
             ("schema", schema.value),
             ("stype_in", stype_in.value),
             ("stype_out", stype_out.value),

tests/test_common_parsing.py

@@ -9,12 +9,12 @@
 from databento.common.parsing import (
     enum_or_str_lowercase,
     enum_or_str_uppercase,
-    maybe_date_to_string,
-    maybe_datetime_to_string,
-    maybe_enum_or_str_lowercase,
-    maybe_enum_or_str_uppercase,
-    maybe_symbols_list_to_string,
-    maybe_values_list_to_string,
+    optional_date_to_string,
+    optional_datetime_to_string,
+    optional_enum_or_str_lowercase,
+    optional_enum_or_str_uppercase,
+    optional_symbols_list_to_string,
+    optional_values_list_to_string,
     parse_flags,
 )
 
@@ -56,7 +56,7 @@ def test_maybe_enum_or_str_lowercase_given_incorrect_types_raises_error(
     ) -> None:
         # Arrange, Act, Assert
         with pytest.raises(TypeError):
-            maybe_enum_or_str_lowercase(INCORRECT_TYPE, "param")
+            optional_enum_or_str_lowercase(INCORRECT_TYPE, "param")
 
     @pytest.mark.parametrize(
         "value, expected",
@@ -73,7 +73,7 @@ def test_maybe_enum_or_str_lowercase_returns_expected_outputs(
         expected: Optional[str],
     ) -> None:
         # Arrange, Act, Assert
-        assert maybe_enum_or_str_lowercase(value, "param") == expected
+        assert optional_enum_or_str_lowercase(value, "param") == expected
 
     def test_enum_or_str_uppercase_given_none_raises_type_error(self) -> None:
         # Arrange, Act, Assert
@@ -106,7 +106,7 @@ def test_maybe_enum_or_str_uppercase_given_incorrect_types_raises_error(
     ) -> None:
         # Arrange, Act, Assert
         with pytest.raises(TypeError):
-            maybe_enum_or_str_lowercase(INCORRECT_TYPE, "param")
+            optional_enum_or_str_lowercase(INCORRECT_TYPE, "param")
 
     @pytest.mark.parametrize(
         "value, expected",
@@ -123,14 +123,14 @@ def test_maybe_enum_or_str_uppercase_returns_expected_outputs(
         expected: Optional[str],
     ) -> None:
         # Arrange, Act, Assert
-        assert maybe_enum_or_str_uppercase(value, "param") == expected
+        assert optional_enum_or_str_uppercase(value, "param") == expected
 
     def test_maybe_values_list_to_string_given_invalid_input_raises_type_error(
         self,
     ) -> None:
         # Arrange, Act, Assert
         with pytest.raises(TypeError):
-            maybe_values_list_to_string(INCORRECT_TYPE)
+            optional_values_list_to_string(INCORRECT_TYPE)
 
     @pytest.mark.parametrize(
         "values, expected",
@@ -149,7 +149,7 @@ def test_maybe_values_list_to_string_given_valid_inputs_returns_expected(
         expected: str,
     ) -> None:
         # Arrange, Act
-        result: Optional[str] = maybe_values_list_to_string(values)
+        result: Optional[str] = optional_values_list_to_string(values)
 
         # Assert
         assert result == expected
@@ -159,12 +159,12 @@ def test_maybe_symbols_list_to_string_given_invalid_input_raises_type_error(
     ) -> None:
         # Arrange, Act, Assert
         with pytest.raises(TypeError):
-            maybe_symbols_list_to_string(INCORRECT_TYPE, SType.NATIVE)
+            optional_symbols_list_to_string(INCORRECT_TYPE, SType.NATIVE)
 
     @pytest.mark.parametrize(
         "symbols, expected",
         [
-            [None, None],
+            [None, "ALL_SYMBOLS"],
             ["ES.fut", "ES.FUT"],
             ["ES,CL", "ES,CL"],
             ["ES,CL,", "ES,CL"],
@@ -180,7 +180,7 @@ def test_maybe_symbols_list_to_string_given_valid_inputs_returns_expected(
         expected: str,
     ) -> None:
         # Arrange, Act
-        result: Optional[str] = maybe_symbols_list_to_string(symbols, SType.SMART)
+        result: str = optional_symbols_list_to_string(symbols, SType.SMART)
 
         # Assert
         assert result == expected
@@ -199,7 +199,7 @@ def test_maybe_date_to_string_give_valid_values_returns_expected_results(
         expected: str,
     ) -> None:
         # Arrange, Act
-        result: Optional[str] = maybe_date_to_string(value)
+        result: Optional[str] = optional_date_to_string(value)
 
         # Assert
         assert result == expected
@@ -220,7 +220,7 @@ def test_maybe_datetime_to_string_give_valid_values_returns_expected_results(
         expected: str,
     ) -> None:
         # Arrange, Act
-        result: Optional[str] = maybe_datetime_to_string(value)
+        result: Optional[str] = optional_datetime_to_string(value)
 
         # Assert
         assert result == expected