MOD: Add semantic string validation for dataset

Author: nmacholl

databento/common/validation.py

@@ -143,6 +143,36 @@ def validate_gateway(
     return urlunsplit(components=("https", url_chunks.path, "", "", ""))
 
 
+def validate_semantic_string(value: str, param: str) -> str:
+    """
+    Validate whether a string contains a semantic value.
+    A string is considered absent of meaning if:
+        - It is empty.
+        - It contains only whitespace.
+        - It contains unprintable characters.
+
+    Parameters
+    ----------
+    value: str
+        The string to validate.
+    param : str
+        The name of the parameter being validated (for any error message).
+
+    Raises
+    ------
+    ValueError
+        If the string is not meaningful.
+
+    """
+    if not value:
+        raise ValueError(f"The `{param}` cannot be an empty string.")
+    if str.isspace(value):
+        raise ValueError(f"The `{param}` cannot contain only whitepsace.")
+    if not str.isprintable(value):
+        raise ValueError(f"The `{param}` cannot contain unprintable characters.")
+    return value
+
+
 def validate_smart_symbol(symbol: str) -> str:
     """
     Validate whether symbol has a valid smart symbol format.

databento/historical/api/batch.py

@@ -26,7 +26,11 @@
     optional_symbols_list_to_string,
     optional_values_list_to_string,
 )
-from databento.common.validation import validate_enum, validate_path
+from databento.common.validation import (
+    validate_enum,
+    validate_path,
+    validate_semantic_string,
+)
 from databento.historical.api import API_VERSION
 from databento.historical.http import (
     BentoHttpAPI,
@@ -121,7 +125,7 @@ def submit_job(
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
             ("symbols", str(symbols_list)),

databento/historical/api/metadata.py

@@ -9,7 +9,11 @@
     optional_datetime_to_string,
     optional_symbols_list_to_string,
 )
-from databento.common.validation import validate_enum, validate_maybe_enum
+from databento.common.validation import (
+    validate_enum,
+    validate_maybe_enum,
+    validate_semantic_string,
+)
 from databento.historical.api import API_VERSION
 from databento.historical.http import BentoHttpAPI
 from requests import Response
@@ -110,7 +114,7 @@ def list_schemas(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start_date", optional_date_to_string(start_date)),
             ("end_date", optional_date_to_string(end_date)),
         ]
@@ -152,7 +156,7 @@ def list_fields(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("schema", validate_maybe_enum(schema, Schema, "schema")),
             ("encoding", validate_maybe_enum(encoding, Encoding, "encoding")),
         ]
@@ -226,7 +230,7 @@ def list_unit_prices(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("mode", validate_maybe_enum(mode, FeedMode, "mode")),
             ("schema", validate_maybe_enum(schema, Schema, "schema")),
         ]
@@ -268,7 +272,7 @@ def list_dataset_conditions(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start_date", optional_date_to_string(start_date)),
             ("end_date", optional_date_to_string(end_date)),
         ]
@@ -301,7 +305,7 @@ def get_dataset_range(
 
         """
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
         ]
 
         response: Response = self._get(
@@ -356,7 +360,7 @@ def get_record_count(
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("symbols", symbols_list),
             ("schema", str(validate_enum(schema, Schema, "schema"))),
             ("start", optional_datetime_to_string(start)),
@@ -421,7 +425,7 @@ def get_billable_size(
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
             ("symbols", symbols_list),
@@ -489,7 +493,7 @@ def get_cost(
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         params: List[Tuple[str, str]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
             ("symbols", symbols_list),

databento/historical/api/symbology.py

@@ -6,7 +6,7 @@
     datetime_to_date_string,
     optional_symbols_list_to_string,
 )
-from databento.common.validation import validate_enum
+from databento.common.validation import validate_enum, validate_semantic_string
 from databento.historical.api import API_VERSION
 from databento.historical.http import BentoHttpAPI
 from requests import Response
@@ -63,7 +63,7 @@ def resolve(
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         params: List[Tuple[str, str]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("symbols", symbols_list),
             ("stype_in", str(stype_in_valid)),
             ("stype_out", str(validate_enum(stype_out, SType, "stype_out"))),

databento/historical/api/timeseries.py

@@ -11,7 +11,7 @@
 from databento.common.deprecated import deprecated
 from databento.common.enums import Compression, Dataset, Encoding, Schema, SType
 from databento.common.parsing import datetime_to_string, optional_symbols_list_to_string
-from databento.common.validation import validate_enum
+from databento.common.validation import validate_enum, validate_semantic_string
 from databento.historical.api import API_VERSION
 from databento.historical.error import BentoWarning
 from databento.historical.http import BentoHttpAPI
@@ -120,7 +120,7 @@ def get_range(
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         schema_valid = validate_enum(schema, Schema, "schema")
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
             ("symbols", symbols_list),
@@ -256,7 +256,7 @@ async def get_range_async(
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
         schema_valid = validate_enum(schema, Schema, "schema")
         params: List[Tuple[str, Optional[str]]] = [
-            ("dataset", dataset),
+            ("dataset", validate_semantic_string(dataset, "dataset")),
             ("start", datetime_to_string(start)),
             ("end", datetime_to_string(end)),
             ("symbols", symbols_list),

tests/test_common_validation.py

@@ -8,6 +8,7 @@
     validate_gateway,
     validate_maybe_enum,
     validate_path,
+    validate_semantic_string,
     validate_smart_symbol,
 )
 
@@ -120,3 +121,29 @@ def test_validate_smart_symbol(
         else:
             with pytest.raises(expected):
                 validate_smart_symbol(symbol)
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    [
+        pytest.param("nick", "nick"),
+        pytest.param("", ValueError, id="empty"),
+        pytest.param(" ", ValueError, id="whitespace"),
+        pytest.param("foo\x00", ValueError, id="unprintable"),
+    ],
+)
+def test_validate_semantic_string(
+    value: str,
+    expected: Union[str, Type[Exception]],
+) -> None:
+    """
+    Test that validate_semantic_string rejects string which are:
+        - empty
+        - whitespace
+        - contain unprintable characters
+    """
+    if isinstance(expected, str):
+        assert validate_semantic_string(value, "unittest") == expected
+    else:
+        with pytest.raises(expected):
+            assert validate_semantic_string(value, "")