ADD: Custom gateway handling to HistoricalClient

Author: nmacholl

databento/common/enums.py

@@ -102,7 +102,7 @@ def __str__(self) -> str:
 class HistoricalGateway(StringyMixin, str, Enum):
     """Represents a historical data center gateway location."""
 
-    BO1 = "bo1"
+    BO1 = "https://hist.databento.com"
 
 
 @unique

databento/common/validation.py

@@ -1,5 +1,6 @@
 from enum import Enum
 from typing import Optional, Type, TypeVar, Union
+from urllib.parse import urlsplit, urlunsplit
 
 
 E = TypeVar("E", bound=Enum)
@@ -76,3 +77,37 @@ def validate_maybe_enum(
     if value is None:
         return None
     return validate_enum(value, enum, param)
+
+
+def validate_gateway(
+    url: str,
+) -> str:
+    """
+    Validate that the given value is a valid gateway URL.
+
+    Parameters
+    ----------
+    url : str
+        The URL to validate
+
+    Returns
+    -------
+    str
+        The gateway URL if it is valid.
+
+    Raises
+    ------
+    ValueError
+        If the URL is invalid.
+
+    """
+    url_chunks = urlsplit(url)
+
+    if not any([url_chunks.netloc, url_chunks.path]):
+        raise ValueError(f"`{url}` is not a valid URL")
+
+    if url_chunks.netloc:
+        return urlunsplit(
+            components=("https", url_chunks.netloc, url_chunks.path, "", ""),
+        )
+    return urlunsplit(components=("https", url_chunks.path, "", "", ""))

databento/historical/client.py

@@ -3,6 +3,7 @@
 
 from databento.common.enums import HistoricalGateway
 from databento.common.logging import log_info
+from databento.common.validation import validate_gateway
 from databento.historical.api.batch import BatchHttpAPI
 from databento.historical.api.metadata import MetadataHttpAPI
 from databento.historical.api.symbology import SymbologyHttpAPI
@@ -39,10 +40,10 @@ def __init__(
         if key is None or not isinstance(key, str) or key.isspace():
             raise ValueError(f"invalid API key, was {key}")
 
-        # Configure data access gateway
-        gateway = str(gateway)
-        if gateway == "bo1":
-            gateway = "https://hist.databento.com"
+        try:
+            gateway = HistoricalGateway(gateway)
+        except ValueError:
+            gateway = validate_gateway(str(gateway))
 
         self._key = key
         self._gateway = gateway

tests/test_common_validation.py

@@ -3,7 +3,11 @@
 
 import pytest
 from databento.common.enums import Encoding
-from databento.common.validation import validate_enum, validate_maybe_enum
+from databento.common.validation import (
+    validate_enum,
+    validate_gateway,
+    validate_maybe_enum,
+)
 
 
 class TestValidation:
@@ -47,3 +51,28 @@ def test_validate_enum_given_valid_value_returns_expected_output(
     def test_validate_maybe_enum_give_none_returns_none(self) -> None:
         # Arrange, Act, Assert
         assert validate_maybe_enum(None, Encoding, "encoding") is None
+
+    @pytest.mark.parametrize(
+        "url, expected",
+        [
+            pytest.param("databento.com", "https://databento.com"),
+            pytest.param("hist.databento.com", "https://hist.databento.com"),
+            pytest.param("http://databento.com", "https://databento.com"),
+            pytest.param("http://hist.databento.com", "https://hist.databento.com"),
+            pytest.param("//", ValueError),
+            pytest.param("", ValueError),
+        ],
+    )
+    def test_validate_gateway(
+        self,
+        url: str,
+        expected: Union[str, Type[Exception]],
+    ) -> None:
+        """
+        Tests several correct and malformed URLs.
+        """
+        if isinstance(expected, str):
+            assert validate_gateway(url) == expected
+        else:
+            with pytest.raises(expected):
+                validate_gateway(url)

tests/test_historical_client.py

@@ -55,7 +55,46 @@ def test_custom_gateway_returns_expected(self) -> None:
         client = db.Historical(key="DUMMY_API_KEY", gateway=ny4_gateway)
 
         # Assert
-        assert client.gateway == ny4_gateway
+        assert client.gateway == "https://ny4.databento.com"
+
+    @pytest.mark.parametrize(
+        "gateway",
+        [
+            "//",
+            "",
+        ],
+    )
+    def test_custom_gateway_error(
+        self,
+        gateway: str,
+    ) -> None:
+        """
+        Test that setting a custom gateway to an invalid url raises an exception.
+        """
+        # Arrange, Act, Assert
+        with pytest.raises(ValueError):
+            db.Historical(key="DUMMY_API_KEY", gateway=gateway)
+
+    @pytest.mark.parametrize(
+        "gateway, expected",
+        [
+            ["hist.databento.com", "https://hist.databento.com"],
+            ["http://hist.databento.com", "https://hist.databento.com"],
+        ],
+    )
+    def test_custom_gateway_force_https(
+        self,
+        gateway: str,
+        expected: str,
+    ) -> None:
+        """
+        Test that custom gateways are forced to the https scheme.
+        """
+        # Arrange Act
+        client = db.Historical(key="DUMMY_API_KEY", gateway=gateway)
+
+        # Assert
+        assert client.gateway == expected
 
     @pytest.mark.skipif(sys.version_info < (3, 8), reason="incompatible mocking")
     def test_re_request_symbology_makes_expected_request(