FIX: Use POST not GET for endpoints with symbols

Author: Zach Banks

databento/common/http.py

@@ -3,6 +3,7 @@
 import json
 import warnings
 from collections.abc import Iterable
+from collections.abc import Mapping
 from io import BytesIO
 from json.decoder import JSONDecodeError
 from os import PathLike
@@ -94,7 +95,7 @@ async def _get_json_async(
     def _post(
         self,
         url: str,
-        data: dict[str, object | None] | None = None,
+        data: Mapping[str, object | None] | None = None,
         params: Iterable[tuple[str, str | None]] | None = None,
         basic_auth: bool = False,
     ) -> Response:

databento/historical/api/metadata.py

@@ -304,22 +304,22 @@ def get_record_count(
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("symbols", ",".join(symbols_list)),
-            ("schema", str(validate_enum(schema, Schema, "schema"))),
-            ("start", optional_datetime_to_string(start)),
-            ("end", optional_datetime_to_string(end)),
-            ("stype_in", str(stype_in_valid)),
-        ]
+        data: dict[str, str | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "symbols": ",".join(symbols_list),
+            "schema": str(validate_enum(schema, Schema, "schema")),
+            "start": optional_datetime_to_string(start),
+            "end": optional_datetime_to_string(end),
+            "stype_in": str(stype_in_valid),
+        }
 
         # Optional Parameters
         if limit is not None:
-            params.append(("limit", str(limit)))
+            data["limit"] = str(limit)
 
-        response: Response = self._get(
+        response: Response = self._post(
             url=self._base_url + ".get_record_count",
-            params=params,
+            data=data,
             basic_auth=True,
         )
 
@@ -373,22 +373,22 @@ def get_billable_size(
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("start", datetime_to_string(start)),
-            ("end", optional_datetime_to_string(end)),
-            ("symbols", ",".join(symbols_list)),
-            ("schema", str(validate_enum(schema, Schema, "schema"))),
-            ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(SType.INSTRUMENT_ID)),
-        ]
+        data: dict[str, str | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "start": datetime_to_string(start),
+            "end": optional_datetime_to_string(end),
+            "symbols": ",".join(symbols_list),
+            "schema": str(validate_enum(schema, Schema, "schema")),
+            "stype_in": str(stype_in_valid),
+            "stype_out": str(SType.INSTRUMENT_ID),
+        }
 
         if limit is not None:
-            params.append(("limit", str(limit)))
+            data["limit"] = str(limit)
 
-        response: Response = self._get(
+        response: Response = self._post(
             url=self._base_url + ".get_billable_size",
-            params=params,
+            data=data,
             basic_auth=True,
         )
 
@@ -445,23 +445,23 @@ def get_cost(
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_list(symbols, stype_in_valid)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("start", datetime_to_string(start)),
-            ("end", optional_datetime_to_string(end)),
-            ("symbols", ",".join(symbols_list)),
-            ("schema", str(validate_enum(schema, Schema, "schema"))),
-            ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(SType.INSTRUMENT_ID)),
-            ("mode", validate_enum(mode, FeedMode, "mode")),
-        ]
+        data: dict[str, str | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "start": datetime_to_string(start),
+            "end": optional_datetime_to_string(end),
+            "symbols": ",".join(symbols_list),
+            "schema": str(validate_enum(schema, Schema, "schema")),
+            "stype_in": str(stype_in_valid),
+            "stype_out": str(SType.INSTRUMENT_ID),
+            "mode": validate_enum(mode, FeedMode, "mode"),
+        }
 
         if limit is not None:
-            params.append(("limit", str(limit)))
+            data["limit"] = str(limit)
 
-        response: Response = self._get(
+        response: Response = self._post(
             url=self._base_url + ".get_cost",
-            params=params,
+            data=data,
             basic_auth=True,
         )
 

tests/test_historical_metadata.py

@@ -191,7 +191,7 @@ def test_get_record_count_sends_expected_request(
     historical_client: Historical,
 ) -> None:
     # Arrange
-    monkeypatch.setattr(requests, "get", mocked_get := MagicMock())
+    monkeypatch.setattr(requests, "post", mocked_post := MagicMock())
 
     # Act
     historical_client.metadata.get_record_count(
@@ -204,20 +204,20 @@ def test_get_record_count_sends_expected_request(
     )
 
     # Assert
-    call = mocked_get.call_args.kwargs
+    call = mocked_post.call_args.kwargs
     assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/metadata.get_record_count"
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
-    assert call["params"] == [
-        ("dataset", "GLBX.MDP3"),
-        ("symbols", "ESH1"),
-        ("schema", "mbo"),
-        ("start", "2020-12-28T12:00"),
-        ("end", "2020-12-29"),
-        ("stype_in", "raw_symbol"),
-        ("limit", "1000000"),
-    ]
+    assert call["data"] == {
+        "dataset": "GLBX.MDP3",
+        "symbols": "ESH1",
+        "schema": "mbo",
+        "start": "2020-12-28T12:00",
+        "end": "2020-12-29",
+        "stype_in": "raw_symbol",
+        "limit": "1000000",
+    }
     assert call["timeout"] == (100, 100)
     assert isinstance(call["auth"], requests.auth.HTTPBasicAuth)
 
@@ -227,7 +227,7 @@ def test_get_billable_size_sends_expected_request(
     historical_client: Historical,
 ) -> None:
     # Arrange
-    monkeypatch.setattr(requests, "get", mocked_get := MagicMock())
+    monkeypatch.setattr(requests, "post", mocked_post := MagicMock())
 
     # Act
     historical_client.metadata.get_billable_size(
@@ -240,23 +240,23 @@ def test_get_billable_size_sends_expected_request(
     )
 
     # Assert
-    call = mocked_get.call_args.kwargs
+    call = mocked_post.call_args.kwargs
     assert (
         call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/metadata.get_billable_size"
     )
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
-    assert call["params"] == [
-        ("dataset", "GLBX.MDP3"),
-        ("start", "2020-12-28T12:00"),
-        ("end", "2020-12-29"),
-        ("symbols", "ESH1"),
-        ("schema", "mbo"),
-        ("stype_in", "raw_symbol"),
-        ("stype_out", "instrument_id"),
-        ("limit", "1000000"),
-    ]
+    assert call["data"] == {
+        "dataset": "GLBX.MDP3",
+        "start": "2020-12-28T12:00",
+        "end": "2020-12-29",
+        "symbols": "ESH1",
+        "schema": "mbo",
+        "stype_in": "raw_symbol",
+        "stype_out": "instrument_id",
+        "limit": "1000000",
+    }
     assert call["timeout"] == (100, 100)
     assert isinstance(call["auth"], requests.auth.HTTPBasicAuth)
 
@@ -266,7 +266,7 @@ def test_get_cost_sends_expected_request(
     historical_client: Historical,
 ) -> None:
     # Arrange
-    monkeypatch.setattr(requests, "get", mocked_get := MagicMock())
+    monkeypatch.setattr(requests, "post", mocked_post := MagicMock())
 
     # Act
     historical_client.metadata.get_cost(
@@ -279,21 +279,21 @@ def test_get_cost_sends_expected_request(
     )
 
     # Assert
-    call = mocked_get.call_args.kwargs
+    call = mocked_post.call_args.kwargs
     assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/metadata.get_cost"
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
-    assert call["params"] == [
-        ("dataset", "GLBX.MDP3"),
-        ("start", "2020-12-28T12:00"),
-        ("end", "2020-12-29"),
-        ("symbols", "ESH1"),
-        ("schema", "mbo"),
-        ("stype_in", "raw_symbol"),
-        ("stype_out", "instrument_id"),
-        ("mode", "historical-streaming"),
-        ("limit", "1000000"),
-    ]
+    assert call["data"] == {
+        "dataset": "GLBX.MDP3",
+        "start": "2020-12-28T12:00",
+        "end": "2020-12-29",
+        "symbols": "ESH1",
+        "schema": "mbo",
+        "stype_in": "raw_symbol",
+        "stype_out": "instrument_id",
+        "mode": "historical-streaming",
+        "limit": "1000000",
+    }
     assert call["timeout"] == (100, 100)
     assert isinstance(call["auth"], requests.auth.HTTPBasicAuth)