MOD: Support form data for POST requests

Author: cjdsellers

databento/common/parsing.py

@@ -233,7 +233,7 @@ def datetime_to_date_string(value: pd.Timestamp | date | str | int) -> str:
 
 
 def optional_datetime_to_string(
-    value: pd.Timestamp | date | str | int,
+    value: pd.Timestamp | date | str | int | None,
 ) -> str | None:
     """
     Return a valid datetime string from the given value (if not None).

databento/historical/api/batch.py

@@ -119,43 +119,34 @@ def submit_job(
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("start", datetime_to_string(start)),
-            ("end", optional_datetime_to_string(end)),
-            ("symbols", str(symbols_list)),
-            ("schema", str(validate_enum(schema, Schema, "schema"))),
-            ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(validate_enum(stype_out, SType, "stype_out"))),
-            ("encoding", str(validate_enum(encoding, Encoding, "encoding"))),
-            (
-                "compression",
-                str(validate_enum(compression, Compression, "compression"))
-                if compression
-                else None,
-            ),
-            (
-                "split_duration",
-                str(validate_enum(split_duration, SplitDuration, "split_duration")),
-            ),
-            (
-                "packaging",
-                str(validate_enum(packaging, Packaging, "packaging"))
-                if packaging
-                else None,
-            ),
-            ("delivery", str(validate_enum(delivery, Delivery, "delivery"))),
-        ]
+        data: dict[str, object | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "start": datetime_to_string(start),
+            "end": optional_datetime_to_string(end),
+            "symbols": str(symbols_list),
+            "schema": str(validate_enum(schema, Schema, "schema")),
+            "stype_in": str(stype_in_valid),
+            "stype_out": str(validate_enum(stype_out, SType, "stype_out")),
+            "encoding": str(validate_enum(encoding, Encoding, "encoding")),
+            "compression": str(validate_enum(compression, Compression, "compression"))
+            if compression
+            else None,
+            "split_duration": str(validate_enum(split_duration, SplitDuration, "split_duration")),
+            "packaging": str(validate_enum(packaging, Packaging, "packaging"))
+            if packaging
+            else None,
+            "delivery": str(validate_enum(delivery, Delivery, "delivery")),
+        }
 
         # Optional Parameters
         if limit is not None:
-            params.append(("limit", str(limit)))
+            data["limit"] = str(limit)
         if split_size is not None:
-            params.append(("split_size", str(split_size)))
+            data["split_size"] = str(split_size)
 
         return self._post(
             url=self._base_url + ".submit_job",
-            params=params,
+            data=data,
             basic_auth=True,
         ).json()
 

databento/historical/api/symbology.py

@@ -38,7 +38,7 @@ def resolve(
         """
         Request symbology mappings resolution from Databento.
 
-        Makes a `GET /symbology.resolve` HTTP request.
+        Makes a `POST /symbology.resolve` HTTP request.
 
         Parameters
         ----------
@@ -66,19 +66,19 @@ def resolve(
         """
         stype_in_valid = validate_enum(stype_in, SType, "stype_in")
         symbols_list = optional_symbols_list_to_string(symbols, stype_in_valid)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("symbols", symbols_list),
-            ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(validate_enum(stype_out, SType, "stype_out"))),
-            ("start_date", datetime_to_date_string(start_date)),
-            ("end_date", optional_date_to_string(end_date)),
-            ("default_value", default_value),
-        ]
+        data: dict[str, object | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "symbols": symbols_list,
+            "stype_in": str(stype_in_valid),
+            "stype_out": str(validate_enum(stype_out, SType, "stype_out")),
+            "start_date": datetime_to_date_string(start_date),
+            "end_date": optional_date_to_string(end_date),
+            "default_value": default_value,
+        }
 
-        response: Response = self._get(
+        response: Response = self._post(
             url=self._base_url + ".resolve",
-            params=params,
+            data=data,
             basic_auth=True,
         )
 

databento/historical/api/timeseries.py

@@ -31,7 +31,7 @@ def __init__(self, key: str, gateway: str) -> None:
 
     def get_range(
         self,
-        dataset: Dataset | str | None,
+        dataset: Dataset | str,
         start: pd.Timestamp | date | str | int,
         end: pd.Timestamp | date | str | int | None = None,
         symbols: list[str] | str | None = None,
@@ -44,7 +44,7 @@ def get_range(
         """
         Request a historical time series data stream from Databento.
 
-        Makes a `GET /timeseries.get_range` HTTP request.
+        Makes a `POST /timeseries.get_range` HTTP request.
 
         Primary method for getting historical intraday market data, daily data,
         instrument definitions and market status data directly into your application.
@@ -99,32 +99,32 @@ def get_range(
         schema_valid = validate_enum(schema, Schema, "schema")
         start_valid = datetime_to_string(start)
         end_valid = optional_datetime_to_string(end)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("start", start_valid),
-            ("end", end_valid),
-            ("symbols", symbols_list),
-            ("schema", str(schema_valid)),
-            ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(validate_enum(stype_out, SType, "stype_out"))),
-            ("encoding", str(Encoding.DBN)),  # Always request dbn
-            ("compression", str(Compression.ZSTD)),  # Always request zstd
-        ]
+        data: dict[str, object | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "start": start_valid,
+            "end": end_valid,
+            "symbols": symbols_list,
+            "schema": str(schema_valid),
+            "stype_in": str(stype_in_valid),
+            "stype_out": str(validate_enum(stype_out, SType, "stype_out")),
+            "encoding": str(Encoding.DBN),  # Always request dbn
+            "compression": str(Compression.ZSTD),  # Always request zstd
+        }
 
         # Optional Parameters
         if limit is not None:
-            params.append(("limit", str(limit)))
+            data["limit"] = str(limit)
 
         return self._stream(
             url=self._base_url + ".get_range",
-            params=params,
+            data=data,
             basic_auth=True,
             path=path,
         )
 
     async def get_range_async(
         self,
-        dataset: Dataset | str | None,
+        dataset: Dataset | str,
         start: pd.Timestamp | date | str | int,
         end: pd.Timestamp | date | str | int | None = None,
         symbols: list[str] | str | None = None,
@@ -138,7 +138,7 @@ async def get_range_async(
         Asynchronously request a historical time series data stream from
         Databento.
 
-        Makes a `GET /timeseries.get_range` HTTP request.
+        Makes a `POST /timeseries.get_range` HTTP request.
 
         Primary method for getting historical intraday market data, daily data,
         instrument definitions and market status data directly into your application.
@@ -193,25 +193,25 @@ async def get_range_async(
         schema_valid = validate_enum(schema, Schema, "schema")
         start_valid = datetime_to_string(start)
         end_valid = optional_datetime_to_string(end)
-        params: list[tuple[str, str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("start", start_valid),
-            ("end", end_valid),
-            ("symbols", symbols_list),
-            ("schema", str(schema_valid)),
-            ("stype_in", str(stype_in_valid)),
-            ("stype_out", str(validate_enum(stype_out, SType, "stype_out"))),
-            ("encoding", str(Encoding.DBN)),  # Always request dbn
-            ("compression", str(Compression.ZSTD)),  # Always request zstd
-        ]
+        data: dict[str, object | None] = {
+            "dataset": validate_semantic_string(dataset, "dataset"),
+            "start": start_valid,
+            "end": end_valid,
+            "symbols": symbols_list,
+            "schema": str(schema_valid),
+            "stype_in": str(stype_in_valid),
+            "stype_out": str(validate_enum(stype_out, SType, "stype_out")),
+            "encoding": str(Encoding.DBN),  # Always request dbn
+            "compression": str(Compression.ZSTD),  # Always request zstd
+        }
 
         # Optional Parameters
         if limit is not None:
-            params.append(("limit", str(limit)))
+            data["limit"] = str(limit)
 
         return await self._stream_async(
             url=self._base_url + ".get_range",
-            params=params,
+            data=data,
             basic_auth=True,
             path=path,
         )

databento/historical/http.py

@@ -24,7 +24,7 @@
 from databento.version import __version__
 
 
-_32KB = 1024 * 32  # 32_768
+_32KIB = 1024 * 32  # 32_768
 WARNING_HEADER_FIELD: str = "X-Warning"
 
 
@@ -94,13 +94,15 @@ async def _get_json_async(
     def _post(
         self,
         url: str,
+        data: dict[str, object | None] | None = None,
         params: Iterable[tuple[str, str | None]] | None = None,
         basic_auth: bool = False,
     ) -> Response:
         self._check_api_key()
 
         with requests.post(
             url=url,
+            data=data,
             params=params,
             headers=self._headers,
             auth=HTTPBasicAuth(username=self._key, password="") if basic_auth else None,
@@ -113,15 +115,15 @@ def _post(
     def _stream(
         self,
         url: str,
-        params: Iterable[tuple[str, str | None]],
+        data: dict[str, object | None],
         basic_auth: bool,
         path: PathLike[str] | str | None = None,
     ) -> DBNStore:
         self._check_api_key()
 
-        with requests.get(
+        with requests.post(
             url=url,
-            params=params,
+            data=data,
             headers=self._headers,
             auth=HTTPBasicAuth(username=self._key, password="") if basic_auth else None,
             timeout=(self.TIMEOUT, self.TIMEOUT),
@@ -135,7 +137,7 @@ def _stream(
             else:
                 writer = open(path, "x+b")
 
-            for chunk in response.iter_content(chunk_size=_32KB):
+            for chunk in response.iter_content(chunk_size=_32KIB):
                 writer.write(chunk)
 
             if path is None:
@@ -148,16 +150,16 @@ def _stream(
     async def _stream_async(
         self,
         url: str,
-        params: Iterable[tuple[str, str | None]],
+        data: dict[str, object | None] | None,
         basic_auth: bool,
         path: PathLike[str] | str | None = None,
     ) -> DBNStore:
         self._check_api_key()
 
         async with aiohttp.ClientSession() as session:
-            async with session.get(
+            async with session.post(
                 url=url,
-                params=[x for x in params if x[1] is not None],
+                data=data,
                 headers=self._headers,
                 auth=aiohttp.BasicAuth(login=self._key, password="", encoding="utf-8")
                 if basic_auth

tests/test_historical_batch.py

@@ -77,27 +77,25 @@ def test_batch_submit_job_sends_expected_request(
 
     # Assert
     call = mocked_post.call_args.kwargs
-    assert (
-        call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.submit_job"
-    )
+    assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.submit_job"
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
-    assert call["params"] == [
-        ("dataset", "GLBX.MDP3"),
-        ("start", "2020-12-28T12:00"),
-        ("end", "2020-12-29"),
-        ("symbols", "ESH1"),
-        ("schema", "trades"),
-        ("stype_in", "raw_symbol"),
-        ("stype_out", "instrument_id"),
-        ("encoding", "csv"),
-        ("compression", "zstd"),
-        ("split_duration", "day"),
-        ("packaging", "none"),
-        ("delivery", "download"),
-        ("split_size", "10000000000"),
-    ]
+    assert call["data"] == {
+        "dataset": "GLBX.MDP3",
+        "start": "2020-12-28T12:00",
+        "end": "2020-12-29",
+        "symbols": "ESH1",
+        "schema": "trades",
+        "stype_in": "raw_symbol",
+        "stype_out": "instrument_id",
+        "encoding": "csv",
+        "compression": "zstd",
+        "split_duration": "day",
+        "packaging": "none",
+        "delivery": "download",
+        "split_size": "10000000000",
+    }
     assert call["timeout"] == (100, 100)
     assert isinstance(call["auth"], requests.auth.HTTPBasicAuth)
 
@@ -114,9 +112,7 @@ def test_batch_list_jobs_sends_expected_request(
 
     # Assert
     call = mocked_get.call_args.kwargs
-    assert (
-        call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.list_jobs"
-    )
+    assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.list_jobs"
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
@@ -141,9 +137,7 @@ def test_batch_list_files_sends_expected_request(
 
     # Assert
     call = mocked_get.call_args.kwargs
-    assert (
-        call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.list_files"
-    )
+    assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.list_files"
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
@@ -174,9 +168,7 @@ def test_batch_download_single_file_sends_expected_request(
 
     # Assert
     call = mocked_get.call_args.kwargs
-    assert (
-        call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.list_files"
-    )
+    assert call["url"] == f"{historical_client.gateway}/v{db.API_VERSION}/batch.list_files"
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
     assert call["headers"]["accept"] == "application/json"
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))