MOD: Reimplement metadata endpoints to RFC spec

Author: cjdsellers

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## TBD
+
+This release includes improvements to the ergonomics of the clients metadata API, you can read more about the changes [here](https://databento.com/blog/api-improvements-august-2023).
+
+#### Breaking changes
+- Changed `metadata.list_publishers()` to return a list of publisher details objects
+- Changed `metadata.list_fields(...)` to return a list of field detail objects for a particular schema and encoding
+- Changed `metadata.list_fields(...)` to require the `schema` and `encoding` parameters
+- Changed `metadata.list_unit_prices(...)` to return a list of unit prices for each feed mode and data schema
+- Changed `metadata.list_unit_prices(...)` to require the `dataset` parameter
+- Removed `metadata.list_unit_prices(...)` `mode` and `schema` parameters
+- Removed `metadata.list_fields(...)` `dataset` parameter
+
 ## 0.16.1 - 2023-08-03
 
 #### Bug fixes

databento/historical/api/metadata.py

@@ -16,7 +16,6 @@
 from databento.common.parsing import optional_datetime_to_string
 from databento.common.parsing import optional_symbols_list_to_list
 from databento.common.validation import validate_enum
-from databento.common.validation import validate_maybe_enum
 from databento.common.validation import validate_semantic_string
 from databento.historical.api import API_VERSION
 from databento.historical.http import BentoHttpAPI
@@ -31,17 +30,17 @@ def __init__(self, key: str, gateway: str) -> None:
         super().__init__(key=key, gateway=gateway)
         self._base_url = gateway + f"/v{API_VERSION}/metadata"
 
-    def list_publishers(self) -> dict[str, int]:
+    def list_publishers(self) -> list[dict[str, Any]]:
         """
         Request all publishers from Databento.
 
         Makes a `GET /metadata.list_publishers` HTTP request.
 
-        Use this method to list the mappings of publisher names to publisher IDs.
+        Use this method to list the details of publishers, including their dataset and venue mappings.
 
         Returns
         -------
-        dict[str, int]
+        list[dict[str, Any]]
 
         """
         response: Response = self._get(
@@ -56,11 +55,11 @@ def list_datasets(
         end_date: date | str | None = None,
     ) -> list[str]:
         """
-        Request all available datasets from Databento.
+        Request all valid dataset codes from Databento.
 
         Makes a `GET /metadata.list_datasets` HTTP request.
 
-        Use this method to list the _names_ of all available datasets, so you
+        Use this method to list the valid dataset _codes (string identifiers), so you
         can use other methods which take the `dataset` parameter.
 
         Parameters
@@ -118,82 +117,67 @@ def list_schemas(self, dataset: Dataset | str) -> list[str]:
 
     def list_fields(
         self,
-        dataset: Dataset | str,
-        schema: Schema | str | None = None,
-        encoding: Encoding | str | None = None,
-    ) -> dict[str, dict[str, str]]:
+        schema: Schema | str,
+        encoding: Encoding | str,
+    ) -> list[dict[str, Any]]:
         """
-        Request all fields for a dataset, schema and encoding from Databento.
+        List all fields for a particular schema and encoding from Databento.
 
         Makes a `GET /metadata.list_fields` HTTP request.
 
-        The `schema` and `encoding` parameters act as optional filters. All
-        metadata for that parameter is returned if they are not specified.
-
         Parameters
         ----------
-        dataset : Dataset or str
-            The dataset code (string identifier) for the request.
-        schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, optional  # noqa
+        schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'},
             The data record schema for the request.
-        encoding : Encoding or str {'dbn', 'csv', 'json'}, optional
+        encoding : Encoding or str {'dbn', 'csv', 'json'}
             The data encoding.
 
         Returns
         -------
-        dict[str, dict[str, str]]
-            A mapping of dataset to encoding to schema to field to data type.
+        list[dict[str, Any]]
+            A list of field details.
 
         """
-        params: list[tuple[str, Dataset | Schema | Encoding | str | None]] = [
-            ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("schema", validate_maybe_enum(schema, Schema, "schema")),
-            ("encoding", validate_maybe_enum(encoding, Encoding, "encoding")),
+        params: list[tuple[str, str | Any]] = [
+            ("schema", validate_enum(schema, Schema, "schema")),
+            ("encoding", validate_enum(encoding, Encoding, "encoding")),
         ]
 
         response: Response = self._get(
             url=self._base_url + ".list_fields",
-            params=params,  # type: ignore [arg-type]
+            params=params,
             basic_auth=True,
         )
         return response.json()
 
     def list_unit_prices(
         self,
         dataset: Dataset | str,
-        mode: FeedMode | str | None = None,
-        schema: Schema | str | None = None,
-    ) -> float | dict[str, Any]:
+    ) -> list[dict[str, Any]]:
         """
-        List unit prices for each data schema in US dollars per gigabyte.
+        List unit prices for each feed mode and data schema in US dollars per
+        gigabyte.
 
         Makes a `GET /metadata.list_unit_prices` HTTP request.
 
         Parameters
         ----------
         dataset : Dataset or str
             The dataset code for the request.
-        mode : FeedMode or str {'live', 'historical-streaming', 'historical'}, optional
-            The data feed mode for the request.
-        schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, optional  # noqa
-            The data record schema for the request.
 
         Returns
         -------
-        float or dict[str, Any]
-            If both `mode` and `schema` are specified, the unit price is returned as a single number.
-            Otherwise, return a map of feed mode to schema to unit price.
+        list[dict[str, Any]]
+            A list of maps of feed mode to schema to unit price.
 
         """
-        params: list[tuple[str, Dataset | FeedMode | Schema | str | None]] = [
+        params: list[tuple[str, Dataset | str]] = [
             ("dataset", validate_semantic_string(dataset, "dataset")),
-            ("mode", validate_maybe_enum(mode, FeedMode, "mode")),
-            ("schema", validate_maybe_enum(schema, Schema, "schema")),
         ]
 
         response: Response = self._get(
             url=self._base_url + ".list_unit_prices",
-            params=params,  # type: ignore [arg-type]
+            params=params,
             basic_auth=True,
         )
         return response.json()

examples/historical_metadata.py

@@ -8,4 +8,4 @@
     print(client.metadata.list_publishers())
     print(client.metadata.list_datasets())
     print(client.metadata.list_schemas(dataset="GLBX.MDP3"))
-    print(client.metadata.list_fields(dataset="GLBX.MDP3"))
+    print(client.metadata.list_fields(schema="trades", encoding="dbn"))

examples/historical_metadata_list_unit_price.py

@@ -7,9 +7,6 @@
     key = "YOUR_API_KEY"
     client = Historical(key=key)
 
-    unit_prices = client.metadata.list_unit_prices(
-        dataset="GLBX.MDP3",
-        mode="historical-streaming",
-    )
+    unit_prices = client.metadata.list_unit_prices(dataset="GLBX.MDP3")
 
     pprint(unit_prices)

tests/test_historical_metadata.py

@@ -6,9 +6,7 @@
 import pytest
 import requests
 from databento.common.enums import Dataset
-from databento.common.enums import FeedMode
 from databento.historical.client import Historical
-from databento_dbn import Schema
 
 
 def test_list_publishers_sends_expected_request(
@@ -95,7 +93,6 @@ def test_list_fields_sends_expected_request(
 
     # Act
     historical_client.metadata.list_fields(
-        dataset="GLBX.MDP3",
         schema="mbo",
         encoding="dbn",
     )
@@ -106,7 +103,6 @@ def test_list_fields_sends_expected_request(
         call["url"]
         == f"{historical_client.gateway}/v{db.API_VERSION}/metadata.list_fields"
     )
-    assert ("dataset", "GLBX.MDP3") in call["params"]
     assert ("schema", "mbo") in call["params"]
     assert ("encoding", "dbn") in call["params"]
     assert sorted(call["headers"].keys()) == ["accept", "user-agent"]
@@ -117,28 +113,22 @@ def test_list_fields_sends_expected_request(
 
 
 @pytest.mark.parametrize(
-    "dataset, schema, mode",
+    "dataset",
     [
-        ["GLBX.MDP3", "mbo", "live"],
-        [Dataset.GLBX_MDP3, Schema.MBO, FeedMode.LIVE],
+        "GLBX.MDP3",
+        Dataset.GLBX_MDP3,
     ],
 )
 def test_list_unit_price_sends_expected_request(
     monkeypatch: pytest.MonkeyPatch,
     historical_client: Historical,
     dataset: Dataset | str,
-    schema: Schema | str,
-    mode: FeedMode | str,
 ) -> None:
     # Arrange
     monkeypatch.setattr(requests, "get", mocked_get := MagicMock())
 
     # Act
-    historical_client.metadata.list_unit_prices(
-        dataset=dataset,
-        schema=schema,
-        mode=mode,
-    )
+    historical_client.metadata.list_unit_prices(dataset=dataset)
 
     # Assert
     call = mocked_get.call_args.kwargs
@@ -151,8 +141,6 @@ def test_list_unit_price_sends_expected_request(
     assert all(v in call["headers"]["user-agent"] for v in ("Databento/", "Python/"))
     assert call["params"] == [
         ("dataset", "GLBX.MDP3"),
-        ("mode", "live"),
-        ("schema", "mbo"),
     ]
     assert call["timeout"] == (100, 100)
     assert isinstance(call["auth"], requests.auth.HTTPBasicAuth)